From c830dd4491dfb62b9c5846512d5a797f2f4b426f Mon Sep 17 00:00:00 2001 From: gitsad Date: Wed, 25 Mar 2026 14:26:36 +0100 Subject: [PATCH 1/9] fix: improved validator and checking behaviour --- demo/src/ValidatorView.tsx | 142 +++++++------- demo/src/styles.css | 38 ++++ demo/src/validator-prompts.ts | 174 ++++++++++++++++++ .../src/transform/validate-component.ts | 18 +- packages/parser/tests/plugin.test.ts | 13 +- packages/validator/src/constants.ts | 17 ++ .../validator/src/fixes/action-references.ts | 10 +- packages/validator/src/fixes/id-format.ts | 13 +- .../validator/src/fixes/schema-defaults.ts | 24 ++- .../validator/src/rules/action-references.ts | 18 +- .../validator/src/rules/binding-resolution.ts | 55 +++++- .../validator/src/rules/chart-validation.ts | 99 ++++++++++ .../validator/src/rules/field-name-typos.ts | 63 +++++++ packages/validator/src/rules/flow-ordering.ts | 102 ++++++++++ packages/validator/src/rules/index.ts | 19 +- .../src/rules/placeholder-content.ts | 84 +++++++++ .../validator/src/rules/schema-conformance.ts | 8 +- .../validator/src/rules/select-options.ts | 64 +++++++ .../validator/src/rules/table-data-keys.ts | 74 ++++++++ .../src/rules/unreferenced-components.ts | 75 ++++++++ packages/validator/src/types.ts | 11 +- .../tests/rules/action-references.test.ts | 76 +++++++- .../tests/rules/binding-resolution.test.ts | 102 ++++++++++ .../tests/rules/chart-validation.test.ts | 90 +++++++++ .../tests/rules/field-name-typos.test.ts | 72 ++++++++ .../tests/rules/flow-ordering.test.ts | 65 +++++++ .../tests/rules/placeholder-content.test.ts | 88 +++++++++ .../tests/rules/schema-conformance.test.ts | 49 +++++ .../tests/rules/select-options.test.ts | 83 +++++++++ .../tests/rules/table-data-keys.test.ts | 89 +++++++++ .../rules/unreferenced-components.test.ts | 77 ++++++++ packages/validator/tests/validate.test.ts | 20 ++ 32 files changed, 1803 insertions(+), 129 deletions(-) create mode 100644 demo/src/validator-prompts.ts create mode 100644 packages/validator/src/constants.ts create mode 100644 packages/validator/src/rules/chart-validation.ts create mode 100644 packages/validator/src/rules/field-name-typos.ts create mode 100644 packages/validator/src/rules/flow-ordering.ts create mode 100644 packages/validator/src/rules/placeholder-content.ts create mode 100644 packages/validator/src/rules/select-options.ts create mode 100644 packages/validator/src/rules/table-data-keys.ts create mode 100644 packages/validator/src/rules/unreferenced-components.ts create mode 100644 packages/validator/tests/rules/chart-validation.test.ts create mode 100644 packages/validator/tests/rules/field-name-typos.test.ts create mode 100644 packages/validator/tests/rules/flow-ordering.test.ts create mode 100644 packages/validator/tests/rules/placeholder-content.test.ts create mode 100644 packages/validator/tests/rules/select-options.test.ts create mode 100644 packages/validator/tests/rules/table-data-keys.test.ts create mode 100644 packages/validator/tests/rules/unreferenced-components.test.ts diff --git a/demo/src/ValidatorView.tsx b/demo/src/ValidatorView.tsx index da9b59d..24b155a 100644 --- a/demo/src/ValidatorView.tsx +++ b/demo/src/ValidatorView.tsx @@ -5,24 +5,7 @@ import { ChatMessage } from './chat/ChatMessage.js'; import { ChatInput } from './chat/ChatInput.js'; import { validate, type ValidationResult, type ValidationIssue } from '@mobile-reality/mdma-validator'; import { customizations } from './custom-components.js'; - -const VALIDATOR_PROMPT = `You are an AI assistant for testing the MDMA validator. -Generate MDMA components with various issues so the validator can demonstrate its capabilities. - -IMPORTANT: When the user asks you to generate MDMA content, you MUST intentionally introduce bugs and issues into your output so the validator can catch and fix them. Mix in these problems across your components: - -1. **Duplicate IDs** — Use the same id for two different components (e.g. two callouts both with id: notice) -2. **Bad ID format** — Use camelCase or snake_case IDs instead of kebab-case (e.g. id: myForm, id: user_table) -3. **Missing sensitive flags** — Include PII fields like email, phone, ssn, address without sensitive: true -4. **Missing thinking block** — Omit the thinking block entirely -5. **Bad binding syntax** — Use single braces {var.path} instead of {{var.path}}, or add extra whitespace {{ var.path }} -6. **Empty callout content** — Create a callout with content: "" or omit the content field -7. **Missing table headers** — Define table columns with just key: but no header: -8. **Missing form labels** — Define form fields with just name: but no label: -9. **YAML document separators** — Add --- at the end of an mdma block -10. **Bare binding in table data** — Use data: some-component.rows instead of data: "{{some-component.rows}}" - -Try to include at least 4-5 different issues in each response. The goal is to stress-test the validator's detection and auto-fix capabilities. Generate real, useful-looking components (forms, tables, callouts, etc.) — just with these intentional mistakes baked in.`; +import { VALIDATOR_PROMPT_VARIANTS } from './validator-prompts.js'; function severityClass(severity: string): string { if (severity === 'error') return 'validator-severity--error'; @@ -120,7 +103,9 @@ function ValidationPanel({ results }: { results: Map } ); } -export function ValidatorView() { +function ValidatorChatInner({ promptKey }: { promptKey: string }) { + const variant = VALIDATOR_PROMPT_VARIANTS.find((v) => v.key === promptKey)!; + const { config, messages, @@ -136,8 +121,8 @@ export function ValidatorView() { clear, updateMessage, } = useChat({ - systemPrompt: VALIDATOR_PROMPT, - storageKey: 'validator', + systemPrompt: variant.prompt, + storageKey: `validator-${promptKey}`, ...(customizations.schemas && { parserOptions: { customSchemas: customizations.schemas } }), }); @@ -185,61 +170,84 @@ export function ValidatorView() { const lastMsgId = messages[messages.length - 1]?.id; + return ( +
+
+ + +
+ {messages.length === 0 && ( +
+

{variant.label}

+

{variant.description}

+

+ Rules tested: {variant.rules.join(', ')} +

+
+ )} + + {messages.map((msg) => ( + + ))} + + {error &&
{error}
} + +
+
+ + 0} + inputRef={inputRef} + /> +
+ + +
+ ); +} + +export function ValidatorView() { + const [activeVariant, setActiveVariant] = useState('all'); + return (
Validator - Chat with the AI — every response is automatically validated by @mobile-reality/mdma-validator. Issues and auto-fixes appear on the right. + Chat with the AI — every response is automatically validated. Issues and auto-fixes appear on the right.
-
-
- - -
- {messages.length === 0 && ( -
-

Validator Chat

-

- Ask the AI to generate MDMA content. Each response will be validated automatically. -

-
- )} - - {messages.map((msg) => ( - - ))} - - {error &&
{error}
} - -
-
- - 0} - inputRef={inputRef} - /> -
- - +
+ {VALIDATOR_PROMPT_VARIANTS.map((v) => ( + + ))}
+ +
); } diff --git a/demo/src/styles.css b/demo/src/styles.css index 8edc112..949b43e 100644 --- a/demo/src/styles.css +++ b/demo/src/styles.css @@ -2356,6 +2356,44 @@ body { color: #2980b9; } +.validator-variant-selector { + display: flex; + gap: 6px; + padding: 8px 16px; + overflow-x: auto; + border-bottom: 1px solid #e5e7eb; + background: #f9fafb; + flex-shrink: 0; +} + +.validator-variant-btn { + padding: 5px 12px; + border: 1px solid #d1d5db; + border-radius: 6px; + background: #fff; + color: #374151; + font-size: 12px; + cursor: pointer; + white-space: nowrap; + transition: all 0.15s; +} + +.validator-variant-btn:hover { + background: #f3f4f6; + border-color: #9ca3af; +} + +.validator-variant-btn--active { + background: #3498db; + color: #fff; + border-color: #3498db; +} + +.validator-variant-btn--active:hover { + background: #2980b9; + border-color: #2980b9; +} + .validator-content { display: flex; flex: 1; diff --git a/demo/src/validator-prompts.ts b/demo/src/validator-prompts.ts new file mode 100644 index 0000000..1ca2808 --- /dev/null +++ b/demo/src/validator-prompts.ts @@ -0,0 +1,174 @@ +export interface ValidatorPromptVariant { + key: string; + label: string; + description: string; + rules: string[]; + prompt: string; +} + +const PREAMBLE = `You are an AI assistant for testing the MDMA validator. +Generate MDMA components with intentional issues so the validator can demonstrate its detection and auto-fix capabilities. +Generate real, useful-looking components — just with the specified intentional mistakes baked in.`; + +export const VALIDATOR_PROMPT_VARIANTS: ValidatorPromptVariant[] = [ + { + key: 'all', + label: 'All Rules', + description: 'Stress-test all validator rules at once', + rules: [ + 'yaml-correctness', 'schema-conformance', 'duplicate-ids', 'id-format', + 'binding-syntax', 'binding-resolution', 'action-references', 'sensitive-flags', + 'required-markers', 'thinking-block', 'select-options', 'placeholder-content', + 'field-name-typos', 'table-data-keys', 'chart-validation', + 'unreferenced-components', 'flow-ordering', + ], + prompt: `${PREAMBLE} + +Mix in as many of these problems as possible across your components: + +1. **Duplicate IDs** — Use the same id for two different components +2. **Bad ID format** — Use camelCase or snake_case IDs instead of kebab-case (e.g. id: myForm, id: user_table) +3. **Missing sensitive flags** — Include PII fields like email, phone, ssn, address without sensitive: true +4. **Missing thinking block** — Omit the thinking block entirely +5. **Bad binding syntax** — Use single braces {var.path} instead of {{var.path}}, or add extra whitespace {{ var.path }} +6. **Empty callout content** — Create a callout with content: "" or omit the content field +7. **Missing table headers** — Define table columns with just key: but no header: +8. **Missing form labels** — Define form fields with just name: but no label: +9. **YAML document separators** — Add --- at the end of an mdma block +10. **Bare binding in table data** — Use data: some-component.rows instead of data: "{{some-component.rows}}" +11. **Select fields without options** — Create a form select field without any options defined +12. **Placeholder content** — Use "TODO", "TBD", "...", or "Lorem ipsum" as field labels or content +13. **Field name typos** — Use "roles" instead of "allowedRoles" on approval-gate, "onClick" instead of "onAction" on button +14. **Table data key mismatch** — Table data rows with keys that don't match defined columns +15. **Invalid chart axes** — Chart with xAxis or yAxis referencing columns not in the CSV data +16. **Invalid action targets** — Use onSubmit/onAction pointing to non-existent component IDs +17. **Unreferenced components** — Add a component that no other component references via bindings or actions +18. **Backward references** — Make action targets point to components defined earlier in the document +19. **Deep binding misses** — Use bindings like {{form.nonexistent_field}} where the field doesn't exist on the form + +Try to include at least 8-10 different issues in each response.`, + }, + { + key: 'structure', + label: 'Structure & YAML', + description: 'YAML correctness, duplicate IDs, ID format, schema conformance', + rules: ['yaml-correctness', 'schema-conformance', 'duplicate-ids', 'id-format'], + prompt: `${PREAMBLE} + +Focus ONLY on these structural issues: + +1. **Duplicate IDs** — Use the same id for two or more components (e.g. two callouts both with id: notice) +2. **Bad ID format** — Use camelCase (myForm), snake_case (user_table), or UPPERCASE (LOUD_BTN) instead of kebab-case +3. **YAML document separators** — Add --- at the end of mdma blocks +4. **Missing required fields** — Omit required fields like "text" on buttons or "content" on callouts +5. **Unknown component types** — Use a type like "card" or "panel" that doesn't exist in the registry +6. **Missing form labels** — Define form fields with just name: but no label: +7. **Missing table headers** — Define table columns with just key: but no header: + +Generate a document with at least 5-6 components that contains multiple structural issues. Include forms, tables, callouts, and buttons.`, + }, + { + key: 'bindings', + label: 'Bindings & References', + description: 'Binding syntax, resolution, deep field validation, action references', + rules: ['binding-syntax', 'binding-resolution', 'action-references'], + prompt: `${PREAMBLE} + +Focus ONLY on binding and reference issues: + +1. **Bad binding syntax** — Use single braces {form.email} instead of {{form.email}} +2. **Whitespace in bindings** — Use {{ form.email }} with extra spaces inside the braces +3. **Empty bindings** — Use {{}} with nothing inside +4. **Non-existent component references** — Use bindings like {{missing_form.email}} where missing_form doesn't exist +5. **Deep binding mismatches** — Use {{myform.nonexistent}} where myform exists but has no field named "nonexistent" +6. **Invalid onSubmit targets** — Set form onSubmit to a non-existent component ID +7. **Invalid onAction targets** — Set button onAction to a non-existent component ID +8. **Invalid webhook trigger** — Set webhook trigger to a non-existent component ID + +Generate a multi-component document (form, table, callout, button, webhook) with bindings between them — but intentionally make the binding paths and action targets wrong.`, + }, + { + key: 'pii', + label: 'PII & Sensitive Data', + description: 'Sensitive flags, required markers', + rules: ['sensitive-flags', 'required-markers'], + prompt: `${PREAMBLE} + +Focus ONLY on PII and data sensitivity issues: + +1. **Missing sensitive flags on form fields** — Include fields named email, phone, ssn, address, card_number, date_of_birth without sensitive: true +2. **Missing sensitive flags on table columns** — Include table columns with keys like email, phone, address without sensitive: true +3. **Missing required on important fields** — Fields named "name", "email", "title" should typically be required but omit the required flag + +Generate a comprehensive form (like a user registration or KYC form) with many PII fields and a table displaying user data — but forget to mark any of them as sensitive or required.`, + }, + { + key: 'forms', + label: 'Form Validation', + description: 'Select options, field name typos, placeholder content', + rules: ['select-options', 'field-name-typos', 'placeholder-content'], + prompt: `${PREAMBLE} + +Focus ONLY on form-specific issues: + +1. **Select fields without options** — Create select fields with no options array at all +2. **Malformed select options** — Create select options as plain strings or objects missing label/value +3. **Placeholder labels** — Use "TODO", "TBD", "...", or "Lorem ipsum" as form field labels +4. **Placeholder content** — Use "FIXME" or "sample" as callout content or component titles +5. **Field name typos** — On buttons use "onClick" instead of "onAction", on forms use "submit" instead of "onSubmit" + +Generate a multi-step form (like an application form) with multiple select fields, text inputs, and buttons — but with these mistakes throughout.`, + }, + { + key: 'tables-charts', + label: 'Tables & Charts', + description: 'Table data keys, chart axis validation', + rules: ['table-data-keys', 'chart-validation'], + prompt: `${PREAMBLE} + +Focus ONLY on table and chart data issues: + +1. **Table data key mismatch** — Define table columns as [name, email, role] but include data rows with keys like [full_name, mail, position] that don't match +2. **Missing column data** — Define a column that no data row populates +3. **Extra data keys** — Include data row keys that aren't in the columns +4. **Invalid chart xAxis** — Set xAxis to a header name that doesn't exist in the CSV data +5. **Invalid chart yAxis** — Set yAxis to header names that don't exist in the CSV +6. **Chart with only headers** — Provide CSV data with only a header row and no data rows +7. **Bare table data binding** — Use data: some-component.rows without wrapping in {{ }} + +Generate a dashboard with 2 tables and 2 charts — a sales summary and a user analytics view — but with mismatched column/data keys and wrong axis references.`, + }, + { + key: 'flow', + label: 'Flow & References', + description: 'Flow ordering, unreferenced components, action targets', + rules: ['flow-ordering', 'unreferenced-components', 'action-references'], + prompt: `${PREAMBLE} + +Focus ONLY on component flow and reference issues: + +1. **Backward action references** — Make onSubmit or onAction point to a component defined EARLIER in the document (target appears before the source) +2. **Circular references** — Create a cycle: component A's onSubmit points to B, and B's onAction points back to A +3. **Unreferenced components** — Add a callout or table that no other component references via bindings or action fields (orphan component) +4. **Invalid action targets** — Use onSubmit, onAction, onComplete, onApprove, onDeny pointing to IDs that don't exist + +Generate a multi-step workflow (form submission → approval → notification) with 6+ components — but intentionally create circular dependencies, orphaned components, and broken action chains.`, + }, + { + key: 'approval', + label: 'Approval & Webhooks', + description: 'Field name typos on approval-gate, action references on webhooks', + rules: ['field-name-typos', 'action-references', 'schema-conformance'], + prompt: `${PREAMBLE} + +Focus ONLY on approval gate and webhook issues: + +1. **"roles" instead of "allowedRoles"** — Use the field name "roles" on approval-gate components +2. **"approvers" instead of "requiredApprovers"** — Use the wrong field name for the approver count +3. **Invalid webhook trigger** — Set webhook trigger to a component ID that doesn't exist +4. **Invalid onApprove/onDeny targets** — Point approval gate actions to non-existent components +5. **Missing required fields** — Omit the required "title" field on approval-gate or "url" on webhook + +Generate an expense approval workflow with: a form for expense details, an approval-gate for manager review, a webhook for notification, and callouts for status — but use the wrong field names and broken references.`, + }, +]; diff --git a/packages/parser/src/transform/validate-component.ts b/packages/parser/src/transform/validate-component.ts index 83b887f..bb9a198 100644 --- a/packages/parser/src/transform/validate-component.ts +++ b/packages/parser/src/transform/validate-component.ts @@ -52,17 +52,17 @@ export function validateComponent( }; } - // Check if it's a known core type + // Unknown core type — pass through as a generic component so the renderer + // can display a proper "Unknown component type" fallback instead of a + // loading skeleton. if (!componentSchemaRegistry.has(type)) { return { - ok: false, - errors: [ - new MdmaParseError( - `Unknown component type: "${type}"`, - ErrorCodes.UNKNOWN_COMPONENT_TYPE, - position, - ), - ], + ok: true, + component: { + id: typeof data.id === 'string' ? data.id : `unknown-${type}`, + type, + ...data, + } as MdmaComponent, }; } diff --git a/packages/parser/tests/plugin.test.ts b/packages/parser/tests/plugin.test.ts index f967fc0..5126010 100644 --- a/packages/parser/tests/plugin.test.ts +++ b/packages/parser/tests/plugin.test.ts @@ -95,10 +95,17 @@ describe('remarkMdma plugin', () => { expect(hasFieldsError).toBe(true); }); - it('reports unknown component type', () => { - const { messages } = parse(fixture('invalid-schema.md')); + it('passes through unknown component type as generic block', () => { + const { root, messages } = parse(fixture('invalid-schema.md')); + // Unknown types are now passed through as generic blocks (no error) const hasUnknown = messages.some((m) => m.includes('Unknown component type')); - expect(hasUnknown).toBe(true); + expect(hasUnknown).toBe(false); + // The unknown type block should still appear in the AST as an mdmaBlock + const blocks = root.children.filter( + (c: { type: string; component?: { type: string } }) => + c.type === 'mdmaBlock' && c.component?.type === 'super-custom-thing', + ); + expect(blocks).toHaveLength(1); }); }); diff --git a/packages/validator/src/constants.ts b/packages/validator/src/constants.ts new file mode 100644 index 0000000..0ec3591 --- /dev/null +++ b/packages/validator/src/constants.ts @@ -0,0 +1,17 @@ +/** + * Maps component types to their fields that are cross-references to other component IDs. + * Used by action-references rule, unreferenced-components rule, flow-ordering rule, + * and the action-references fix. + */ +export const ACTION_REFERENCE_FIELDS: Record = { + form: ['onSubmit'], + button: ['onAction'], + tasklist: ['onComplete'], + 'approval-gate': ['onApprove', 'onDeny'], + webhook: ['trigger'], +}; + +/** Flat list of all action reference field names (for id-format fix updates). */ +export const ACTION_FIELD_NAMES: string[] = [ + ...new Set(Object.values(ACTION_REFERENCE_FIELDS).flat()), +]; diff --git a/packages/validator/src/fixes/action-references.ts b/packages/validator/src/fixes/action-references.ts index 880a14d..2228989 100644 --- a/packages/validator/src/fixes/action-references.ts +++ b/packages/validator/src/fixes/action-references.ts @@ -1,13 +1,9 @@ import type { FixContext } from '../types.js'; +import { ACTION_REFERENCE_FIELDS } from '../constants.js'; /** - * Only webhook.trigger is a true cross-reference that can be invalid. - * Remove invalid trigger references from webhooks. + * Remove invalid cross-reference field values for all component types. */ -const CROSS_REFERENCE_FIELDS: Record = { - webhook: ['trigger'], -}; - export function fixActionReferences(context: FixContext): void { for (const issue of context.issues) { if (issue.ruleId !== 'action-references' || issue.fixed) continue; @@ -21,7 +17,7 @@ export function fixActionReferences(context: FixContext): void { const type = block.data.type; if (typeof type !== 'string') continue; - const fields = CROSS_REFERENCE_FIELDS[type]; + const fields = ACTION_REFERENCE_FIELDS[type]; if (!fields || !fields.includes(field)) continue; // Remove the invalid cross-reference field diff --git a/packages/validator/src/fixes/id-format.ts b/packages/validator/src/fixes/id-format.ts index b15c339..19bb7bf 100644 --- a/packages/validator/src/fixes/id-format.ts +++ b/packages/validator/src/fixes/id-format.ts @@ -1,4 +1,5 @@ import type { FixContext } from '../types.js'; +import { ACTION_FIELD_NAMES } from '../constants.js'; export function toKebabCase(id: string): string { return id @@ -10,16 +11,6 @@ export function toKebabCase(id: string): string { .replace(/-{2,}/g, '-'); } -/** Action handler fields that may reference other component IDs */ -const ACTION_FIELDS = [ - 'onSubmit', - 'onAction', - 'onComplete', - 'onApprove', - 'onDeny', - 'trigger', -]; - export function fixIdFormat(context: FixContext): void { const idRenames = new Map(); // old -> new @@ -47,7 +38,7 @@ export function fixIdFormat(context: FixContext): void { if (block.data === null) continue; // Update action reference fields - for (const field of ACTION_FIELDS) { + for (const field of ACTION_FIELD_NAMES) { if (typeof block.data[field] === 'string') { const newId = idRenames.get(block.data[field] as string); if (newId) { diff --git a/packages/validator/src/fixes/schema-defaults.ts b/packages/validator/src/fixes/schema-defaults.ts index 4c7d51c..fb5e996 100644 --- a/packages/validator/src/fixes/schema-defaults.ts +++ b/packages/validator/src/fixes/schema-defaults.ts @@ -157,6 +157,21 @@ function patchCalloutContent(data: Record): void { data.content = 'Information'; } +/** + * Pre-fix button: fill missing `text` from `id`. + */ +function patchButtonText(data: Record): void { + if (data.type !== 'button') return; + if (typeof data.text === 'string' && data.text.trim().length > 0) return; + + if (typeof data.id === 'string') { + data.text = keyToHeader(data.id); + return; + } + + data.text = 'Button'; +} + /** * Component-level properties that are Zod defaults and should be stripped * from the output to keep it concise. Key = property name, value = default value. @@ -227,7 +242,13 @@ export function fixSchemaDefaults(context: FixContext): void { const type = block.data.type; if (typeof type !== 'string') continue; - const schema = componentSchemaRegistry.get(type); + let schema = componentSchemaRegistry.get(type); + + // Fall back to custom schemas for types not in the built-in registry + if (!schema && context.options.customSchemas?.[type]) { + schema = context.options.customSchemas[type] as import('zod').ZodType; + } + if (!schema) continue; // Patch known gaps before Zod re-parse @@ -235,6 +256,7 @@ export function fixSchemaDefaults(context: FixContext): void { patchTableData(block.data); patchFormFields(block.data); patchCalloutContent(block.data); + patchButtonText(block.data); // Re-parse with Zod to apply defaults and coercions const result = schema.safeParse(block.data); diff --git a/packages/validator/src/rules/action-references.ts b/packages/validator/src/rules/action-references.ts index 4b1cee9..d859d9f 100644 --- a/packages/validator/src/rules/action-references.ts +++ b/packages/validator/src/rules/action-references.ts @@ -1,23 +1,11 @@ import type { ValidationRule } from '../types.js'; - -/** - * Fields that are cross-references to other component IDs. - * Only `webhook.trigger` is a true cross-reference — it must point to - * an existing component's action. - * - * Fields like form.onSubmit, button.onAction, tasklist.onComplete are - * action identifiers (event names), NOT references to other components. - * They are always valid as-is and should not be flagged. - */ -const CROSS_REFERENCE_FIELDS: Record = { - webhook: ['trigger'], -}; +import { ACTION_REFERENCE_FIELDS } from '../constants.js'; export const actionReferencesRule: ValidationRule = { id: 'action-references', name: 'Action References', description: - 'Checks that cross-reference fields (e.g. webhook trigger) reference valid component IDs', + 'Checks that action and cross-reference fields (onSubmit, onAction, onComplete, onApprove, onDeny, trigger) reference valid component IDs', defaultSeverity: 'warning', validate(context) { @@ -30,7 +18,7 @@ export const actionReferencesRule: ValidationRule = { const id = typeof block.data.id === 'string' ? block.data.id : null; - const fields = CROSS_REFERENCE_FIELDS[type]; + const fields = ACTION_REFERENCE_FIELDS[type]; if (!fields) continue; for (const field of fields) { diff --git a/packages/validator/src/rules/binding-resolution.ts b/packages/validator/src/rules/binding-resolution.ts index 068f990..f393e22 100644 --- a/packages/validator/src/rules/binding-resolution.ts +++ b/packages/validator/src/rules/binding-resolution.ts @@ -5,7 +5,7 @@ export const bindingResolutionRule: ValidationRule = { id: 'binding-resolution', name: 'Binding Resolution', description: - 'Checks that binding expressions reference existing components in the document', + 'Checks that binding expressions reference existing components and valid fields', defaultSeverity: 'warning', validate(context) { @@ -17,8 +17,8 @@ export const bindingResolutionRule: ValidationRule = { const bindings = extractBindings(id ?? '', block.data); for (const binding of bindings) { - // The root segment of the path should reference a known component ID - const rootSegment = binding.path.split('.')[0]; + const segments = binding.path.split('.'); + const rootSegment = segments[0]; if (!context.idMap.has(rootSegment)) { // Check for near-matches (e.g., user_form vs user-form) @@ -41,6 +41,55 @@ export const bindingResolutionRule: ValidationRule = { blockIndex: block.index, fixed: false, }); + continue; + } + + // Deep field validation: check sub-path against known component structure + if (segments.length > 1) { + const targetBlockIndex = context.idMap.get(rootSegment)!; + const targetBlock = context.blocks[targetBlockIndex]; + if (!targetBlock?.data) continue; + + const subField = segments[1]; + const targetType = targetBlock.data.type; + + if (targetType === 'form' && Array.isArray(targetBlock.data.fields)) { + const fieldNames = new Set(); + for (const f of targetBlock.data.fields) { + if (typeof f === 'object' && f !== null && typeof f.name === 'string') { + fieldNames.add(f.name); + } + } + if (fieldNames.size > 0 && !fieldNames.has(subField)) { + context.issues.push({ + ruleId: 'binding-resolution', + severity: 'info', + message: `Binding "{{${binding.path}}}" in ${binding.field}: form "${rootSegment}" has no field named "${subField}" (available: ${[...fieldNames].join(', ')})`, + componentId: id, + field: binding.field, + blockIndex: block.index, + fixed: false, + }); + } + } else if (targetType === 'table' && Array.isArray(targetBlock.data.columns)) { + const columnKeys = new Set(); + for (const col of targetBlock.data.columns) { + if (typeof col === 'object' && col !== null && typeof col.key === 'string') { + columnKeys.add(col.key); + } + } + if (columnKeys.size > 0 && !columnKeys.has(subField)) { + context.issues.push({ + ruleId: 'binding-resolution', + severity: 'info', + message: `Binding "{{${binding.path}}}" in ${binding.field}: table "${rootSegment}" has no column named "${subField}" (available: ${[...columnKeys].join(', ')})`, + componentId: id, + field: binding.field, + blockIndex: block.index, + fixed: false, + }); + } + } } } } diff --git a/packages/validator/src/rules/chart-validation.ts b/packages/validator/src/rules/chart-validation.ts new file mode 100644 index 0000000..a07486c --- /dev/null +++ b/packages/validator/src/rules/chart-validation.ts @@ -0,0 +1,99 @@ +import type { ValidationRule } from '../types.js'; + +const BINDING_PATTERN = /^\{\{.*\}\}$/s; + +function parseCsvHeaders(csv: string): string[] { + const lines = csv.trim().split('\n'); + if (lines.length === 0) return []; + return lines[0].split(',').map((h) => h.trim()).filter(Boolean); +} + +export const chartValidationRule: ValidationRule = { + id: 'chart-validation', + name: 'Chart Validation', + description: + 'Validates chart data format and axis references against CSV headers', + defaultSeverity: 'warning', + + validate(context) { + for (const block of context.blocks) { + if (block.data === null) continue; + if (block.data.type !== 'chart') continue; + + const id = + typeof block.data.id === 'string' ? block.data.id : null; + const data = block.data.data; + + if (typeof data !== 'string') continue; + + // Skip binding expressions + if (BINDING_PATTERN.test(data)) continue; + + // Treat as CSV + const lines = data.trim().split('\n'); + if (lines.length < 2) { + context.issues.push({ + ruleId: 'chart-validation', + severity: 'warning', + message: 'Chart data does not appear to be valid CSV (expected at least a header row and one data row)', + componentId: id, + field: 'data', + blockIndex: block.index, + fixed: false, + }); + continue; + } + + const headers = parseCsvHeaders(data); + if (headers.length === 0) { + context.issues.push({ + ruleId: 'chart-validation', + severity: 'warning', + message: 'Chart CSV data has no recognizable headers', + componentId: id, + field: 'data', + blockIndex: block.index, + fixed: false, + }); + continue; + } + + const headerSet = new Set(headers); + + // Validate xAxis + if (typeof block.data.xAxis === 'string' && !headerSet.has(block.data.xAxis)) { + context.issues.push({ + ruleId: 'chart-validation', + severity: 'warning', + message: `xAxis "${block.data.xAxis}" does not match any CSV header (available: ${headers.join(', ')})`, + componentId: id, + field: 'xAxis', + blockIndex: block.index, + fixed: false, + }); + } + + // Validate yAxis + const yAxis = block.data.yAxis; + const yAxes: string[] = typeof yAxis === 'string' + ? [yAxis] + : Array.isArray(yAxis) + ? yAxis.filter((v): v is string => typeof v === 'string') + : []; + + for (const axis of yAxes) { + if (!headerSet.has(axis)) { + context.issues.push({ + ruleId: 'chart-validation', + severity: 'warning', + message: `yAxis "${axis}" does not match any CSV header (available: ${headers.join(', ')})`, + componentId: id, + field: 'yAxis', + blockIndex: block.index, + fixed: false, + }); + } + } + } + }, +}; diff --git a/packages/validator/src/rules/field-name-typos.ts b/packages/validator/src/rules/field-name-typos.ts new file mode 100644 index 0000000..c86cc26 --- /dev/null +++ b/packages/validator/src/rules/field-name-typos.ts @@ -0,0 +1,63 @@ +import type { ValidationRule } from '../types.js'; + +/** + * Known field name typos per component type. + * Maps incorrect field name → correct field name. + */ +const TYPO_MAP: Record> = { + 'approval-gate': { + roles: 'allowedRoles', + role: 'allowedRoles', + approvers: 'requiredApprovers', + }, + table: { + field: 'key', + label: 'header', + }, + form: { + submit: 'onSubmit', + action: 'onAction', + }, + button: { + action: 'onAction', + click: 'onAction', + onClick: 'onAction', + }, +}; + +export const fieldNameTyposRule: ValidationRule = { + id: 'field-name-typos', + name: 'Field Name Typos', + description: + 'Detects common field name mistakes in component definitions (e.g. "roles" instead of "allowedRoles")', + defaultSeverity: 'warning', + + validate(context) { + for (const block of context.blocks) { + if (block.data === null) continue; + + const type = block.data.type; + if (typeof type !== 'string') continue; + + const typos = TYPO_MAP[type]; + if (!typos) continue; + + const id = + typeof block.data.id === 'string' ? block.data.id : null; + + for (const [wrong, correct] of Object.entries(typos)) { + if (wrong in block.data) { + context.issues.push({ + ruleId: 'field-name-typos', + severity: 'warning', + message: `Field "${wrong}" is likely a typo — did you mean "${correct}"?`, + componentId: id, + field: wrong, + blockIndex: block.index, + fixed: false, + }); + } + } + } + }, +}; diff --git a/packages/validator/src/rules/flow-ordering.ts b/packages/validator/src/rules/flow-ordering.ts new file mode 100644 index 0000000..8115a89 --- /dev/null +++ b/packages/validator/src/rules/flow-ordering.ts @@ -0,0 +1,102 @@ +import type { ValidationRule } from '../types.js'; +import { ACTION_REFERENCE_FIELDS } from '../constants.js'; + +export const flowOrderingRule: ValidationRule = { + id: 'flow-ordering', + name: 'Flow Ordering', + description: + 'Checks that action targets reference components defined later in the document and detects circular references', + defaultSeverity: 'info', + + validate(context) { + // Build adjacency list for cycle detection + const graph = new Map(); + + for (const block of context.blocks) { + if (block.data === null) continue; + + const type = block.data.type; + if (typeof type !== 'string') continue; + + const sourceId = + typeof block.data.id === 'string' ? block.data.id : null; + + const fields = ACTION_REFERENCE_FIELDS[type]; + if (!fields) continue; + + for (const field of fields) { + const targetId = block.data[field]; + if (typeof targetId !== 'string') continue; + + // Only check targets that are known component IDs + const targetIndex = context.idMap.get(targetId); + if (targetIndex === undefined) continue; + + // Check forward reference: target should be defined after source + if (targetIndex <= block.index) { + context.issues.push({ + ruleId: 'flow-ordering', + severity: 'info', + message: `Action target "${targetId}" in ${field} is defined before the referencing component (backward reference)`, + componentId: sourceId, + field, + blockIndex: block.index, + fixed: false, + }); + } + + // Build graph for cycle detection + if (sourceId) { + if (!graph.has(sourceId)) graph.set(sourceId, []); + graph.get(sourceId)!.push(targetId); + } + } + } + + // Detect cycles via DFS + const visited = new Set(); + const inStack = new Set(); + + function dfs(node: string, path: string[]): string[] | null { + if (inStack.has(node)) { + const cycleStart = path.indexOf(node); + return path.slice(cycleStart).concat(node); + } + if (visited.has(node)) return null; + + visited.add(node); + inStack.add(node); + path.push(node); + + const neighbors = graph.get(node) ?? []; + for (const neighbor of neighbors) { + const cycle = dfs(neighbor, path); + if (cycle) return cycle; + } + + path.pop(); + inStack.delete(node); + return null; + } + + const reportedCycles = new Set(); + for (const node of graph.keys()) { + if (visited.has(node)) continue; + const cycle = dfs(node, []); + if (cycle) { + const cycleKey = [...cycle].sort().join(','); + if (!reportedCycles.has(cycleKey)) { + reportedCycles.add(cycleKey); + context.issues.push({ + ruleId: 'flow-ordering', + severity: 'info', + message: `Circular reference detected: ${cycle.join(' → ')}`, + componentId: cycle[0], + blockIndex: context.idMap.get(cycle[0]) ?? 0, + fixed: false, + }); + } + } + } + }, +}; diff --git a/packages/validator/src/rules/index.ts b/packages/validator/src/rules/index.ts index f32a192..2439942 100644 --- a/packages/validator/src/rules/index.ts +++ b/packages/validator/src/rules/index.ts @@ -1,5 +1,6 @@ import type { ValidationRule, ValidationRuleId } from '../types.js'; import { yamlCorrectnessRule } from './yaml-correctness.js'; +import { fieldNameTyposRule } from './field-name-typos.js'; import { schemaConformanceRule } from './schema-conformance.js'; import { duplicateIdsRule } from './duplicate-ids.js'; import { idFormatRule } from './id-format.js'; @@ -9,17 +10,25 @@ import { actionReferencesRule } from './action-references.js'; import { sensitiveFlagsRule } from './sensitive-flags.js'; import { requiredMarkersRule } from './required-markers.js'; import { thinkingBlockRule } from './thinking-block.js'; +import { tableDataKeysRule } from './table-data-keys.js'; +import { selectOptionsRule } from './select-options.js'; +import { chartValidationRule } from './chart-validation.js'; +import { placeholderContentRule } from './placeholder-content.js'; +import { unreferencedComponentsRule } from './unreferenced-components.js'; +import { flowOrderingRule } from './flow-ordering.js'; /** * Ordered list of all validation rules. * * Order matters: * 1. yaml-correctness runs first (blocks with bad YAML are excluded from later rules) - * 2. schema-conformance runs second - * 3. All other rules run after + * 2. field-name-typos runs before schema-conformance (detects typos before schema fix normalizes them) + * 3. schema-conformance runs early + * 4. All other rules run after */ export const ALL_RULES: readonly ValidationRule[] = [ yamlCorrectnessRule, + fieldNameTyposRule, schemaConformanceRule, duplicateIdsRule, idFormatRule, @@ -29,6 +38,12 @@ export const ALL_RULES: readonly ValidationRule[] = [ sensitiveFlagsRule, requiredMarkersRule, thinkingBlockRule, + tableDataKeysRule, + selectOptionsRule, + chartValidationRule, + placeholderContentRule, + unreferencedComponentsRule, + flowOrderingRule, ]; export function getRulesExcluding( diff --git a/packages/validator/src/rules/placeholder-content.ts b/packages/validator/src/rules/placeholder-content.ts new file mode 100644 index 0000000..1f86dbf --- /dev/null +++ b/packages/validator/src/rules/placeholder-content.ts @@ -0,0 +1,84 @@ +import type { ValidationRule } from '../types.js'; + +const PLACEHOLDER_PATTERNS: RegExp[] = [ + /^\.{3,}$/, + /^(TODO|TBD|FIXME|placeholder|lorem\s*ipsum|example|sample)\b/i, +]; + +const TOP_LEVEL_FIELDS = ['content', 'title', 'label', 'text', 'description']; + +function isPlaceholder(value: string): boolean { + const trimmed = value.trim(); + return PLACEHOLDER_PATTERNS.some((p) => p.test(trimmed)); +} + +export const placeholderContentRule: ValidationRule = { + id: 'placeholder-content', + name: 'Placeholder Content', + description: + 'Warns when fields contain placeholder or stub content (TODO, TBD, "...", etc.)', + defaultSeverity: 'info', + + validate(context) { + for (const block of context.blocks) { + if (block.data === null) continue; + + const id = + typeof block.data.id === 'string' ? block.data.id : null; + + // Check top-level fields + for (const field of TOP_LEVEL_FIELDS) { + const value = block.data[field]; + if (typeof value === 'string' && isPlaceholder(value)) { + context.issues.push({ + ruleId: 'placeholder-content', + severity: 'info', + message: `Field "${field}" contains placeholder content: "${value}"`, + componentId: id, + field, + blockIndex: block.index, + fixed: false, + }); + } + } + + // Check form fields[].label + if (block.data.type === 'form' && Array.isArray(block.data.fields)) { + for (let i = 0; i < block.data.fields.length; i++) { + const f = block.data.fields[i]; + if (typeof f !== 'object' || f === null) continue; + if (typeof f.label === 'string' && isPlaceholder(f.label)) { + context.issues.push({ + ruleId: 'placeholder-content', + severity: 'info', + message: `Form field label contains placeholder content: "${f.label}"`, + componentId: id, + field: `fields[${i}].label`, + blockIndex: block.index, + fixed: false, + }); + } + } + } + + // Check table columns[].header + if (block.data.type === 'table' && Array.isArray(block.data.columns)) { + for (let i = 0; i < block.data.columns.length; i++) { + const col = block.data.columns[i]; + if (typeof col !== 'object' || col === null) continue; + if (typeof col.header === 'string' && isPlaceholder(col.header)) { + context.issues.push({ + ruleId: 'placeholder-content', + severity: 'info', + message: `Table column header contains placeholder content: "${col.header}"`, + componentId: id, + field: `columns[${i}].header`, + blockIndex: block.index, + fixed: false, + }); + } + } + } + } + }, +}; diff --git a/packages/validator/src/rules/schema-conformance.ts b/packages/validator/src/rules/schema-conformance.ts index 6ca2037..b05b9d2 100644 --- a/packages/validator/src/rules/schema-conformance.ts +++ b/packages/validator/src/rules/schema-conformance.ts @@ -27,7 +27,13 @@ export const schemaConformanceRule: ValidationRule = { continue; } - const schema = componentSchemaRegistry.get(type); + let schema = componentSchemaRegistry.get(type); + + // Fall back to custom schemas for types not in the built-in registry + if (!schema && context.options.customSchemas?.[type]) { + schema = context.options.customSchemas[type] as import('zod').ZodType; + } + if (!schema) { context.issues.push({ ruleId: 'schema-conformance', diff --git a/packages/validator/src/rules/select-options.ts b/packages/validator/src/rules/select-options.ts new file mode 100644 index 0000000..c42d371 --- /dev/null +++ b/packages/validator/src/rules/select-options.ts @@ -0,0 +1,64 @@ +import type { ValidationRule } from '../types.js'; + +export const selectOptionsRule: ValidationRule = { + id: 'select-options', + name: 'Select Options', + description: + 'Checks that form fields with type "select" have options defined', + defaultSeverity: 'warning', + + validate(context) { + for (const block of context.blocks) { + if (block.data === null) continue; + if (block.data.type !== 'form') continue; + + const fields = block.data.fields; + if (!Array.isArray(fields)) continue; + + const id = + typeof block.data.id === 'string' ? block.data.id : null; + + for (let i = 0; i < fields.length; i++) { + const field = fields[i]; + if (typeof field !== 'object' || field === null) continue; + if (field.type !== 'select') continue; + + const name = typeof field.name === 'string' ? field.name : `fields[${i}]`; + + if (field.options == null) { + context.issues.push({ + ruleId: 'select-options', + severity: 'warning', + message: `Select field "${name}" is missing options`, + componentId: id, + field: `fields[${i}]`, + blockIndex: block.index, + fixed: false, + }); + continue; + } + + // Skip binding expressions (strings are valid as binding refs) + if (typeof field.options === 'string') continue; + + if (Array.isArray(field.options)) { + for (let j = 0; j < field.options.length; j++) { + const opt = field.options[j]; + if (typeof opt !== 'object' || opt === null) continue; + if (typeof opt.label !== 'string' || typeof opt.value !== 'string') { + context.issues.push({ + ruleId: 'select-options', + severity: 'warning', + message: `Select field "${name}" has malformed option at index ${j} (expected {label, value})`, + componentId: id, + field: `fields[${i}].options[${j}]`, + blockIndex: block.index, + fixed: false, + }); + } + } + } + } + } + }, +}; diff --git a/packages/validator/src/rules/table-data-keys.ts b/packages/validator/src/rules/table-data-keys.ts new file mode 100644 index 0000000..ef3a403 --- /dev/null +++ b/packages/validator/src/rules/table-data-keys.ts @@ -0,0 +1,74 @@ +import type { ValidationRule } from '../types.js'; + +export const tableDataKeysRule: ValidationRule = { + id: 'table-data-keys', + name: 'Table Data Keys', + description: + 'Checks that table data row keys match defined column keys', + defaultSeverity: 'warning', + + validate(context) { + for (const block of context.blocks) { + if (block.data === null) continue; + if (block.data.type !== 'table') continue; + + const columns = block.data.columns; + const data = block.data.data; + + if (!Array.isArray(columns) || !Array.isArray(data)) continue; + + const id = + typeof block.data.id === 'string' ? block.data.id : null; + + const columnKeys = new Set(); + for (const col of columns) { + if (typeof col === 'object' && col !== null && typeof col.key === 'string') { + columnKeys.add(col.key); + } + } + + if (columnKeys.size === 0) continue; + + // Track which columns have at least one matching data key + const matchedColumns = new Set(); + const reportedExtraKeys = new Set(); + + for (let rowIdx = 0; rowIdx < data.length; rowIdx++) { + const row = data[rowIdx]; + if (typeof row !== 'object' || row === null) continue; + + for (const key of Object.keys(row)) { + if (columnKeys.has(key)) { + matchedColumns.add(key); + } else if (!reportedExtraKeys.has(key)) { + reportedExtraKeys.add(key); + context.issues.push({ + ruleId: 'table-data-keys', + severity: 'warning', + message: `Data key "${key}" does not match any column (defined columns: ${[...columnKeys].join(', ')})`, + componentId: id, + field: `data[${rowIdx}].${key}`, + blockIndex: block.index, + fixed: false, + }); + } + } + } + + // Warn about columns with no matching data + for (const colKey of columnKeys) { + if (!matchedColumns.has(colKey) && data.length > 0) { + context.issues.push({ + ruleId: 'table-data-keys', + severity: 'warning', + message: `Column "${colKey}" has no matching keys in any data row`, + componentId: id, + field: `columns`, + blockIndex: block.index, + fixed: false, + }); + } + } + } + }, +}; diff --git a/packages/validator/src/rules/unreferenced-components.ts b/packages/validator/src/rules/unreferenced-components.ts new file mode 100644 index 0000000..da6c05a --- /dev/null +++ b/packages/validator/src/rules/unreferenced-components.ts @@ -0,0 +1,75 @@ +import { extractBindings } from '@mobile-reality/mdma-parser'; +import type { ValidationRule } from '../types.js'; +import { ACTION_REFERENCE_FIELDS } from '../constants.js'; + +export const unreferencedComponentsRule: ValidationRule = { + id: 'unreferenced-components', + name: 'Unreferenced Components', + description: + 'Flags components whose ID is never referenced by bindings or action fields in other components', + defaultSeverity: 'info', + + validate(context) { + const allIds = new Map(); // id → blockIndex + for (const block of context.blocks) { + if (block.data === null) continue; + if (typeof block.data.id === 'string') { + allIds.set(block.data.id, block.index); + } + } + + if (allIds.size <= 1) return; // Nothing to check with 0–1 components + + const referencedIds = new Set(); + + for (const block of context.blocks) { + if (block.data === null) continue; + + const blockId = typeof block.data.id === 'string' ? block.data.id : ''; + + // Collect binding root segments + const bindings = extractBindings(blockId, block.data); + for (const binding of bindings) { + const rootSegment = binding.path.split('.')[0]; + referencedIds.add(rootSegment); + } + + // Collect action reference field values + const type = block.data.type; + if (typeof type === 'string') { + const fields = ACTION_REFERENCE_FIELDS[type]; + if (fields) { + for (const field of fields) { + const value = block.data[field]; + if (typeof value === 'string') { + referencedIds.add(value); + } + } + } + } + } + + // Find unreferenced components + for (const [id, blockIndex] of allIds) { + if (referencedIds.has(id)) continue; + + const block = context.blocks[blockIndex]; + if (!block?.data) continue; + + // Skip thinking blocks (standalone by nature) + if (block.data.type === 'thinking') continue; + + // Skip the first component (often standalone) + if (blockIndex === 0) continue; + + context.issues.push({ + ruleId: 'unreferenced-components', + severity: 'info', + message: `Component "${id}" is not referenced by any binding or action in the document`, + componentId: id, + blockIndex, + fixed: false, + }); + } + }, +}; diff --git a/packages/validator/src/types.ts b/packages/validator/src/types.ts index 2aa3be4..cc5f1c6 100644 --- a/packages/validator/src/types.ts +++ b/packages/validator/src/types.ts @@ -10,7 +10,14 @@ export type ValidationRuleId = | 'required-markers' | 'id-format' | 'thinking-block' - | 'yaml-correctness'; + | 'yaml-correctness' + | 'table-data-keys' + | 'select-options' + | 'chart-validation' + | 'placeholder-content' + | 'unreferenced-components' + | 'flow-ordering' + | 'field-name-typos'; export interface ValidationIssue { /** Which rule flagged this */ @@ -96,6 +103,8 @@ export interface ValidatorOptions { autoFix?: boolean; /** Custom PII field name patterns to check (in addition to defaults). */ customPiiPatterns?: RegExp[]; + /** Custom component Zod schemas for types not in the built-in registry. */ + customSchemas?: Record; } export interface ValidationResult { diff --git a/packages/validator/tests/rules/action-references.test.ts b/packages/validator/tests/rules/action-references.test.ts index 2ef427a..62a7e6d 100644 --- a/packages/validator/tests/rules/action-references.test.ts +++ b/packages/validator/tests/rules/action-references.test.ts @@ -34,7 +34,7 @@ describe('action-references rule', () => { type: 'button', id: 'submit-btn', text: 'Submit', - onAction: 'submit-action', + onAction: 'submit-btn', }), createBlock(1, { type: 'webhook', @@ -69,7 +69,7 @@ describe('action-references rule', () => { type: 'button', id: 'submit-btn', text: 'Go', - onAction: 'do-it', + onAction: 'submit-btn', }), createBlock(1, { type: 'webhook', @@ -84,25 +84,83 @@ describe('action-references rule', () => { expect(ctx.issues[0].message).toContain('submit-btn'); }); - it('does not flag non-webhook components', () => { + it('flags form onSubmit referencing non-existent component', () => { const ctx = createContext([ createBlock(0, { - type: 'button', - id: 'btn', - text: 'Submit', - onAction: 'some-action-that-doesnt-exist-as-id', + type: 'form', + id: 'f', + fields: [], + onSubmit: 'nonexistent-action', }), - createBlock(1, { + ]); + actionReferencesRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].message).toContain('nonexistent-action'); + }); + + it('passes when form onSubmit references valid component', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'f', fields: [], - onSubmit: 'another-nonexistent-action', + onSubmit: 'wh', + }), + createBlock(1, { + type: 'webhook', + id: 'wh', + url: 'https://api.example.com', + trigger: 'f', }), ]); actionReferencesRule.validate(ctx); expect(ctx.issues).toHaveLength(0); }); + it('flags button onAction referencing non-existent component', () => { + const ctx = createContext([ + createBlock(0, { + type: 'button', + id: 'btn', + text: 'Submit', + onAction: 'does-not-exist', + }), + ]); + actionReferencesRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].message).toContain('does-not-exist'); + }); + + it('flags tasklist onComplete referencing non-existent component', () => { + const ctx = createContext([ + createBlock(0, { + type: 'tasklist', + id: 'tl', + items: [], + onComplete: 'missing-target', + }), + ]); + actionReferencesRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].message).toContain('missing-target'); + }); + + it('flags approval-gate onApprove and onDeny referencing non-existent components', () => { + const ctx = createContext([ + createBlock(0, { + type: 'approval-gate', + id: 'ag', + title: 'Approve', + onApprove: 'missing-approve', + onDeny: 'missing-deny', + }), + ]); + actionReferencesRule.validate(ctx); + expect(ctx.issues).toHaveLength(2); + expect(ctx.issues[0].message).toContain('missing-approve'); + expect(ctx.issues[1].message).toContain('missing-deny'); + }); + it('skips blocks with null data', () => { const blocks: ParsedBlock[] = [ { index: 0, rawYaml: '', data: null, startOffset: 0, endOffset: 0, yamlStartOffset: 0, yamlEndOffset: 0 }, diff --git a/packages/validator/tests/rules/binding-resolution.test.ts b/packages/validator/tests/rules/binding-resolution.test.ts index 3237adf..79886b3 100644 --- a/packages/validator/tests/rules/binding-resolution.test.ts +++ b/packages/validator/tests/rules/binding-resolution.test.ts @@ -84,6 +84,108 @@ describe('binding-resolution rule', () => { expect(ctx.issues[0].message).toContain('contact-form'); }); + it('passes deep binding when form has matching field', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', + id: 'my-form', + fields: [ + { name: 'email', type: 'email', label: 'Email' }, + { name: 'name', type: 'text', label: 'Name' }, + ], + }), + createBlock(1, { + type: 'callout', + id: 'info', + content: 'hi', + visible: '{{my-form.email}}', + }), + ]); + bindingResolutionRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('flags deep binding referencing non-existent form field at info level', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', + id: 'myform', + fields: [ + { name: 'email', type: 'email', label: 'Email' }, + ], + }), + createBlock(1, { + type: 'callout', + id: 'info', + content: 'hi', + visible: '{{myform.nonexistent}}', + }), + ]); + bindingResolutionRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].severity).toBe('info'); + expect(ctx.issues[0].message).toContain('nonexistent'); + expect(ctx.issues[0].message).toContain('email'); + }); + + it('passes deep binding when table has matching column', () => { + const ctx = createContext([ + createBlock(0, { + type: 'table', + id: 'mytable', + columns: [{ key: 'name', header: 'Name' }], + data: [], + }), + createBlock(1, { + type: 'callout', + id: 'info', + content: 'hi', + visible: '{{mytable.name}}', + }), + ]); + bindingResolutionRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('flags deep binding referencing non-existent table column', () => { + const ctx = createContext([ + createBlock(0, { + type: 'table', + id: 'mytable', + columns: [{ key: 'name', header: 'Name' }], + data: [], + }), + createBlock(1, { + type: 'callout', + id: 'info', + content: 'hi', + visible: '{{mytable.missing}}', + }), + ]); + bindingResolutionRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].severity).toBe('info'); + expect(ctx.issues[0].message).toContain('missing'); + }); + + it('skips deep validation on unknown component types', () => { + const ctx = createContext([ + createBlock(0, { + type: 'callout', + id: 'mycallout', + content: 'hi', + }), + createBlock(1, { + type: 'callout', + id: 'info', + content: 'hi', + visible: '{{mycallout.something}}', + }), + ]); + bindingResolutionRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + it('skips blocks with null data', () => { const blocks: ParsedBlock[] = [ { index: 0, rawYaml: '', data: null, startOffset: 0, endOffset: 0, yamlStartOffset: 0, yamlEndOffset: 0 }, diff --git a/packages/validator/tests/rules/chart-validation.test.ts b/packages/validator/tests/rules/chart-validation.test.ts new file mode 100644 index 0000000..ffb12b7 --- /dev/null +++ b/packages/validator/tests/rules/chart-validation.test.ts @@ -0,0 +1,90 @@ +import { describe, it, expect } from 'vitest'; +import { chartValidationRule } from '../../src/rules/chart-validation.js'; +import type { ValidationRuleContext, ParsedBlock } from '../../src/types.js'; + +function createBlock(index: number, data: Record): ParsedBlock { + return { index, rawYaml: '', data, startOffset: 0, endOffset: 0, yamlStartOffset: 0, yamlEndOffset: 0 }; +} + +function createContext(blocks: ParsedBlock[]): ValidationRuleContext { + const idMap = new Map(); + for (const block of blocks) { + if (block.data && typeof block.data.id === 'string') idMap.set(block.data.id, block.index); + } + return { blocks, idMap, issues: [], options: {} }; +} + +describe('chart-validation rule', () => { + it('passes for valid CSV with matching axes', () => { + const ctx = createContext([ + createBlock(0, { + type: 'chart', id: 'ch', + data: 'Month,Revenue,Costs\nJan,100,80\nFeb,120,90', + xAxis: 'Month', + yAxis: ['Revenue', 'Costs'], + }), + ]); + chartValidationRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('warns when xAxis not in CSV headers', () => { + const ctx = createContext([ + createBlock(0, { + type: 'chart', id: 'ch', + data: 'Month,Revenue\nJan,100', + xAxis: 'Date', + }), + ]); + chartValidationRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].message).toContain('xAxis'); + expect(ctx.issues[0].message).toContain('Date'); + }); + + it('warns when yAxis item not in CSV headers', () => { + const ctx = createContext([ + createBlock(0, { + type: 'chart', id: 'ch', + data: 'Month,Revenue\nJan,100', + yAxis: 'Profit', + }), + ]); + chartValidationRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].message).toContain('yAxis'); + expect(ctx.issues[0].message).toContain('Profit'); + }); + + it('skips binding expression data', () => { + const ctx = createContext([ + createBlock(0, { + type: 'chart', id: 'ch', + data: '{{sales-data.csv}}', + xAxis: 'anything', + }), + ]); + chartValidationRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('warns on CSV with only header row (no data)', () => { + const ctx = createContext([ + createBlock(0, { + type: 'chart', id: 'ch', + data: 'Month,Revenue', + }), + ]); + chartValidationRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].message).toContain('valid CSV'); + }); + + it('skips non-chart components', () => { + const ctx = createContext([ + createBlock(0, { type: 'callout', id: 'c', content: 'hi' }), + ]); + chartValidationRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); +}); diff --git a/packages/validator/tests/rules/field-name-typos.test.ts b/packages/validator/tests/rules/field-name-typos.test.ts new file mode 100644 index 0000000..e7bd646 --- /dev/null +++ b/packages/validator/tests/rules/field-name-typos.test.ts @@ -0,0 +1,72 @@ +import { describe, it, expect } from 'vitest'; +import { fieldNameTyposRule } from '../../src/rules/field-name-typos.js'; +import type { ValidationRuleContext, ParsedBlock } from '../../src/types.js'; + +function createBlock(index: number, data: Record): ParsedBlock { + return { index, rawYaml: '', data, startOffset: 0, endOffset: 0, yamlStartOffset: 0, yamlEndOffset: 0 }; +} + +function createContext(blocks: ParsedBlock[]): ValidationRuleContext { + const idMap = new Map(); + for (const block of blocks) { + if (block.data && typeof block.data.id === 'string') idMap.set(block.data.id, block.index); + } + return { blocks, idMap, issues: [], options: {} }; +} + +describe('field-name-typos rule', () => { + it('flags approval-gate with "roles" instead of "allowedRoles"', () => { + const ctx = createContext([ + createBlock(0, { + type: 'approval-gate', id: 'ag', title: 'Approve', + roles: ['admin', 'manager'], + }), + ]); + fieldNameTyposRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].ruleId).toBe('field-name-typos'); + expect(ctx.issues[0].message).toContain('allowedRoles'); + }); + + it('passes for correct field names', () => { + const ctx = createContext([ + createBlock(0, { + type: 'approval-gate', id: 'ag', title: 'Approve', + allowedRoles: ['admin'], + }), + ]); + fieldNameTyposRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('flags button with "onClick" instead of "onAction"', () => { + const ctx = createContext([ + createBlock(0, { + type: 'button', id: 'btn', text: 'Go', + onClick: 'do-something', + }), + ]); + fieldNameTyposRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].message).toContain('onAction'); + }); + + it('skips component types without typo map', () => { + const ctx = createContext([ + createBlock(0, { type: 'callout', id: 'c', content: 'hi' }), + ]); + fieldNameTyposRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('skips blocks with null data', () => { + const ctx: ValidationRuleContext = { + blocks: [{ index: 0, rawYaml: '', data: null, startOffset: 0, endOffset: 0, yamlStartOffset: 0, yamlEndOffset: 0 }], + idMap: new Map(), + issues: [], + options: {}, + }; + fieldNameTyposRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); +}); diff --git a/packages/validator/tests/rules/flow-ordering.test.ts b/packages/validator/tests/rules/flow-ordering.test.ts new file mode 100644 index 0000000..fbcebb4 --- /dev/null +++ b/packages/validator/tests/rules/flow-ordering.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from 'vitest'; +import { flowOrderingRule } from '../../src/rules/flow-ordering.js'; +import type { ValidationRuleContext, ParsedBlock } from '../../src/types.js'; + +function createBlock(index: number, data: Record): ParsedBlock { + return { index, rawYaml: '', data, startOffset: 0, endOffset: 0, yamlStartOffset: 0, yamlEndOffset: 0 }; +} + +function createContext(blocks: ParsedBlock[]): ValidationRuleContext { + const idMap = new Map(); + for (const block of blocks) { + if (block.data && typeof block.data.id === 'string') idMap.set(block.data.id, block.index); + } + return { blocks, idMap, issues: [], options: {} }; +} + +describe('flow-ordering rule', () => { + it('passes for forward-only references', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'f', fields: [], onSubmit: 'btn' }), + createBlock(1, { type: 'button', id: 'btn', text: 'Go' }), + ]); + flowOrderingRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('flags backward references', () => { + const ctx = createContext([ + createBlock(0, { type: 'webhook', id: 'wh', url: 'https://api.example.com', trigger: 'btn' }), + createBlock(1, { type: 'button', id: 'btn', text: 'Go', onAction: 'wh' }), + ]); + flowOrderingRule.validate(ctx); + // wh.trigger → btn is a forward ref (btn at index 1 > wh at index 0): OK + // btn.onAction → wh is a backward ref (wh at index 0 < btn at index 1): flagged + const backwardIssues = ctx.issues.filter((i) => i.message.includes('backward')); + expect(backwardIssues.length).toBeGreaterThan(0); + }); + + it('detects circular references', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'a', fields: [], onSubmit: 'b' }), + createBlock(1, { type: 'form', id: 'b', fields: [], onSubmit: 'a' }), + ]); + flowOrderingRule.validate(ctx); + const cycleIssues = ctx.issues.filter((i) => i.message.includes('Circular')); + expect(cycleIssues).toHaveLength(1); + }); + + it('ignores unknown target IDs', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'f', fields: [], onSubmit: 'nonexistent' }), + ]); + flowOrderingRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('all issues have info severity', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'a', fields: [], onSubmit: 'b' }), + createBlock(1, { type: 'form', id: 'b', fields: [], onSubmit: 'a' }), + ]); + flowOrderingRule.validate(ctx); + expect(ctx.issues.every((i) => i.severity === 'info')).toBe(true); + }); +}); diff --git a/packages/validator/tests/rules/placeholder-content.test.ts b/packages/validator/tests/rules/placeholder-content.test.ts new file mode 100644 index 0000000..cb5abb5 --- /dev/null +++ b/packages/validator/tests/rules/placeholder-content.test.ts @@ -0,0 +1,88 @@ +import { describe, it, expect } from 'vitest'; +import { placeholderContentRule } from '../../src/rules/placeholder-content.js'; +import type { ValidationRuleContext, ParsedBlock } from '../../src/types.js'; + +function createBlock(index: number, data: Record): ParsedBlock { + return { index, rawYaml: '', data, startOffset: 0, endOffset: 0, yamlStartOffset: 0, yamlEndOffset: 0 }; +} + +function createContext(blocks: ParsedBlock[]): ValidationRuleContext { + const idMap = new Map(); + for (const block of blocks) { + if (block.data && typeof block.data.id === 'string') idMap.set(block.data.id, block.index); + } + return { blocks, idMap, issues: [], options: {} }; +} + +describe('placeholder-content rule', () => { + it('flags "TODO" in title', () => { + const ctx = createContext([ + createBlock(0, { type: 'callout', id: 'c', content: 'real content', title: 'TODO fix this' }), + ]); + placeholderContentRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].severity).toBe('info'); + expect(ctx.issues[0].field).toBe('title'); + }); + + it('flags "..." in content', () => { + const ctx = createContext([ + createBlock(0, { type: 'callout', id: 'c', content: '...' }), + ]); + placeholderContentRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].field).toBe('content'); + }); + + it('flags "lorem ipsum"', () => { + const ctx = createContext([ + createBlock(0, { type: 'callout', id: 'c', content: 'Lorem ipsum dolor sit amet' }), + ]); + placeholderContentRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + }); + + it('passes for normal content', () => { + const ctx = createContext([ + createBlock(0, { type: 'callout', id: 'c', content: 'Please fill in the form below' }), + ]); + placeholderContentRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('detects placeholder in form field labels', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', id: 'f', + fields: [{ name: 'email', type: 'email', label: 'TBD' }], + }), + ]); + placeholderContentRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].field).toBe('fields[0].label'); + }); + + it('detects placeholder in table column headers', () => { + const ctx = createContext([ + createBlock(0, { + type: 'table', id: 't', + columns: [{ key: 'name', header: 'FIXME' }], + data: [], + }), + ]); + placeholderContentRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].field).toBe('columns[0].header'); + }); + + it('skips blocks with null data', () => { + const ctx: ValidationRuleContext = { + blocks: [{ index: 0, rawYaml: '', data: null, startOffset: 0, endOffset: 0, yamlStartOffset: 0, yamlEndOffset: 0 }], + idMap: new Map(), + issues: [], + options: {}, + }; + placeholderContentRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); +}); diff --git a/packages/validator/tests/rules/schema-conformance.test.ts b/packages/validator/tests/rules/schema-conformance.test.ts index bbaa4c0..d7f2b4e 100644 --- a/packages/validator/tests/rules/schema-conformance.test.ts +++ b/packages/validator/tests/rules/schema-conformance.test.ts @@ -82,6 +82,55 @@ describe('schema-conformance rule', () => { expect(ctx.issues).toHaveLength(0); }); + it('validates custom component type via customSchemas', () => { + const { z } = require('zod'); + const customSchema = z.object({ + type: z.literal('progress'), + id: z.string().min(1), + value: z.number().min(0).max(100), + label: z.string().optional(), + }); + const ctx: ValidationRuleContext = { + blocks: [createBlock(0, { type: 'progress', id: 'p', value: 50 })], + idMap: new Map([['p', 0]]), + issues: [], + options: { customSchemas: { progress: customSchema } }, + }; + schemaConformanceRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('flags invalid custom component', () => { + const { z } = require('zod'); + const customSchema = z.object({ + type: z.literal('progress'), + id: z.string().min(1), + value: z.number().min(0).max(100), + }); + const ctx: ValidationRuleContext = { + blocks: [createBlock(0, { type: 'progress', id: 'p', value: 200 })], + idMap: new Map([['p', 0]]), + issues: [], + options: { customSchemas: { progress: customSchema } }, + }; + schemaConformanceRule.validate(ctx); + expect(ctx.issues.length).toBeGreaterThan(0); + }); + + it('does not use custom schema for built-in types', () => { + const { z } = require('zod'); + const fakeSchema = z.object({ type: z.literal('callout') }); + const ctx: ValidationRuleContext = { + blocks: [createBlock(0, { type: 'callout', id: 'c', content: 'Hello' })], + idMap: new Map([['c', 0]]), + issues: [], + options: { customSchemas: { callout: fakeSchema } }, + }; + schemaConformanceRule.validate(ctx); + // Should use built-in schema (which passes), not the fake one + expect(ctx.issues).toHaveLength(0); + }); + it('passes for a valid callout', () => { const ctx = createContext([ createBlock(0, { diff --git a/packages/validator/tests/rules/select-options.test.ts b/packages/validator/tests/rules/select-options.test.ts new file mode 100644 index 0000000..3a616d9 --- /dev/null +++ b/packages/validator/tests/rules/select-options.test.ts @@ -0,0 +1,83 @@ +import { describe, it, expect } from 'vitest'; +import { selectOptionsRule } from '../../src/rules/select-options.js'; +import type { ValidationRuleContext, ParsedBlock } from '../../src/types.js'; + +function createBlock(index: number, data: Record): ParsedBlock { + return { index, rawYaml: '', data, startOffset: 0, endOffset: 0, yamlStartOffset: 0, yamlEndOffset: 0 }; +} + +function createContext(blocks: ParsedBlock[]): ValidationRuleContext { + const idMap = new Map(); + for (const block of blocks) { + if (block.data && typeof block.data.id === 'string') idMap.set(block.data.id, block.index); + } + return { blocks, idMap, issues: [], options: {} }; +} + +describe('select-options rule', () => { + it('passes for select with valid options array', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', id: 'f', + fields: [{ name: 'color', type: 'select', label: 'Color', options: [{ label: 'Red', value: 'red' }] }], + }), + ]); + selectOptionsRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('passes for select with binding string options', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', id: 'f', + fields: [{ name: 'country', type: 'select', label: 'Country', options: '{{countries}}' }], + }), + ]); + selectOptionsRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('warns for select with missing options', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', id: 'f', + fields: [{ name: 'color', type: 'select', label: 'Color' }], + }), + ]); + selectOptionsRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].ruleId).toBe('select-options'); + expect(ctx.issues[0].message).toContain('missing options'); + }); + + it('warns for malformed option objects', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', id: 'f', + fields: [{ name: 'color', type: 'select', label: 'Color', options: [{ label: 'Red' }] }], + }), + ]); + selectOptionsRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].message).toContain('malformed option'); + }); + + it('skips non-select fields', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', id: 'f', + fields: [{ name: 'email', type: 'email', label: 'Email' }], + }), + ]); + selectOptionsRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('skips non-form components', () => { + const ctx = createContext([ + createBlock(0, { type: 'callout', id: 'c', content: 'hi' }), + ]); + selectOptionsRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); +}); diff --git a/packages/validator/tests/rules/table-data-keys.test.ts b/packages/validator/tests/rules/table-data-keys.test.ts new file mode 100644 index 0000000..08f9f9e --- /dev/null +++ b/packages/validator/tests/rules/table-data-keys.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect } from 'vitest'; +import { tableDataKeysRule } from '../../src/rules/table-data-keys.js'; +import type { ValidationRuleContext, ParsedBlock } from '../../src/types.js'; + +function createBlock(index: number, data: Record): ParsedBlock { + return { index, rawYaml: '', data, startOffset: 0, endOffset: 0, yamlStartOffset: 0, yamlEndOffset: 0 }; +} + +function createContext(blocks: ParsedBlock[]): ValidationRuleContext { + const idMap = new Map(); + for (const block of blocks) { + if (block.data && typeof block.data.id === 'string') idMap.set(block.data.id, block.index); + } + return { blocks, idMap, issues: [], options: {} }; +} + +describe('table-data-keys rule', () => { + it('passes when data keys match columns', () => { + const ctx = createContext([ + createBlock(0, { + type: 'table', id: 't', + columns: [{ key: 'name', header: 'Name' }, { key: 'email', header: 'Email' }], + data: [{ name: 'Alice', email: 'alice@example.com' }], + }), + ]); + tableDataKeysRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('flags extra keys in data rows', () => { + const ctx = createContext([ + createBlock(0, { + type: 'table', id: 't', + columns: [{ key: 'name', header: 'Name' }], + data: [{ name: 'Alice', phone: '555-1234' }], + }), + ]); + tableDataKeysRule.validate(ctx); + expect(ctx.issues.some((i) => i.message.includes('phone'))).toBe(true); + }); + + it('flags columns with no matching data keys', () => { + const ctx = createContext([ + createBlock(0, { + type: 'table', id: 't', + columns: [{ key: 'name', header: 'Name' }, { key: 'email', header: 'Email' }], + data: [{ name: 'Alice' }], + }), + ]); + tableDataKeysRule.validate(ctx); + expect(ctx.issues.some((i) => i.message.includes('email'))).toBe(true); + }); + + it('skips binding expression data', () => { + const ctx = createContext([ + createBlock(0, { + type: 'table', id: 't', + columns: [{ key: 'name', header: 'Name' }], + data: '{{some-component.rows}}', + }), + ]); + tableDataKeysRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('skips non-table components', () => { + const ctx = createContext([ + createBlock(0, { type: 'callout', id: 'c', content: 'hi' }), + ]); + tableDataKeysRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('reports each extra key only once across rows', () => { + const ctx = createContext([ + createBlock(0, { + type: 'table', id: 't', + columns: [{ key: 'name', header: 'Name' }], + data: [ + { name: 'Alice', phone: '555' }, + { name: 'Bob', phone: '666' }, + ], + }), + ]); + tableDataKeysRule.validate(ctx); + const phoneIssues = ctx.issues.filter((i) => i.message.includes('phone')); + expect(phoneIssues).toHaveLength(1); + }); +}); diff --git a/packages/validator/tests/rules/unreferenced-components.test.ts b/packages/validator/tests/rules/unreferenced-components.test.ts new file mode 100644 index 0000000..4dbd26a --- /dev/null +++ b/packages/validator/tests/rules/unreferenced-components.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect } from 'vitest'; +import { unreferencedComponentsRule } from '../../src/rules/unreferenced-components.js'; +import type { ValidationRuleContext, ParsedBlock } from '../../src/types.js'; + +function createBlock(index: number, data: Record): ParsedBlock { + return { index, rawYaml: '', data, startOffset: 0, endOffset: 0, yamlStartOffset: 0, yamlEndOffset: 0 }; +} + +function createContext(blocks: ParsedBlock[]): ValidationRuleContext { + const idMap = new Map(); + for (const block of blocks) { + if (block.data && typeof block.data.id === 'string') idMap.set(block.data.id, block.index); + } + return { blocks, idMap, issues: [], options: {} }; +} + +describe('unreferenced-components rule', () => { + it('does not flag component referenced via binding', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'my-form', fields: [{ name: 'email', type: 'email', label: 'Email' }] }), + createBlock(1, { type: 'callout', id: 'info', content: 'hi', visible: '{{my-form.email}}' }), + ]); + unreferencedComponentsRule.validate(ctx); + // my-form is referenced, info is at index 1 and referenced by nothing BUT my-form is referenced + const issues = ctx.issues.filter((i) => i.componentId === 'my-form'); + expect(issues).toHaveLength(0); + }); + + it('does not flag component referenced via onAction', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'my-form', fields: [] }), + createBlock(1, { type: 'button', id: 'btn', text: 'Go', onAction: 'my-form' }), + ]); + unreferencedComponentsRule.validate(ctx); + const issues = ctx.issues.filter((i) => i.componentId === 'my-form'); + expect(issues).toHaveLength(0); + }); + + it('flags unreferenced non-first, non-thinking component', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'my-form', fields: [] }), + createBlock(1, { type: 'callout', id: 'orphan', content: 'I am alone' }), + ]); + unreferencedComponentsRule.validate(ctx); + const issues = ctx.issues.filter((i) => i.componentId === 'orphan'); + expect(issues).toHaveLength(1); + expect(issues[0].severity).toBe('info'); + }); + + it('does not flag the first component', () => { + const ctx = createContext([ + createBlock(0, { type: 'callout', id: 'first', content: 'I am first' }), + createBlock(1, { type: 'callout', id: 'second', content: 'hi', visible: '{{first.something}}' }), + ]); + unreferencedComponentsRule.validate(ctx); + const issues = ctx.issues.filter((i) => i.componentId === 'first'); + expect(issues).toHaveLength(0); + }); + + it('does not flag thinking blocks', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'f', fields: [] }), + createBlock(1, { type: 'thinking', id: 'think', content: 'Reasoning...', status: 'done', collapsed: true }), + ]); + unreferencedComponentsRule.validate(ctx); + const issues = ctx.issues.filter((i) => i.componentId === 'think'); + expect(issues).toHaveLength(0); + }); + + it('skips when only one component exists', () => { + const ctx = createContext([ + createBlock(0, { type: 'callout', id: 'only', content: 'Solo' }), + ]); + unreferencedComponentsRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); +}); diff --git a/packages/validator/tests/validate.test.ts b/packages/validator/tests/validate.test.ts index 287203b..154fec9 100644 --- a/packages/validator/tests/validate.test.ts +++ b/packages/validator/tests/validate.test.ts @@ -282,6 +282,26 @@ data: personal-info.rows expect(result.output).toContain('{{personal-info.rows}}'); }); + it('auto-fixes button with missing text by deriving from id', () => { + const md = `\`\`\`mdma +type: button +id: submit-order +variant: primary +onAction: some-form +\`\`\` +`; + const result = validate(md); + + const schemaIssues = result.issues.filter( + (i) => i.ruleId === 'schema-conformance' && i.componentId === 'submit-order', + ); + expect(schemaIssues.length).toBeGreaterThan(0); + expect(schemaIssues.every((i) => i.fixed)).toBe(true); + + // text should be derived from id + expect(result.output).toContain('Submit Order'); + }); + it('auto-splits multi-component mdma blocks into separate blocks', () => { const md = `# Test From d864d6397db7d0903cab94c334a1e1699232e629 Mon Sep 17 00:00:00 2001 From: gitsad Date: Wed, 25 Mar 2026 14:35:52 +0100 Subject: [PATCH 2/9] fix: improved bindings in validator --- .../validator/src/fixes/binding-syntax.ts | 3 + .../validator/src/fixes/schema-defaults.ts | 56 ++++++++++++++----- packages/validator/tests/validate.test.ts | 22 ++++++++ 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/packages/validator/src/fixes/binding-syntax.ts b/packages/validator/src/fixes/binding-syntax.ts index ee2c631..c1d8809 100644 --- a/packages/validator/src/fixes/binding-syntax.ts +++ b/packages/validator/src/fixes/binding-syntax.ts @@ -52,6 +52,9 @@ function fixBindingsInObject(obj: Record): boolean { function fixBindingString(str: string): string { let result = str; + // Remove empty bindings: {{ }} or {{}} -> empty string + result = result.replace(/\{\{\s*\}\}/g, ''); + // Fix whitespace in bindings: {{ var.path }} -> {{var.path}} result = result.replace( /\{\{\s+([a-zA-Z_][a-zA-Z0-9_.]*)\s*\}\}/g, diff --git a/packages/validator/src/fixes/schema-defaults.ts b/packages/validator/src/fixes/schema-defaults.ts index fb5e996..c68c5da 100644 --- a/packages/validator/src/fixes/schema-defaults.ts +++ b/packages/validator/src/fixes/schema-defaults.ts @@ -111,25 +111,55 @@ function patchFormFields(data: Record): void { if ((!f.label || f.label === null) && typeof f.name === 'string') { f.label = keyToHeader(f.name); } + + // Wrap bare bind values in {{ }} + wrapBareBinding(f, 'bind'); } } +/** Binding path pattern: identifier with dots and hyphens (e.g. "form.email", "my-form.field") */ +const BARE_BINDING_PATH = /^[a-zA-Z_][\w.-]*$/; +const WRAPPED_BINDING = /^\{\{.+\}\}$/s; + /** - * Pre-fix table data: if `data` is a bare string that looks like a binding - * path but isn't wrapped in {{ }}, wrap it. LLMs often generate - * `data: personal-info.rows` instead of `data: "{{personal-info.rows}}"`. + * If `obj[field]` is a bare string that looks like a binding path but isn't + * wrapped in {{ }}, wrap it. LLMs often omit the double-brace wrapper. */ -function patchTableData(data: Record): void { - if (data.type !== 'table') return; - const d = data.data; - if (typeof d !== 'string') return; +function wrapBareBinding(obj: Record, field: string): void { + const val = obj[field]; + if (typeof val !== 'string') return; + if (WRAPPED_BINDING.test(val)) return; + if (BARE_BINDING_PATH.test(val)) { + obj[field] = `{{${val}}}`; + } +} + +/** + * Pre-fix any component: wrap bare binding paths for fields that accept + * binding expressions (`disabled`, `visible`, table `data`, form field `bind`). + */ +function patchBareBindings(data: Record): void { + // Component base: disabled, visible + wrapBareBinding(data, 'disabled'); + wrapBareBinding(data, 'visible'); + + // Table data + if (data.type === 'table') { + wrapBareBinding(data, 'data'); + } - // Already wrapped — nothing to do - if (/^\{\{.+\}\}$/s.test(d)) return; + // Tasklist item binds + if (data.type === 'tasklist' && Array.isArray(data.items)) { + for (const item of data.items) { + if (typeof item === 'object' && item !== null) { + wrapBareBinding(item as Record, 'bind'); + } + } + } - // Looks like a binding path (e.g. "component.field" or "component") - if (/^[a-zA-Z_][\w.-]*$/.test(d)) { - data.data = `{{${d}}}`; + // Webhook url and body bindings + if (data.type === 'webhook') { + wrapBareBinding(data, 'url'); } } @@ -253,7 +283,7 @@ export function fixSchemaDefaults(context: FixContext): void { // Patch known gaps before Zod re-parse patchTableColumns(block.data); - patchTableData(block.data); + patchBareBindings(block.data); patchFormFields(block.data); patchCalloutContent(block.data); patchButtonText(block.data); diff --git a/packages/validator/tests/validate.test.ts b/packages/validator/tests/validate.test.ts index 154fec9..ef12fef 100644 --- a/packages/validator/tests/validate.test.ts +++ b/packages/validator/tests/validate.test.ts @@ -282,6 +282,28 @@ data: personal-info.rows expect(result.output).toContain('{{personal-info.rows}}'); }); + it('auto-fixes form field bind as bare path by wrapping in {{ }}', () => { + const md = `\`\`\`mdma +type: form +id: my-form +fields: + - name: email + type: email + label: Email + bind: other-form.email +\`\`\` +`; + const result = validate(md); + + const schemaIssues = result.issues.filter( + (i) => i.ruleId === 'schema-conformance' && i.componentId === 'my-form', + ); + expect(schemaIssues.length).toBeGreaterThan(0); + expect(schemaIssues.every((i) => i.fixed)).toBe(true); + + expect(result.output).toContain('{{other-form.email}}'); + }); + it('auto-fixes button with missing text by deriving from id', () => { const md = `\`\`\`mdma type: button From a9007960c38c9ab68524241ffb68025c065b9edd Mon Sep 17 00:00:00 2001 From: gitsad Date: Wed, 1 Apr 2026 13:45:40 +0200 Subject: [PATCH 3/9] feat: working flow & references with fixes --- demo/src/ValidatorView.tsx | 472 ++++++++++-- demo/src/chat/ChatInput.tsx | 15 +- demo/src/chat/ChatSettings.tsx | 92 ++- demo/src/llm-client.ts | 28 +- demo/src/styles.css | 389 +++++++++- demo/src/validator-prompts.ts | 60 +- .../assertions/fixer-preserves-components.mjs | 24 + evals/assertions/fixer-resolves-errors.mjs | 66 ++ evals/assertions/no-multi-step-flow.mjs | 33 + evals/assertions/no-placeholder-content.mjs | 55 ++ evals/package.json | 4 +- evals/prompt-fixer.mjs | 36 + evals/promptfooconfig.fixer-flow.yaml | 26 + evals/promptfooconfig.fixer.yaml | 26 + evals/tests-fixer-flow.yaml | 296 ++++++++ evals/tests-fixer.yaml | 717 ++++++++++++++++++ packages/prompt-pack/src/index.ts | 16 + packages/prompt-pack/src/loader.ts | 2 + .../prompt-pack/src/prompts/mdma-fixer.ts | 249 ++++++ packages/prompt-pack/tests/loader.test.ts | 5 +- .../validator/src/fixes/schema-defaults.ts | 8 +- packages/validator/src/index.ts | 7 + packages/validator/src/rules/flow-ordering.ts | 123 ++- packages/validator/src/types.ts | 7 + packages/validator/src/validate-flow.ts | 157 ++++ .../tests/rules/flow-ordering.test.ts | 62 +- 26 files changed, 2846 insertions(+), 129 deletions(-) create mode 100644 evals/assertions/fixer-preserves-components.mjs create mode 100644 evals/assertions/fixer-resolves-errors.mjs create mode 100644 evals/assertions/no-multi-step-flow.mjs create mode 100644 evals/assertions/no-placeholder-content.mjs create mode 100644 evals/prompt-fixer.mjs create mode 100644 evals/promptfooconfig.fixer-flow.yaml create mode 100644 evals/promptfooconfig.fixer.yaml create mode 100644 evals/tests-fixer-flow.yaml create mode 100644 evals/tests-fixer.yaml create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer.ts create mode 100644 packages/validator/src/validate-flow.ts diff --git a/demo/src/ValidatorView.tsx b/demo/src/ValidatorView.tsx index 24b155a..40453c3 100644 --- a/demo/src/ValidatorView.tsx +++ b/demo/src/ValidatorView.tsx @@ -3,9 +3,11 @@ import { useChat } from './chat/use-chat.js'; import { ChatSettings } from './chat/ChatSettings.js'; import { ChatMessage } from './chat/ChatMessage.js'; import { ChatInput } from './chat/ChatInput.js'; -import { validate, type ValidationResult, type ValidationIssue } from '@mobile-reality/mdma-validator'; +import { validate, validateFlow, type ValidationResult, type ValidationIssue, type FlowValidationResult } from '@mobile-reality/mdma-validator'; +import { buildFixerPrompt, buildFixerMessage, buildSystemPrompt } from '@mobile-reality/mdma-prompt-pack'; +import { chatCompletion } from './llm-client.js'; import { customizations } from './custom-components.js'; -import { VALIDATOR_PROMPT_VARIANTS } from './validator-prompts.js'; +import { VALIDATOR_PROMPT_VARIANTS, FIXER_FLOW_RULES, FLOW_STEPS } from './validator-prompts.js'; function severityClass(severity: string): string { if (severity === 'error') return 'validator-severity--error'; @@ -29,7 +31,13 @@ function IssueRow({ issue }: { issue: ValidationIssue }) { ); } -function ValidationPanel({ results }: { results: Map }) { +interface ValidationPanelProps { + results: Map; + onRequestFix?: (msgId: number) => void; + isGenerating?: boolean; +} + +function ValidationPanel({ results, onRequestFix, isGenerating }: ValidationPanelProps) { const entries = useMemo( () => Array.from(results.entries()).reverse(), [results], @@ -47,58 +55,130 @@ function ValidationPanel({ results }: { results: Map } return (
- {entries.map(([msgId, result]) => ( -
-
- - {result.ok ? 'PASS' : 'FAIL'} - - msg #{msgId} - - {result.summary.errors > 0 && ( - - {result.summary.errors} error{result.summary.errors > 1 ? 's' : ''} - - )} - {result.summary.warnings > 0 && ( - - {result.summary.warnings} warning{result.summary.warnings > 1 ? 's' : ''} - - )} - {result.summary.infos > 0 && ( - - {result.summary.infos} info{result.summary.infos > 1 ? 's' : ''} - - )} - {result.fixCount > 0 && ( - - {result.fixCount} auto-fixed - - )} - -
+ {entries.map(([msgId, result]) => { + const unfixedErrors = result.issues.filter((i) => !i.fixed && i.severity === 'error'); + const unfixedWarnings = result.issues.filter((i) => !i.fixed && i.severity === 'warning'); + const hasUnfixed = unfixedErrors.length > 0 || unfixedWarnings.length > 0; - {result.issues.length > 0 && ( -
-

Issues ({result.issues.length})

-
- {result.issues.map((issue, i) => ( - - ))} -
+ return ( +
+
+ + {result.ok ? 'PASS' : 'FAIL'} + + msg #{msgId} + + {result.summary.errors > 0 && ( + + {result.summary.errors} error{result.summary.errors > 1 ? 's' : ''} + + )} + {result.summary.warnings > 0 && ( + + {result.summary.warnings} warning{result.summary.warnings > 1 ? 's' : ''} + + )} + {result.summary.infos > 0 && ( + + {result.summary.infos} info{result.summary.infos > 1 ? 's' : ''} + + )} + {result.fixCount > 0 && ( + + {result.fixCount} auto-fixed + + )} +
- )} - {result.fixCount > 0 && ( -
-
- View fixed output - 
{result.output}
-
+ {result.issues.length > 0 && ( +
+

Issues ({result.issues.length})

+
+ {result.issues.map((issue, i) => ( + + ))} +
+
+ )} + + {hasUnfixed && onRequestFix && ( + + )} + + {result.fixCount > 0 && ( +
+
+ View fixed output + 
{result.output}
+
+
+ )} +
+ ); + })} +
+ ); +} + +function FlowProgressPanel({ steps, result }: { + steps: import('@mobile-reality/mdma-validator').FlowStepDefinition[]; + result: FlowValidationResult | null; +}) { + // Match issues to steps by "Step N" prefix + const stepStatuses = steps.map((step, i) => { + if (!result) return 'pending' as const; + const stepPrefix = `Step ${i + 1} `; + const matchingIssue = result.issues.find((iss) => iss.message.startsWith(stepPrefix)); + if (!matchingIssue) { + // Check "not yet shown" + const notShown = result.issues.find((iss) => iss.message.includes(step.id) && iss.message.includes('not yet shown')); + return notShown ? 'pending' as const : 'pending' as const; + } + if (matchingIssue.severity === 'info' && matchingIssue.message.includes('correct')) return 'done' as const; + if (matchingIssue.severity === 'error') return 'error' as const; + return 'pending' as const; + }); + + const completedSteps = stepStatuses.filter((s) => s === 'done').length; + + return ( +
+

Flow Progress

+
+ {steps.map((step, i) => { + const status = stepStatuses[i]; + const stepPrefix = `Step ${i + 1} `; + const issue = result?.issues.find((iss) => iss.message.startsWith(stepPrefix) && iss.severity === 'error'); + + return ( +
+ {i + 1} + {step.label} + {step.type}#{step.id} + {status === 'done' && done} + {status === 'error' && issue && ( + {issue.message} + )}
- )} -
- ))} + ); + })} +
+
+ {completedSteps}/{steps.length} steps completed + {result && !result.ok && ( + + flow errors detected + + )} +
); } @@ -126,8 +206,59 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { ...(customizations.schemas && { parserOptions: { customSchemas: customizations.schemas } }), }); + const [fixerModel, setFixerModel] = useState(() => + localStorage.getItem('mdma-fixer-model') || '', + ); + const [customFixerModel, setCustomFixerModel] = useState(() => + localStorage.getItem('mdma-fixer-custom-model') || '', + ); + const [autoFixWithLlm, setAutoFixWithLlm] = useState(() => + localStorage.getItem('mdma-auto-fix-llm') !== 'false', + ); + const [validationResults, setValidationResults] = useState>(new Map()); + // Subscribe to user-action events to auto-advance the conversation + const subscribedStores = useRef(new Set()); + const pendingSendRef = useRef(false); + const setInputRef = useRef(setInput); + setInputRef.current = setInput; + const flowCompleteRef = useRef(false); + const [showFlowComplete, setShowFlowComplete] = useState(false); + + useEffect(() => { + const ADVANCE_EVENTS = ['ACTION_TRIGGERED', 'APPROVAL_GRANTED', 'APPROVAL_DENIED'] as const; + for (const msg of messages) { + if (msg.role === 'assistant' && msg.store && !subscribedStores.current.has(msg.store)) { + subscribedStores.current.add(msg.store); + for (const eventType of ADVANCE_EVENTS) { + msg.store.getEventBus().on(eventType, () => { + if (flowCompleteRef.current) { + setShowFlowComplete(true); + return; + } + setTimeout(() => { + pendingSendRef.current = true; + setInputRef.current('Continue to the next step'); + }, 500); + }); + } + } + } + }, [messages]); + + // Auto-send when pending action trigger sets the input + useEffect(() => { + if (pendingSendRef.current && input.trim() && !isGenerating) { + pendingSendRef.current = false; + send(); + } + }, [input, isGenerating, send]); + + useEffect(() => { + return () => { subscribedStores.current.clear(); }; + }, []); + // Track which messages we've already validated const validatedRef = useRef>(new Set()); @@ -137,7 +268,23 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { for (const msg of messages) { if (msg.role === 'assistant' && msg.content && !validatedRef.current.has(msg.id)) { validatedRef.current.add(msg.id); - const result = validate(msg.content); + + // Collect component IDs from all prior assistant messages + // so flow-ordering can detect regenerated steps + const priorComponentIds: string[] = []; + for (const prev of messages) { + if (prev.id >= msg.id) break; + if (prev.role !== 'assistant' || !prev.content) continue; + const idMatches = prev.content.matchAll(/```mdma[\s\S]*?```/g); + for (const match of idMatches) { + const idMatch = match[0].match(/id:\s*(\S+)/); + if (idMatch) priorComponentIds.push(idMatch[1]); + } + } + + const result = validate(msg.content, { + ...(priorComponentIds.length > 0 && { priorComponentIds }), + }); setValidationResults((prev) => { const next = new Map(prev); next.set(msg.id, result); @@ -168,6 +315,101 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { validatedRef.current = new Set(); }, [clear]); + const [fixingMsgId, setFixingMsgId] = useState(null); + const isFixing = fixingMsgId !== null; + const fixAbortRef = useRef(null); + + const handleRequestFix = useCallback(async (msgId: number) => { + const result = validationResults.get(msgId); + const msg = messages.find((m) => m.id === msgId); + if (!result || !msg || isFixing) return; + + const unfixed = result.issues.filter((i) => !i.fixed && (i.severity === 'error' || i.severity === 'warning')); + if (unfixed.length === 0) return; + + setFixingMsgId(msgId); + fixAbortRef.current = new AbortController(); + + try { + const systemPrompt = `${buildSystemPrompt()}\n\n---\n\n${buildFixerPrompt(promptKey)}`; + + // Build conversation history from messages before the broken one + const history = messages + .filter((m) => m.id < msgId) + .map((m) => ({ role: m.role as 'user' | 'assistant', content: m.content })); + + const userMessage = buildFixerMessage(result.output, unfixed, { + conversationHistory: history.length > 0 ? history : undefined, + promptContext: FIXER_FLOW_RULES[promptKey] ?? variant.prompt, + }); + + const resolvedModel = fixerModel === '__custom__' ? customFixerModel : fixerModel; + const fixerConfig = resolvedModel + ? { ...config, model: resolvedModel } + : config; + + const fixedContent = await chatCompletion( + fixerConfig, + [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: userMessage }, + ], + fixAbortRef.current.signal, + ); + + if (fixedContent) { + // Overwrite the original message with the fixed content + updateMessage(msg.id, fixedContent); + // Clear old validation result so it gets re-validated + validatedRef.current.delete(msg.id); + setValidationResults((prev) => { + const next = new Map(prev); + next.delete(msg.id); + return next; + }); + } + } catch (err) { + if (!(err instanceof DOMException && err.name === 'AbortError')) { + console.error('Fixer error:', err); + } + } finally { + setFixingMsgId(null); + fixAbortRef.current = null; + } + }, [validationResults, messages, config, isFixing, updateMessage, variant.prompt, fixerModel, customFixerModel, promptKey]); + + // Auto-fix with LLM when enabled and unfixed issues detected + const autoFixTriggeredRef = useRef>(new Set()); + useEffect(() => { + if (!autoFixWithLlm || isFixing || isGenerating) return; + for (const [msgId, result] of validationResults) { + if (autoFixTriggeredRef.current.has(msgId)) continue; + const unfixed = result.issues.filter((i) => !i.fixed && (i.severity === 'error' || i.severity === 'warning')); + if (unfixed.length > 0) { + autoFixTriggeredRef.current.add(msgId); + setTimeout(() => handleRequestFix(msgId), 300); + break; + } + } + }, [validationResults, autoFixWithLlm, isFixing, isGenerating, handleRequestFix]); + + // Flow validation — run against all assistant messages when steps are defined + const flowSteps = FLOW_STEPS[promptKey]; + const flowResult = useMemo(() => { + if (!flowSteps) return null; + const assistantContents = messages + .filter((m) => m.role === 'assistant' && m.content) + .map((m) => m.content); + if (assistantContents.length === 0) return null; + return validateFlow(assistantContents, { steps: flowSteps }); + }, [flowSteps, messages]); + + // All flow steps completed — check by counting info "correct" issues + const flowComplete = flowSteps != null && flowResult != null + && flowResult.issues.filter((i) => i.severity === 'info' && i.message.includes('correct')).length >= flowSteps.length; + + flowCompleteRef.current = flowComplete; + const lastMsgId = messages[messages.length - 1]?.id; return ( @@ -191,12 +433,19 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { )} {messages.map((msg) => ( - +
+ + {fixingMsgId === msg.id && ( +
+
+ Fixing with LLM... +
+ )} +
))} {error &&
{error}
} @@ -213,10 +462,115 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { isGenerating={isGenerating} hasMessages={messages.length > 0} inputRef={inputRef} + disabled={flowComplete} + placeholder={flowComplete + ? 'Flow completed — all steps validated successfully' + : undefined} + /> +
+ +
+
+

Fixer Settings

+ +
+ Model + +
+ {fixerModel === '__custom__' && ( +
+ Custom Model ID + { + setCustomFixerModel(e.target.value); + localStorage.setItem('mdma-fixer-custom-model', e.target.value); + }} + placeholder="e.g. openrouter/auto" + /> +
+ )} +
+ + {flowSteps && ( + + )} +
- + {showFlowComplete && ( +
setShowFlowComplete(false)}> +
e.stopPropagation()}> +
✓
+

Flow Completed!

+

All {flowSteps?.length} steps have been validated successfully.

+ +
+
+ )}
); } diff --git a/demo/src/chat/ChatInput.tsx b/demo/src/chat/ChatInput.tsx index 373191a..a3ea67a 100644 --- a/demo/src/chat/ChatInput.tsx +++ b/demo/src/chat/ChatInput.tsx @@ -9,6 +9,10 @@ export interface ChatInputProps { isGenerating: boolean; hasMessages: boolean; inputRef: RefObject; + /** When true, the input is disabled (e.g. flow completed). */ + disabled?: boolean; + /** Placeholder text override. */ + placeholder?: string; } export const ChatInput = memo(function ChatInput({ @@ -20,7 +24,11 @@ export const ChatInput = memo(function ChatInput({ isGenerating, hasMessages, inputRef, + disabled, + placeholder, }: ChatInputProps) { + const isDisabled = disabled && !isGenerating; + return (
{hasMessages && ( @@ -38,12 +46,13 @@ export const ChatInput = memo(function ChatInput({ className="chat-input" value={value} onChange={(e) => onChange(e.target.value)} - placeholder="Describe the interactive document you need..." + placeholder={placeholder ?? "Describe the interactive document you need..."} rows={2} + disabled={isDisabled} onKeyDown={(e) => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); - onSend(); + if (!isDisabled) onSend(); } }} /> @@ -56,7 +65,7 @@ export const ChatInput = memo(function ChatInput({ type="button" className="chat-send-btn" onClick={onSend} - disabled={!value.trim()} + disabled={isDisabled || !value.trim()} > Send diff --git a/demo/src/chat/ChatSettings.tsx b/demo/src/chat/ChatSettings.tsx index c5e16aa..53604d6 100644 --- a/demo/src/chat/ChatSettings.tsx +++ b/demo/src/chat/ChatSettings.tsx @@ -1,6 +1,50 @@ import { memo, useState } from 'react'; import { PROVIDER_PRESETS, type LlmConfig } from '../llm-client.js'; +const MODEL_OPTIONS: Record> = { + openai: [ + { value: 'gpt-5.4', label: 'gpt-5.4' }, + { value: 'gpt-5.4-mini', label: 'gpt-5.4-mini' }, + { value: 'gpt-5.4-nano', label: 'gpt-5.4-nano' }, + { value: 'gpt-5.3-codex', label: 'gpt-5.3-codex' }, + { value: 'gpt-5-mini', label: 'gpt-5-mini' }, + { value: 'o3', label: 'o3' }, + { value: 'o3-pro', label: 'o3-pro' }, + { value: 'o4-mini', label: 'o4-mini' }, + { value: 'gpt-4.1', label: 'gpt-4.1' }, + { value: 'gpt-4.1-mini', label: 'gpt-4.1-mini' }, + ], + anthropic: [ + { value: 'claude-opus-4-6', label: 'claude-opus-4.6' }, + { value: 'claude-sonnet-4-6', label: 'claude-sonnet-4.6' }, + { value: 'claude-haiku-4-5-20251001', label: 'claude-haiku-4.5' }, + { value: 'claude-sonnet-4-5-20250929', label: 'claude-sonnet-4.5' }, + ], + gemini: [ + { value: 'gemini-2.5-pro', label: 'gemini-2.5-pro' }, + { value: 'gemini-2.5-flash', label: 'gemini-2.5-flash' }, + { value: 'gemini-2.0-flash', label: 'gemini-2.0-flash' }, + ], + openrouter: [ + { value: 'openai/gpt-5.4', label: 'openai/gpt-5.4' }, + { value: 'openai/gpt-5.4-mini', label: 'openai/gpt-5.4-mini' }, + { value: 'anthropic/claude-opus-4-6', label: 'anthropic/claude-opus-4.6' }, + { value: 'anthropic/claude-sonnet-4-6', label: 'anthropic/claude-sonnet-4.6' }, + { value: 'google/gemini-2.5-pro', label: 'google/gemini-2.5-pro' }, + { value: 'google/gemini-2.5-flash', label: 'google/gemini-2.5-flash' }, + { value: 'meta-llama/llama-4-scout', label: 'meta-llama/llama-4-scout' }, + { value: 'deepseek/deepseek-r1', label: 'deepseek/deepseek-r1' }, + { value: 'qwen/qwen3-235b', label: 'qwen/qwen3-235b' }, + ], +}; + +function detectProvider(baseUrl: string): string | null { + for (const [name, preset] of Object.entries(PROVIDER_PRESETS)) { + if (baseUrl === preset.baseUrl) return name; + } + return null; +} + export interface ChatSettingsProps { config: LlmConfig; onUpdate: (patch: Partial) => void; @@ -14,6 +58,11 @@ export const ChatSettings = memo(function ChatSettings({ }: ChatSettingsProps) { const [open, setOpen] = useState(false); + const provider = detectProvider(config.baseUrl); + const models = provider ? MODEL_OPTIONS[provider] : null; + const isKnownModel = models?.some((m) => m.value === config.model); + const isCustom = !models || !isKnownModel; + return (
)} diff --git a/demo/src/llm-client.ts b/demo/src/llm-client.ts index c1ea02e..3899448 100644 --- a/demo/src/llm-client.ts +++ b/demo/src/llm-client.ts @@ -20,31 +20,31 @@ export interface ChatMessage { } export const DEFAULT_CONFIG: LlmConfig = { - baseUrl: 'http://localhost:11434/v1', + baseUrl: 'https://api.openai.com/v1', apiKey: '', - model: 'llama3', + model: 'gpt-5.4-mini', }; export const PROVIDER_PRESETS: Record = { - ollama: { - baseUrl: 'http://localhost:11434/v1', - apiKey: '', - model: 'llama3', - }, openai: { baseUrl: 'https://api.openai.com/v1', apiKey: '', - model: 'gpt-4o', + model: 'gpt-5.4-mini', }, anthropic: { baseUrl: 'https://api.anthropic.com/v1', apiKey: '', - model: 'claude-sonnet-4-5-20250929', + model: 'claude-sonnet-4-6', + }, + gemini: { + baseUrl: 'https://generativelanguage.googleapis.com/v1beta/openai', + apiKey: '', + model: 'gemini-2.5-flash', }, - groq: { - baseUrl: 'https://api.groq.com/openai/v1', + openrouter: { + baseUrl: 'https://openrouter.ai/api/v1', apiKey: '', - model: 'llama-3.3-70b-versatile', + model: 'openai/gpt-5.4-mini', }, }; @@ -63,7 +63,7 @@ export async function* streamChatCompletion( 'Content-Type': 'application/json', }; if (config.apiKey) { - headers['Authorization'] = `Bearer ${config.apiKey}`; + headers.Authorization = `Bearer ${config.apiKey}`; } const response = await fetch(url, { @@ -127,7 +127,7 @@ export async function chatCompletion( 'Content-Type': 'application/json', }; if (config.apiKey) { - headers['Authorization'] = `Bearer ${config.apiKey}`; + headers.Authorization = `Bearer ${config.apiKey}`; } const response = await fetch(url, { diff --git a/demo/src/styles.css b/demo/src/styles.css index 949b43e..ca29b54 100644 --- a/demo/src/styles.css +++ b/demo/src/styles.css @@ -1035,14 +1035,13 @@ body { } .chat-settings-fields { - display: flex; - gap: 12px; - flex-wrap: wrap; + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px 12px; } -.chat-settings-fields .ai-setting { - flex: 1; - min-width: 160px; +.chat-settings-fields .ai-setting:first-child { + grid-column: 1 / -1; } .ai-settings-presets { @@ -1052,12 +1051,12 @@ body { } .ai-preset-btn { - padding: 3px 8px; - font-size: 11px; - font-weight: 500; + padding: 5px 14px; + font-size: 12px; + font-weight: 600; text-transform: capitalize; border: 1px solid #d1d5db; - border-radius: 4px; + border-radius: 6px; background: #fff; color: #555; cursor: pointer; @@ -1078,35 +1077,51 @@ body { .ai-setting { display: flex; flex-direction: column; - gap: 2px; - margin-bottom: 8px; -} - -.ai-setting:last-child { - margin-bottom: 0; + gap: 3px; } .ai-setting span { font-size: 11px; font-weight: 600; color: #555; + text-transform: uppercase; + letter-spacing: 0.3px; } -.ai-setting input { - padding: 5px 8px; - font-size: 12px; +.ai-setting input, +.ai-setting select { + padding: 6px 10px; + font-size: 13px; border: 1px solid #d1d5db; - border-radius: 5px; + border-radius: 6px; background: #fff; color: #1a1a2e; outline: none; transition: border-color 0.15s; + width: 100%; + box-sizing: border-box; } -.ai-setting input:focus { +.ai-setting input:focus, +.ai-setting select:focus { border-color: #6c5ce7; } +.ai-setting select { + cursor: pointer; +} + +.ai-setting-model-group { + display: flex; + gap: 6px; +} + +.ai-setting-model-group select, +.ai-setting-model-group input { + flex: 1; + min-width: 0; +} + /* ===== Chat Messages ===== */ .chat-messages { flex: 1; @@ -2408,7 +2423,7 @@ body { overflow: hidden; } -.validator-results-panel { +.validator-side-panel { width: 520px; flex-shrink: 0; display: flex; @@ -2418,6 +2433,193 @@ body { border-left: 1px solid #e0e0e0; } +.validator-results-panel { + flex: 1; + display: flex; + flex-direction: column; + overflow-y: auto; +} + +/* Flow progress panel */ +.flow-progress-panel { + padding: 12px 16px; + border-bottom: 2px solid #d0d0d0; + background: #f0f4ff; +} + +.flow-progress-panel h3 { + margin: 0 0 8px 0; + font-size: 13px; + font-weight: 600; + color: #374151; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.flow-steps { + display: flex; + flex-direction: column; + gap: 6px; +} + +.flow-step { + display: flex; + align-items: center; + gap: 8px; + padding: 6px 10px; + border-radius: 6px; + font-size: 13px; + background: #fff; + border: 1px solid #e0e0e0; +} + +.flow-step--done { + background: #ecfdf5; + border-color: #6ee7b7; +} + +.flow-step--error { + background: #fef2f2; + border-color: #fca5a5; +} + +.flow-step-num { + width: 22px; + height: 22px; + border-radius: 50%; + background: #e5e7eb; + display: flex; + align-items: center; + justify-content: center; + font-weight: 700; + font-size: 12px; + color: #6b7280; + flex-shrink: 0; +} + +.flow-step--done .flow-step-num { + background: #10b981; + color: #fff; +} + +.flow-step--error .flow-step-num { + background: #ef4444; + color: #fff; +} + +.flow-step-label { + font-weight: 600; + color: #374151; +} + +.flow-step-type { + font-size: 11px; + color: #9ca3af; + font-family: monospace; +} + +.flow-step-badge { + margin-left: auto; + font-size: 11px; + padding: 2px 6px; + border-radius: 4px; + font-weight: 600; +} + +.flow-step-badge--done { + background: #d1fae5; + color: #065f46; +} + +.flow-step-badge--error { + background: #fee2e2; + color: #991b1b; + max-width: 200px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.flow-progress-summary { + margin-top: 8px; + font-size: 12px; + color: #6b7280; + font-weight: 500; +} + +/* Flow complete modal */ +.flow-complete-overlay { + position: fixed; + inset: 0; + background: rgba(0, 0, 0, 0.5); + display: flex; + align-items: center; + justify-content: center; + z-index: 1000; + animation: fadeIn 0.2s ease; +} + +.flow-complete-modal { + background: #fff; + border-radius: 16px; + padding: 40px 48px; + text-align: center; + box-shadow: 0 20px 60px rgba(0, 0, 0, 0.2); + max-width: 400px; + animation: scaleIn 0.3s ease; +} + +.flow-complete-icon { + width: 64px; + height: 64px; + border-radius: 50%; + background: linear-gradient(135deg, #10b981, #059669); + color: #fff; + font-size: 32px; + display: flex; + align-items: center; + justify-content: center; + margin: 0 auto 16px; +} + +.flow-complete-modal h2 { + margin: 0 0 8px; + font-size: 24px; + color: #111827; +} + +.flow-complete-modal p { + margin: 0 0 24px; + color: #6b7280; + font-size: 15px; +} + +.flow-complete-btn { + padding: 10px 32px; + border-radius: 8px; + border: none; + background: linear-gradient(135deg, #10b981, #059669); + color: #fff; + font-size: 15px; + font-weight: 600; + cursor: pointer; + transition: opacity 0.2s; +} + +.flow-complete-btn:hover { + opacity: 0.9; +} + +@keyframes fadeIn { + from { opacity: 0; } + to { opacity: 1; } +} + +@keyframes scaleIn { + from { transform: scale(0.9); opacity: 0; } + to { transform: scale(1); opacity: 1; } +} + /* Manual paste-and-validate section */ .manual-validator { border-bottom: 2px solid #d0d0d0; @@ -2791,6 +2993,149 @@ body { overflow-y: auto; } +/* Fixer settings panel (right sidebar) */ +.fixer-settings-panel { + padding: 12px 16px; + border-bottom: 1px solid #e0e0e0; + background: #f9fafb; +} + +.fixer-settings-panel h3 { + margin: 0 0 8px; + font-size: 11px; + font-weight: 700; + color: #374151; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.fixer-settings-checkbox { + display: flex; + align-items: center; + gap: 6px; + margin-bottom: 10px; + cursor: pointer; +} + +.fixer-settings-checkbox input[type="checkbox"] { + width: 16px; + height: 16px; + accent-color: #6c5ce7; + cursor: pointer; +} + +.fixer-settings-checkbox span { + font-size: 12px; + font-weight: 500; + color: #374151; + text-transform: none; + letter-spacing: 0; +} + +.fixer-settings-field { + display: flex; + flex-direction: column; + gap: 3px; + margin-bottom: 8px; +} + +.fixer-settings-field:last-child { + margin-bottom: 0; +} + +.fixer-settings-field span { + font-size: 11px; + font-weight: 600; + color: #6b7280; + text-transform: uppercase; + letter-spacing: 0.3px; +} + +.fixer-settings-field select, +.fixer-settings-field input { + padding: 6px 10px; + font-size: 13px; + border: 1px solid #d1d5db; + border-radius: 6px; + background: #fff; + color: #1a1a2e; + outline: none; + width: 100%; + box-sizing: border-box; + transition: border-color 0.15s; +} + +.fixer-settings-field select { + cursor: pointer; +} + +.fixer-settings-field input:focus, +.fixer-settings-field select:focus { + border-color: #6c5ce7; +} + +/* Fixing overlay on message */ +.validator-msg-wrapper { + position: relative; +} + +.validator-fixing-overlay { + position: absolute; + inset: 0; + background: rgba(255, 255, 255, 0.85); + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 12px; + border-radius: 8px; + z-index: 10; + animation: fadeIn 0.2s ease; +} + +.validator-fixing-overlay span { + font-size: 13px; + font-weight: 600; + color: #6c5ce7; +} + +.validator-fixing-spinner { + width: 32px; + height: 32px; + border: 3px solid #e5e7eb; + border-top-color: #6c5ce7; + border-radius: 50%; + animation: spin 0.8s linear infinite; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} + +.validator-fix-btn { + display: block; + width: calc(100% - 32px); + margin: 8px 16px; + padding: 8px 16px; + border: none; + border-radius: 6px; + background: #e74c3c; + color: #fff; + font-size: 13px; + font-weight: 600; + cursor: pointer; + transition: background 0.15s; +} + +.validator-fix-btn:hover:not(:disabled) { + background: #c0392b; +} + +.validator-fix-btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + /* ─── Stepper Tab ───────────────────────────────────────────────────────── */ .stepper-info { diff --git a/demo/src/validator-prompts.ts b/demo/src/validator-prompts.ts index 1ca2808..af924fd 100644 --- a/demo/src/validator-prompts.ts +++ b/demo/src/validator-prompts.ts @@ -145,14 +145,32 @@ Generate a dashboard with 2 tables and 2 charts — a sales summary and a user a rules: ['flow-ordering', 'unreferenced-components', 'action-references'], prompt: `${PREAMBLE} -Focus ONLY on component flow and reference issues: +Focus ONLY on component flow and reference issues. Generate a user registration and approval workflow with ALL of these intentional problems: -1. **Backward action references** — Make onSubmit or onAction point to a component defined EARLIER in the document (target appears before the source) -2. **Circular references** — Create a cycle: component A's onSubmit points to B, and B's onAction points back to A -3. **Unreferenced components** — Add a callout or table that no other component references via bindings or action fields (orphan component) -4. **Invalid action targets** — Use onSubmit, onAction, onComplete, onApprove, onDeny pointing to IDs that don't exist +## Required broken structure -Generate a multi-step workflow (form submission → approval → notification) with 6+ components — but intentionally create circular dependencies, orphaned components, and broken action chains.`, +Generate exactly these components in ONE message (this is intentionally wrong — the validator should catch it): + +1. \`\`\`mdma block: **form** id: \`registration-form\` with fields: full-name (text, required), email (email, required, sensitive), department (select with options: Engineering, Marketing, Sales) + - Set \`onSubmit: approval-gate\` (this creates a multi-step flow error — form targets an interactive component) + +2. \`\`\`mdma block: **approval-gate** id: \`approval-gate\` title: "Manager Approval" + - Set \`onApprove: registration-form\` (this creates a circular reference — approval points back to form) + - Set \`onDeny: nonexistent-rejection\` (this is an invalid action target — ID doesn't exist) + +3. \`\`\`mdma block: **button** id: \`notify-btn\` text: "Send Notification" + - Set \`onAction: approval-gate\` (backward reference + multi-step chain) + +4. \`\`\`mdma block: **callout** id: \`orphan-notice\` variant: info, content: "This notice is not referenced by anything" + (orphaned component — no other component points to it) + +5. \`\`\`mdma block: **callout** id: \`orphan-table-info\` variant: warning, content: "Another orphan" + (second orphaned component) + +6. \`\`\`mdma block: **webhook** id: \`notify-webhook\` url: https://api.example.com/notify, method: POST + - Set \`trigger: missing-component\` (invalid action target — ID doesn't exist) + +Generate all 6 components in a single message. The validator should detect: multi-step flow errors, circular references, orphaned components, invalid action targets, and backward references.`, }, { key: 'approval', @@ -172,3 +190,33 @@ Focus ONLY on approval gate and webhook issues: Generate an expense approval workflow with: a form for expense details, an approval-gate for manager review, a webhook for notification, and callouts for status — but use the wrong field names and broken references.`, }, ]; + +/** + * Defines the correct flow for each variant so the fixer knows what each step + * should look like. Passed as promptContext to buildFixerMessage(). + * + * Only variants that involve multi-step workflows need entries here. + */ +export const FIXER_FLOW_RULES: Record = { + flow: `This is a user registration and approval workflow. The correct flow split across conversation turns: + +- **Step 1:** Form \`registration-form\` with fields: full-name (text, required), email (email, required, sensitive), department (select). Include a success callout \`registration-submitted\`. The form's onSubmit should point to the callout. + +- **Step 2:** Approval gate \`approval-gate\` with title "Manager Approval", description "Please review and approve this registration." Include a callout \`approval-complete\` (variant: success). The gate's onApprove should point to the callout. + +- **Step 3:** Webhook \`notify-webhook\` (url: https://api.example.com/notify, method: POST) triggered by a button \`send-notification\` (text: "Send Notification"). Include a success callout \`workflow-complete\`. The webhook's trigger should point to the button. + +Each step must contain exactly ONE interactive component + its supporting callouts/webhooks. Do not include components from other steps.`, +}; + +/** + * Structured flow step definitions for deterministic validation via validateFlow(). + * Keyed by variant key — only variants with multi-step workflows need entries. + */ +export const FLOW_STEPS: Record = { + flow: [ + { label: 'Registration Form', type: 'form', id: 'registration-form' }, + { label: 'Manager Approval', type: 'approval-gate', id: 'approval-gate' }, + { label: 'Send Notification & Webhook', type: 'webhook', id: 'notify-webhook' }, + ], +}; diff --git a/evals/assertions/fixer-preserves-components.mjs b/evals/assertions/fixer-preserves-components.mjs new file mode 100644 index 0000000..4461b91 --- /dev/null +++ b/evals/assertions/fixer-preserves-components.mjs @@ -0,0 +1,24 @@ +/** + * Custom promptfoo assertion for fixer eval. + * + * Verifies that the fixer didn't drop components. The fixed output + * should contain at least config.min mdma blocks (default: same as input). + */ +export default function (output, { config } = {}) { + const min = config?.min ?? 1; + const blockCount = (output.match(/```mdma/g) ?? []).length; + + if (blockCount < min) { + return { + pass: false, + score: 0, + reason: `Fixer output has ${blockCount} mdma block(s) but expected at least ${min}`, + }; + } + + return { + pass: true, + score: 1, + reason: `Fixer preserved ${blockCount} mdma block(s) (min: ${min})`, + }; +} diff --git a/evals/assertions/fixer-resolves-errors.mjs b/evals/assertions/fixer-resolves-errors.mjs new file mode 100644 index 0000000..f2cb997 --- /dev/null +++ b/evals/assertions/fixer-resolves-errors.mjs @@ -0,0 +1,66 @@ +import { validate } from '@mobile-reality/mdma-validator'; + +/** + * Custom promptfoo assertion for fixer eval. + * + * Validates that the LLM-fixed output: + * 1. Contains at least one mdma block (didn't strip everything) + * 2. Has zero unfixed errors after validation + * 3. Reports remaining warnings/infos for transparency + * + * The config.maxWarnings option (default: Infinity) allows tests to assert + * that the fixer also resolved warnings. + */ +export default function (output, { config } = {}) { + const maxWarnings = config?.maxWarnings ?? Infinity; + + // Check the output actually contains mdma blocks + const blockCount = (output.match(/```mdma/g) ?? []).length; + if (blockCount === 0) { + return { + pass: false, + score: 0, + reason: 'Fixer output contains no ```mdma blocks — the LLM may have stripped the document', + }; + } + + const result = validate(output, { + exclude: ['thinking-block'], + autoFix: false, + }); + + const unfixedErrors = result.issues.filter( + (i) => i.severity === 'error', + ); + const unfixedWarnings = result.issues.filter( + (i) => i.severity === 'warning', + ); + + if (unfixedErrors.length > 0) { + const details = unfixedErrors + .map((i) => `[${i.ruleId}] ${i.componentId ?? '?'}: ${i.message}`) + .join('\n'); + return { + pass: false, + score: 0, + reason: `Fixer output still has ${unfixedErrors.length} error(s):\n${details}`, + }; + } + + if (unfixedWarnings.length > maxWarnings) { + const details = unfixedWarnings + .map((i) => `[${i.ruleId}] ${i.componentId ?? '?'}: ${i.message}`) + .join('\n'); + return { + pass: false, + score: 0.5, + reason: `Fixer output has ${unfixedWarnings.length} warning(s) (max ${maxWarnings}):\n${details}`, + }; + } + + return { + pass: true, + score: 1, + reason: `Fixer resolved all errors (${result.summary.warnings} warnings, ${result.summary.infos} info, ${blockCount} blocks)`, + }; +} diff --git a/evals/assertions/no-multi-step-flow.mjs b/evals/assertions/no-multi-step-flow.mjs new file mode 100644 index 0000000..634f03c --- /dev/null +++ b/evals/assertions/no-multi-step-flow.mjs @@ -0,0 +1,33 @@ +import { validate } from '@mobile-reality/mdma-validator'; + +/** + * Custom promptfoo assertion for fixer eval. + * + * Verifies that the fixer output has no flow-ordering errors. + * This relies on the validator's own logic for detecting multi-step + * flows, circular references, and multiple interactive types. + */ +export default function (output) { + const result = validate(output, { + exclude: ['thinking-block'], + autoFix: false, + }); + + const flowErrors = result.issues.filter( + (i) => i.ruleId === 'flow-ordering' && i.severity === 'error', + ); + + if (flowErrors.length > 0) { + return { + pass: false, + score: 0, + reason: `Fixer output still has ${flowErrors.length} flow-ordering error(s):\n${flowErrors.map((i) => i.message).join('\n')}`, + }; + } + + return { + pass: true, + score: 1, + reason: 'No flow-ordering errors', + }; +} diff --git a/evals/assertions/no-placeholder-content.mjs b/evals/assertions/no-placeholder-content.mjs new file mode 100644 index 0000000..28144b0 --- /dev/null +++ b/evals/assertions/no-placeholder-content.mjs @@ -0,0 +1,55 @@ +/** + * Custom promptfoo assertion that checks for placeholder content + * in visible text and mdma blocks (excluding thinking blocks). + * + * Thinking blocks may mention placeholders as part of reasoning — + * that's fine. We only care about placeholders in rendered content. + */ +const PLACEHOLDER_PATTERNS = [ + /\bTODO\b/i, + /\bTBD\b/i, + /\bFIXME\b/i, + /\bLorem\s*ipsum\b/i, + /^\.{3,}$/m, +]; + +export default function (output) { + // Extract mdma blocks and classify them + const blocks = [...output.matchAll(/```mdma\s*([\s\S]*?)```/g)]; + + for (const block of blocks) { + const yaml = block[1]; + // Skip thinking blocks + if (/^\s*type:\s*thinking\b/m.test(yaml)) continue; + + for (const pattern of PLACEHOLDER_PATTERNS) { + if (pattern.test(yaml)) { + const match = yaml.match(pattern); + return { + pass: false, + score: 0, + reason: `Placeholder content "${match[0]}" found in mdma block`, + }; + } + } + } + + // Check visible prose (everything outside mdma blocks) + const prose = output.replace(/```mdma[\s\S]*?```/g, ''); + for (const pattern of PLACEHOLDER_PATTERNS) { + if (pattern.test(prose)) { + const match = prose.match(pattern); + return { + pass: false, + score: 0, + reason: `Placeholder content "${match[0]}" found in visible text`, + }; + } + } + + return { + pass: true, + score: 1, + reason: 'No placeholder content found in visible output', + }; +} diff --git a/evals/package.json b/evals/package.json index 55ce8c8..6efd256 100644 --- a/evals/package.json +++ b/evals/package.json @@ -8,7 +8,9 @@ "eval:conversation": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.conversation.yaml; exit 0", "eval:prompt-builder": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.prompt-builder.yaml; exit 0", "eval:flows": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.flows.yaml; exit 0", - "eval:all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.custom.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.conversation.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.prompt-builder.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.flows.yaml; exit 0", + "eval:fixer": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer.yaml; exit 0", + "eval:fixer-flow": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer-flow.yaml; exit 0", + "eval:all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.custom.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.conversation.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.prompt-builder.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.flows.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer-flow.yaml; exit 0", "eval:view": "promptfoo view" }, "dependencies": { diff --git a/evals/prompt-fixer.mjs b/evals/prompt-fixer.mjs new file mode 100644 index 0000000..cb61eb7 --- /dev/null +++ b/evals/prompt-fixer.mjs @@ -0,0 +1,36 @@ +import { buildFixerPrompt, buildFixerMessage, buildSystemPrompt } from '@mobile-reality/mdma-prompt-pack'; +import { validate } from '@mobile-reality/mdma-validator'; + +/** + * Promptfoo prompt function for fixer eval tests. + * + * Each test case provides: + * - `brokenDocument` — MDMA markdown with intentional issues + * - `conversationHistory` (optional) — prior messages for multi-step context + * - `promptContext` (optional) — the original prompt that describes expected structure + * - `variantKey` (optional) — validator variant key to select relevant fixer extensions + * + * This function: + * 1. Runs the validator (with autoFix) to fix what it can + * 2. Collects remaining unfixed issues + * 3. Sends the fixer system prompt (with variant-specific extensions) + user message + */ +export default function ({ vars }) { + const result = validate(vars.brokenDocument, { exclude: ['thinking-block'] }); + const unfixed = result.issues.filter( + (i) => !i.fixed && (i.severity === 'error' || i.severity === 'warning'), + ); + + const fixerPrompt = buildFixerPrompt(vars.variantKey ?? undefined); + const systemPrompt = `${buildSystemPrompt()}\n\n---\n\n${fixerPrompt}`; + + const userMessage = buildFixerMessage(result.output, unfixed, { + conversationHistory: vars.conversationHistory ?? undefined, + promptContext: vars.promptContext ?? undefined, + }); + + return [ + { role: 'system', content: `{% raw %}${systemPrompt}{% endraw %}` }, + { role: 'user', content: `{% raw %}${userMessage}{% endraw %}` }, + ]; +} diff --git a/evals/promptfooconfig.fixer-flow.yaml b/evals/promptfooconfig.fixer-flow.yaml new file mode 100644 index 0000000..f68c1e4 --- /dev/null +++ b/evals/promptfooconfig.fixer-flow.yaml @@ -0,0 +1,26 @@ +# MDMA Fixer — Flow & References eval config +# +# Run: pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow +# View: pnpm --filter @mobile-reality/mdma-evals eval:view + +description: MDMA Fixer — Flow & References Eval + +envPath: .env +outputPath: results-fixer-flow.json + +prompts: + - file://prompt-fixer.mjs + +providers: + - openai:gpt-4.1 + +defaultTest: + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 1 + +tests: tests-fixer-flow.yaml diff --git a/evals/promptfooconfig.fixer.yaml b/evals/promptfooconfig.fixer.yaml new file mode 100644 index 0000000..c3fbbc8 --- /dev/null +++ b/evals/promptfooconfig.fixer.yaml @@ -0,0 +1,26 @@ +# MDMA Fixer Prompt — promptfoo evaluation config +# +# Run: pnpm --filter @mobile-reality/mdma-evals eval:fixer +# View: pnpm --filter @mobile-reality/mdma-evals eval:view + +description: MDMA Fixer Prompt Eval + +envPath: .env +outputPath: results-fixer.json + +prompts: + - file://prompt-fixer.mjs + +providers: + - openai:gpt-4.1-mini + +defaultTest: + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 2 + +tests: tests-fixer.yaml diff --git a/evals/tests-fixer-flow.yaml b/evals/tests-fixer-flow.yaml new file mode 100644 index 0000000..3b38b03 --- /dev/null +++ b/evals/tests-fixer-flow.yaml @@ -0,0 +1,296 @@ +# MDMA Fixer — Flow & References Test Cases +# +# Tests the fixer's ability to fix multi-step flow errors: splitting +# interactive components into separate steps, fixing circular references, +# removing orphans, and complying with the original prompt requirements. +# +# Each test uses the exact broken structure from the Flow & References +# validator prompt, matching the concrete example in the fixer prompt. + +# --------------------------------------------------------------------------- +# 1. Exact broken registration workflow — no history (step 1) +# --------------------------------------------------------------------------- +- description: Fixes exact broken registration workflow to step 1 only + vars: + variantKey: flow + promptContext: | + User registration and approval workflow. + Each step should be in a separate conversation turn. + brokenDocument: | + # User Registration + + ```mdma + type: form + id: registration-form + fields: + - name: full-name + type: text + label: Full Name + required: true + - name: email + type: email + label: Email Address + required: true + sensitive: true + - name: department + type: select + label: Department + options: + - label: Engineering + value: engineering + - label: Marketing + value: marketing + - label: Sales + value: sales + onSubmit: approval-gate + ``` + + ```mdma + type: approval-gate + id: approval-gate + title: Manager Approval + requiredApprovers: 1 + onApprove: registration-form + onDeny: nonexistent-rejection + ``` + + ```mdma + type: button + id: notify-btn + text: Send Notification + onAction: approval-gate + ``` + + ```mdma + type: callout + id: orphan-notice + variant: info + content: This notice is not referenced by anything + ``` + + ```mdma + type: callout + id: orphan-table-info + variant: warning + content: Another orphan + ``` + + ```mdma + type: webhook + id: notify-webhook + url: https://api.example.com/notify + method: POST + trigger: missing-component + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/no-multi-step-flow.mjs + - type: icontains + value: registration-form + - type: not-icontains + value: "type: approval-gate" + - type: not-icontains + value: notify-btn + +# --------------------------------------------------------------------------- +# 2. Same broken structure — step 1 done, show step 2 +# --------------------------------------------------------------------------- +- description: Fixes to step 2 (approval-gate) when form was in prior message + vars: + variantKey: flow + promptContext: | + User registration and approval workflow. + Each step should be in a separate conversation turn. + conversationHistory: + - role: user + content: Create a user registration workflow + - role: assistant + content: | + # User Registration — Step 1 + + ```mdma + type: form + id: registration-form + fields: + - name: full-name + type: text + label: Full Name + required: true + - name: email + type: email + label: Email Address + required: true + sensitive: true + - name: department + type: select + label: Department + options: + - label: Engineering + value: engineering + - label: Marketing + value: marketing + - label: Sales + value: sales + onSubmit: registration-submitted + ``` + + ```mdma + type: callout + id: registration-submitted + variant: success + content: Registration submitted! Awaiting approval. + ``` + - role: user + content: Continue to the next step + brokenDocument: | + # Approval Step + + ```mdma + type: form + id: registration-form + fields: + - name: full-name + type: text + label: Full Name + required: true + - name: email + type: email + label: Email Address + required: true + sensitive: true + onSubmit: approval-gate + ``` + + ```mdma + type: approval-gate + id: approval-gate + title: Manager Approval + requiredApprovers: 1 + onApprove: registration-form + onDeny: denied-callout + ``` + + ```mdma + type: callout + id: denied-callout + variant: error + content: Registration denied. + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/no-multi-step-flow.mjs + - type: not-icontains + value: "type: form" + +# --------------------------------------------------------------------------- +# 3. Expense form → approval-gate chain +# --------------------------------------------------------------------------- +- description: Strips expense workflow to form step only + vars: + variantKey: flow + promptContext: | + Expense approval workflow. + Steps: expense form → manager review → notification. + One step per message. + brokenDocument: | + # Expense Approval + + ```mdma + type: form + id: expense-form + fields: + - name: amount + type: number + label: Amount + required: true + - name: reason + type: textarea + label: Reason + onSubmit: manager-gate + ``` + + ```mdma + type: approval-gate + id: manager-gate + title: Manager Review + requiredApprovers: 1 + onApprove: approved-callout + ``` + + ```mdma + type: callout + id: approved-callout + variant: success + content: Expense approved! + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/no-multi-step-flow.mjs + - type: icontains + value: expense-form + - type: not-icontains + value: "type: approval-gate" + +# --------------------------------------------------------------------------- +# 4. Feedback form with orphans +# --------------------------------------------------------------------------- +- description: Removes orphans and splits feedback workflow + vars: + variantKey: flow + promptContext: | + Feedback collection workflow. + Step 1: feedback form. Step 2: review. + One step per message. + brokenDocument: | + # Feedback Collection + + ```mdma + type: form + id: feedback-form + fields: + - name: rating + type: number + label: Rating + required: true + - name: comment + type: textarea + label: Comment + onSubmit: review-gate + ``` + + ```mdma + type: approval-gate + id: review-gate + title: Review Feedback + requiredApprovers: 1 + onApprove: thank-you + ``` + + ```mdma + type: callout + id: thank-you + variant: success + content: Thank you for your feedback! + ``` + + ```mdma + type: callout + id: orphan-notice + variant: info + content: This is an orphaned notice nobody references. + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/no-multi-step-flow.mjs + - type: icontains + value: feedback-form + - type: not-icontains + value: "type: approval-gate" diff --git a/evals/tests-fixer.yaml b/evals/tests-fixer.yaml new file mode 100644 index 0000000..e0da719 --- /dev/null +++ b/evals/tests-fixer.yaml @@ -0,0 +1,717 @@ +# MDMA Fixer Prompt — Eval Test Cases +# +# Each test provides a brokenDocument containing intentionally broken MDMA +# blocks. The fixer prompt + validator pipeline sends remaining unfixed +# issues to the LLM. Assertions verify the output is a valid MDMA document. + +# --------------------------------------------------------------------------- +# 1. Missing webhook trigger + invalid onSubmit target +# --------------------------------------------------------------------------- +- description: Fixes missing webhook trigger and broken action references + vars: + brokenDocument: | + # Order Submission + + ```mdma + type: form + id: order-form + fields: + - name: product + type: text + label: Product Name + required: true + - name: quantity + type: number + label: Quantity + onSubmit: nonexistent-handler + ``` + + ```mdma + type: webhook + id: order-webhook + url: https://api.example.com/orders + method: POST + ``` + + ```mdma + type: callout + id: order-status + variant: success + content: Your order has been submitted! + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 3 + - type: icontains + value: trigger + +# --------------------------------------------------------------------------- +# 2. Unknown component type + missing required button text +# --------------------------------------------------------------------------- +- description: Fixes unknown component type and missing button text + vars: + brokenDocument: | + # Dashboard + + ```mdma + type: card + id: stats-card + title: Monthly Stats + value: 42 + ``` + + ```mdma + type: button + id: refresh-btn + variant: primary + onAction: stats-card + ``` + + ```mdma + type: callout + id: dashboard-info + variant: info + content: Welcome to your dashboard + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 2 + +# --------------------------------------------------------------------------- +# 3. Missing webhook trigger + unknown type + missing button text +# --------------------------------------------------------------------------- +- description: Fixes missing webhook trigger with multiple broken components + vars: + brokenDocument: | + # User Profile + + ```mdma + type: form + id: profile-form + fields: + - name: email + type: email + label: Email + required: true + sensitive: true + - name: display-name + type: text + label: Display Name + - name: bio + type: textarea + label: Bio + onSubmit: save-profile + ``` + + ```mdma + type: webhook + id: profile-webhook + url: https://api.example.com/profile + method: POST + ``` + + ```mdma + type: button + id: save-profile + variant: primary + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 3 + - type: icontains + value: trigger + +# --------------------------------------------------------------------------- +# 4. Select fields without options + field name typos +# --------------------------------------------------------------------------- +- description: Fixes select without options and field name typos on approval-gate + vars: + brokenDocument: | + # Leave Request + + ```mdma + type: form + id: leave-form + fields: + - name: leave-type + type: select + label: Leave Type + required: true + - name: start-date + type: date + label: Start Date + required: true + - name: reason + type: textarea + label: Reason + onSubmit: leave-approval + ``` + + ```mdma + type: approval-gate + id: leave-approval + title: Manager Approval + roles: + - manager + - hr + approvers: 2 + onApprove: leave-confirmed + ``` + + ```mdma + type: callout + id: leave-confirmed + variant: success + content: Your leave request has been approved! + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 3 + - type: icontains + value: options + +# --------------------------------------------------------------------------- +# 5. Table data key mismatch + chart axis mismatch +# --------------------------------------------------------------------------- +- description: Fixes table data key mismatch and chart axis errors + vars: + brokenDocument: | + # Sales Report + + ```mdma + type: table + id: sales-table + columns: + - key: product + header: Product + - key: revenue + header: Revenue + - key: units + header: Units Sold + data: + - product_name: Widget A + total_revenue: 50000 + quantity: 120 + - product_name: Widget B + total_revenue: 32000 + quantity: 85 + ``` + + ```mdma + type: chart + id: sales-chart + variant: bar + data: | + Month,Revenue,Costs + Jan,10000,8000 + Feb,12000,9000 + Mar,15000,11000 + xAxis: Date + yAxis: + - Profit + - Expenses + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 2 + +# --------------------------------------------------------------------------- +# 6. Missing sensitive flags + missing required fields +# --------------------------------------------------------------------------- +- description: Fixes missing PII sensitivity and missing required schema fields + vars: + brokenDocument: | + # Patient Registration + + ```mdma + type: form + id: patient-form + fields: + - name: full-name + type: text + label: Full Name + required: true + - name: email + type: email + label: Email Address + - name: phone + type: text + label: Phone Number + - name: ssn + type: text + label: Social Security Number + - name: address + type: textarea + label: Home Address + ``` + + ```mdma + type: table + id: patient-records + columns: + - key: name + header: Patient Name + - key: email + header: Email + - key: phone + header: Phone + - key: dob + header: Date of Birth + data: + - name: Jane Doe + email: jane@example.com + phone: 555-0101 + dob: 1990-01-15 + ``` + + ```mdma + type: button + id: submit-registration + variant: primary + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 3 + - type: javascript + value: file://assertions/has-sensitive.mjs + +# --------------------------------------------------------------------------- +# 7. Circular action references + backward flow +# --------------------------------------------------------------------------- +- description: Fixes circular and backward action references + vars: + brokenDocument: | + # Feedback Loop + + ```mdma + type: form + id: feedback-form + fields: + - name: rating + type: number + label: Rating + required: true + - name: comment + type: textarea + label: Comment + onSubmit: review-gate + ``` + + ```mdma + type: approval-gate + id: review-gate + title: Review Feedback + requiredApprovers: 1 + onApprove: feedback-form + onDeny: rejection-notice + ``` + + ```mdma + type: callout + id: rejection-notice + variant: error + content: Your feedback was not accepted. Please revise. + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 1 + +# --------------------------------------------------------------------------- +# 8. Mixed issues — kitchen sink +# --------------------------------------------------------------------------- +- description: Fixes a document with many different issue types + vars: + brokenDocument: | + # Employee Onboarding + + ```mdma + type: form + id: employee_form + fields: + - name: full_name + type: text + label: TODO + required: true + - name: email + type: email + - name: department + type: select + label: Department + - name: start_date + type: date + label: Start Date + onSubmit: missing-handler + ``` + + ```mdma + type: tasklist + id: onboarding-tasks + items: + - id: task-1 + text: Complete HR paperwork + - id: task-2 + text: Set up workstation + - id: task-3 + text: Meet team lead + onComplete: nonexistent-webhook + ``` + + ```mdma + type: button + id: employee_form + variant: primary + onClick: onboarding-tasks + ``` + + ```mdma + type: webhook + id: notify-hr + url: https://api.example.com/hr + method: POST + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 4 + - type: javascript + value: file://assertions/unique-kebab-ids.mjs + +# --------------------------------------------------------------------------- +# 9. Webhook with broken references + form missing onSubmit target +# --------------------------------------------------------------------------- +- description: Fixes webhook trigger and form onSubmit pointing to missing components + vars: + brokenDocument: | + # Support Ticket + + ```mdma + type: form + id: ticket-form + fields: + - name: subject + type: text + label: Subject + required: true + - name: priority + type: select + label: Priority + options: + - label: Low + value: low + - label: Medium + value: medium + - label: High + value: high + - name: description + type: textarea + label: Description + onSubmit: submit-ticket + ``` + + ```mdma + type: webhook + id: ticket-webhook + url: https://api.example.com/tickets + method: POST + body: + subject: "{{ticket-form.subject}}" + priority: "{{ticket-form.priority}}" + ``` + + ```mdma + type: callout + id: ticket-success + variant: success + content: Ticket submitted successfully! + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 3 + - type: icontains + value: trigger + +# --------------------------------------------------------------------------- +# 10. Placeholder content throughout +# --------------------------------------------------------------------------- +- description: Fixes placeholder content in labels and fields + vars: + brokenDocument: | + # Project Setup + + ```mdma + type: form + id: project-form + fields: + - name: project-name + type: text + label: "TODO: add label" + required: true + - name: description + type: textarea + label: "..." + - name: team-size + type: number + label: FIXME + onSubmit: project-summary + ``` + + ```mdma + type: callout + id: project-summary + variant: info + title: TBD + content: Lorem ipsum dolor sit amet + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 2 + - type: javascript + value: file://assertions/no-placeholder-content.mjs + +# --------------------------------------------------------------------------- +# 11. Multi-step flow — no conversation history (first turn) +# --------------------------------------------------------------------------- +- description: Splits multi-step flow to first step only (no prior context) + vars: + brokenDocument: | + # Employee Onboarding + + ```mdma + type: form + id: personal-info + fields: + - name: full-name + type: text + label: Full Name + required: true + - name: email + type: email + label: Email + required: true + sensitive: true + onSubmit: department-form + ``` + + ```mdma + type: form + id: department-form + fields: + - name: department + type: select + label: Department + options: + - label: Engineering + value: engineering + - label: Marketing + value: marketing + - name: start-date + type: date + label: Start Date + onSubmit: onboarding-tasks + ``` + + ```mdma + type: tasklist + id: onboarding-tasks + items: + - id: task-1 + text: Complete HR paperwork + - id: task-2 + text: Setup workstation + onComplete: welcome-callout + ``` + + ```mdma + type: callout + id: welcome-callout + variant: success + content: Welcome aboard! + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/no-multi-step-flow.mjs + - type: icontains + value: personal-info + +# --------------------------------------------------------------------------- +# 12. Multi-step flow — user already completed step 1 +# --------------------------------------------------------------------------- +- description: Splits multi-step flow to step 2 when step 1 was in prior message + vars: + conversationHistory: + - role: user + content: Start the employee onboarding process + - role: assistant + content: | + # Employee Onboarding — Step 1 + + ```mdma + type: form + id: personal-info + fields: + - name: full-name + type: text + label: Full Name + required: true + - name: email + type: email + label: Email + required: true + sensitive: true + onSubmit: step-1-complete + ``` + + ```mdma + type: callout + id: step-1-complete + variant: success + content: Personal info saved! + ``` + - role: user + content: I've submitted the form. What's next? + brokenDocument: | + # Employee Onboarding — Step 2 + + ```mdma + type: form + id: department-form + fields: + - name: department + type: select + label: Department + options: + - label: Engineering + value: engineering + - label: Marketing + value: marketing + - name: start-date + type: date + label: Start Date + onSubmit: onboarding-tasks + ``` + + ```mdma + type: tasklist + id: onboarding-tasks + items: + - id: task-1 + text: Complete HR paperwork + - id: task-2 + text: Setup workstation + onComplete: welcome-callout + ``` + + ```mdma + type: callout + id: welcome-callout + variant: success + content: Welcome aboard! + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 1 + - type: javascript + value: file://assertions/no-multi-step-flow.mjs + - type: icontains + value: department-form + +# --------------------------------------------------------------------------- +# 13. Multi-step flow — 3-step approval pipeline in one message +# --------------------------------------------------------------------------- +- description: Splits 3-step approval pipeline to first step only + vars: + brokenDocument: | + # Expense Approval + + ```mdma + type: form + id: expense-form + fields: + - name: amount + type: number + label: Amount + required: true + - name: description + type: textarea + label: Description + onSubmit: manager-approval + ``` + + ```mdma + type: approval-gate + id: manager-approval + title: Manager Approval + requiredApprovers: 1 + allowedRoles: + - manager + onApprove: finance-approval + ``` + + ```mdma + type: approval-gate + id: finance-approval + title: Finance Approval + requiredApprovers: 1 + allowedRoles: + - finance + onApprove: approved-callout + ``` + + ```mdma + type: callout + id: approved-callout + variant: success + content: Expense approved! + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/no-multi-step-flow.mjs + - type: icontains + value: expense-form diff --git a/packages/prompt-pack/src/index.ts b/packages/prompt-pack/src/index.ts index ba693ee..af0c4aa 100644 --- a/packages/prompt-pack/src/index.ts +++ b/packages/prompt-pack/src/index.ts @@ -1,4 +1,20 @@ export { loadPrompt, listPrompts } from './loader.js'; export { MDMA_AUTHOR_PROMPT } from './prompts/mdma-author.js'; export { MDMA_REVIEWER_PROMPT } from './prompts/mdma-reviewer.js'; +export { + MDMA_FIXER_PROMPT, + MDMA_FIXER_BASE, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_PII, + MDMA_FIXER_FORMS, + MDMA_FIXER_TABLES_CHARTS, + MDMA_FIXER_FLOW, + MDMA_FIXER_APPROVAL, + FIXER_EXTENSIONS, + buildFixerPrompt, + buildFixerMessage, + type FixerIssue, + type FixerMessageOptions, +} from './prompts/mdma-fixer.js'; export { buildSystemPrompt, type BuildSystemPromptOptions } from './build-system-prompt.js'; diff --git a/packages/prompt-pack/src/loader.ts b/packages/prompt-pack/src/loader.ts index 9b76f0d..4c5b151 100644 --- a/packages/prompt-pack/src/loader.ts +++ b/packages/prompt-pack/src/loader.ts @@ -1,5 +1,6 @@ import { MDMA_AUTHOR_PROMPT } from './prompts/mdma-author.js'; import { MDMA_REVIEWER_PROMPT } from './prompts/mdma-reviewer.js'; +import { MDMA_FIXER_PROMPT } from './prompts/mdma-fixer.js'; /** * Static registry of all available prompts. @@ -10,6 +11,7 @@ import { MDMA_REVIEWER_PROMPT } from './prompts/mdma-reviewer.js'; const PROMPTS: Record = { 'mdma-author': MDMA_AUTHOR_PROMPT, 'mdma-reviewer': MDMA_REVIEWER_PROMPT, + 'mdma-fixer': MDMA_FIXER_PROMPT, }; /** diff --git a/packages/prompt-pack/src/prompts/mdma-fixer.ts b/packages/prompt-pack/src/prompts/mdma-fixer.ts new file mode 100644 index 0000000..58364ad --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer.ts @@ -0,0 +1,249 @@ +/** + * Base fixer prompt — general rules that apply to all fix scenarios. + */ +export const MDMA_FIXER_BASE = `You are an MDMA document fixer. You receive a Markdown document containing \`\`\`mdma component blocks along with a list of validation errors that could NOT be auto-fixed. Your job is to output a corrected version of the entire document that resolves every listed issue. + +## Rules + +1. **Fix every listed issue.** Each error includes a rule ID, component ID, field, and description. Address them all. +2. **Preserve everything else.** Do not change parts of the document that are not related to the reported errors. Keep all headings, paragraphs, and working components exactly as they are. +3. **Output the full document.** Return the complete corrected Markdown — not just the changed blocks. The output must be a valid MDMA document ready to render. +4. **Follow MDMA conventions:** + - IDs must be unique and kebab-case + - PII fields must have \`sensitive: true\` + - Bindings use \`{{component-id.field}}\` syntax + - Select fields must have \`options\` defined + - Action targets (\`onSubmit\`, \`onAction\`, \`trigger\`, etc.) must reference existing component IDs + - Every \`\`\`mdma block contains exactly one component in YAML +5. **Do NOT wrap your response in an outer code fence.** Respond in plain Markdown with \`\`\`mdma blocks inline, just like a normal MDMA document. +6. **Do NOT add explanations or commentary.** Output only the fixed document. +7. **Do NOT introduce new errors.** Every component you output must be valid. Use real URLs (e.g. \`https://api.example.com/endpoint\`), real labels, and real content. Never output placeholder or dummy values. +8. **Replace ALL placeholder text.** If any field contains "TODO", "TBD", "FIXME", "...", "Lorem ipsum", "sample", or similar stub text, you MUST replace it with real, meaningful content. This is mandatory — do not keep any placeholder text in your output. + +## Prompt Compliance + +When **Original Prompt Requirements** are provided, you MUST ensure the fixed document complies with them: +- Use the exact component IDs specified in the prompt +- Include the exact field names, types, and labels the prompt requires +- Use the correct select options, approval roles, webhook URLs, etc. +- If the original document used wrong names/IDs that differ from the prompt, fix them to match the prompt +- The prompt requirements take precedence over whatever the original document contained`; + +/** + * Extension: Structure & YAML fixes. + */ +export const MDMA_FIXER_STRUCTURE = ` +## Structure & YAML Fixes + +| Error | How to fix | +|-------|-----------| +| \`Duplicate ID\` | Rename one of the duplicates to a unique kebab-case ID | +| \`ID is not kebab-case\` | Convert to kebab-case: \`myForm\` → \`my-form\`, \`user_table\` → \`user-table\` | +| \`Unknown component type\` | Change to a valid type: form, button, table, callout, tasklist, approval-gate, webhook, chart, thinking | +| \`text: Required\` | Add a \`text\` field with a human-readable button label | +| \`content: Required\` | Add a \`content\` field with meaningful text | +| \`Missing table headers\` | Add \`header\` to each column, derived from \`key\` (e.g. \`first_name\` → \`First Name\`) | +| \`Missing form labels\` | Add \`label\` to each field, derived from \`name\` |`; + +/** + * Extension: Binding & reference fixes. + */ +export const MDMA_FIXER_BINDINGS = ` +## Binding & Reference Fixes + +| Error | How to fix | +|-------|-----------| +| \`Binding must be wrapped in {{ }}\` | Wrap the bare path in double braces AND quote it: \`bind: "{{form.field}}"\`. This applies to ANY field that accepts bindings: \`bind\`, \`disabled\`, \`visible\`, \`data\`. ALWAYS use the format \`"{{path}}"\` with double braces and quotes. | +| \`Empty binding expression\` | The value is \`{{ }}\` or \`{{}}\` which is meaningless. Replace it with a valid binding path like \`"{{component.field}}"\` or remove the \`bind\` property entirely. | +| \`Cross-reference does not match any component ID\` | Fix the target to reference an existing component ID in the document | +| \`component not found in document\` | The binding references a non-existent component. Fix the component ID in the binding path. | +| \`form has no field named\` | The binding references a field that doesn't exist on the form. Fix the field name to match an actual field. |`; + +/** + * Extension: PII & sensitive data fixes. + */ +export const MDMA_FIXER_PII = ` +## PII & Sensitive Data Fixes + +Fields containing PII (email, phone, SSN, address, card numbers, DOB, medical data) MUST have \`sensitive: true\`. + +Check both: +- Form fields: add \`sensitive: true\` to the field object +- Table columns: add \`sensitive: true\` to the column object + +Also check for fields that should be \`required: true\` — names, emails, titles are typically required.`; + +/** + * Extension: Form-specific fixes. + */ +export const MDMA_FIXER_FORMS = ` +## Form-Specific Fixes + +| Error | How to fix | +|-------|-----------| +| \`Missing options on select field\` | Add an \`options\` array with \`{label, value}\` objects. Generate realistic options for the field context. | +| \`field is likely a typo\` | Rename the field to the suggested correct name (e.g. \`onClick\` → \`onAction\`, \`submit\` → \`onSubmit\`) | +| \`placeholder content\` | Replace placeholder text like "TODO", "TBD", "FIXME", "...", or "Lorem ipsum" with real, meaningful content appropriate to the context. NEVER keep placeholder text — always replace it. |`; + +/** + * Extension: Table & chart fixes. + */ +export const MDMA_FIXER_TABLES_CHARTS = ` +## Table & Chart Fixes + +| Error | How to fix | +|-------|-----------| +| \`Data key does not match any column\` | Rename the data keys to match defined column keys, or add missing columns | +| \`Column has no matching keys in any data row\` | Either add matching data or remove the unused column | +| \`xAxis does not match any CSV header\` | Fix xAxis to reference an actual CSV column header | +| \`yAxis does not match any CSV header\` | Fix yAxis values to reference actual CSV column headers | +| \`Chart data does not appear to be valid CSV\` | Ensure CSV has a header row and at least one data row |`; + +/** + * Extension: Flow & reference fixes (multi-step splitting). + */ +export const MDMA_FIXER_FLOW = ` +## Multi-Step Flow Fix + +When you see **"Multi-step flow in single message"** or **"Multiple interactive component types in single message"** errors, the document has multiple workflow stages crammed into one message. This is the most important fix to get right. + +**Interactive component types:** form, button (that targets another interactive component), tasklist, approval-gate. + +**The rule:** A single message must contain AT MOST ONE interactive component type (form OR approval-gate OR tasklist — never multiple). A form + a submit button that targets a callout is OK. A form + an approval-gate is NEVER OK. + +**How to fix — step by step:** +1. **Identify the current step.** Check conversation history. If no history, the current step is the FIRST interactive component. +2. **Keep ONLY the current step's interactive component** (e.g., just the form, or just the approval-gate). +3. **Keep non-interactive supporting components** that belong to this step: callouts and webhooks that are directly referenced by the kept interactive component. +4. **DELETE every other interactive component** (forms, approval-gates, tasklists, buttons that chain to other interactive components). They belong to future conversation turns. +5. **Fix dangling references:** If the kept component's \`onSubmit\`/\`onAction\` pointed to a removed component, change it to point to a callout in the same message instead. +6. **Remove orphaned components** that are no longer referenced by anything after the deletions. + +### Determining the current step + +Use the **Conversation Context** and **Original Prompt Requirements** (if provided) to figure out which step to output: + +| Situation | What to output | +|-----------|---------------| +| No conversation history | The FIRST step of the workflow (usually a form) | +| Prior messages contain step 1 components | The NEXT step (e.g. approval-gate if step 1 was a form) | +| Prior messages contain steps 1 and 2 | Step 3, and so on | +| A component ID from a prior message appears in the broken document | That component was already shown — skip it, output the next step | +| The error says "was already shown in a previous message" | Remove that component and output the next unshown step | + +**Key principle:** Read the prompt requirements to understand the full workflow sequence, then output ONLY the step that hasn't been shown yet. Each step = one interactive component + its supporting callouts/webhooks. + +### How to split + +1. **Map the workflow stages** from the broken document: identify which interactive components represent which step (e.g. form = step 1, approval-gate = step 2, notification button = step 3) +2. **Determine the current step number** from conversation history (count how many interactive steps were already shown) +3. **Keep only the current step's interactive component** and its directly-referenced non-interactive components (callouts, webhooks) +4. **Remove everything else** — other interactive components, orphaned components, components from past or future steps +5. **Fix dangling references:** If the kept component's \`onSubmit\`/\`onAction\` pointed to a removed component, redirect it to a callout in the same message +6. **The output must have exactly 1 interactive component** (plus supporting callouts/webhooks) + +### STRICT: No extra components + +Your output MUST contain ONLY the components for the current step. Do NOT add components that are not defined in the prompt requirements for this step. Do NOT carry over orphaned components from the broken input. Do NOT invent new components that weren't requested. + +If the prompt requirements say Step 2 is "approval-gate + callout", output exactly those two \`\`\`mdma blocks — nothing else. No extra callouts, no tables, no buttons unless explicitly specified for this step.`; + +/** + * Extension: Approval & webhook fixes. + */ +export const MDMA_FIXER_APPROVAL = ` +## Approval & Webhook Fixes + +| Error | How to fix | +|-------|-----------| +| \`field is likely a typo\` on approval-gate | \`roles\` → \`allowedRoles\`, \`approvers\` → \`requiredApprovers\` | +| \`trigger: Required\` | Add a \`trigger\` field pointing to the component ID that should activate this webhook | +| \`Cross-reference in trigger does not match\` | Fix the trigger to reference an existing component ID | +| Missing \`title\` on approval-gate | Add a descriptive title | +| Missing \`url\` on webhook | Add a valid URL (e.g. \`https://api.example.com/endpoint\`) |`; + +/** + * Map from validator variant keys to their fixer extensions. + */ +export const FIXER_EXTENSIONS: Record = { + all: [MDMA_FIXER_STRUCTURE, MDMA_FIXER_BINDINGS, MDMA_FIXER_PII, MDMA_FIXER_FORMS, MDMA_FIXER_TABLES_CHARTS, MDMA_FIXER_FLOW, MDMA_FIXER_APPROVAL], + structure: [MDMA_FIXER_STRUCTURE], + bindings: [MDMA_FIXER_BINDINGS], + pii: [MDMA_FIXER_PII], + forms: [MDMA_FIXER_FORMS], + 'tables-charts': [MDMA_FIXER_TABLES_CHARTS], + flow: [MDMA_FIXER_FLOW], + approval: [MDMA_FIXER_APPROVAL, MDMA_FIXER_STRUCTURE], +}; + +/** + * Build a complete fixer system prompt for a given variant. + * Combines the base prompt with only the relevant extensions. + */ +export function buildFixerPrompt(variantKey?: string): string { + const extensions = variantKey && FIXER_EXTENSIONS[variantKey] + ? FIXER_EXTENSIONS[variantKey] + : [MDMA_FIXER_STRUCTURE, MDMA_FIXER_BINDINGS, MDMA_FIXER_PII, MDMA_FIXER_FORMS, MDMA_FIXER_TABLES_CHARTS, MDMA_FIXER_FLOW, MDMA_FIXER_APPROVAL]; + + return `${MDMA_FIXER_BASE}\n${extensions.join('\n')}`; +} + +/** @deprecated Use buildFixerPrompt() instead. Kept for backward compatibility. */ +export const MDMA_FIXER_PROMPT = buildFixerPrompt(); + +export interface FixerIssue { + ruleId: string; + severity: string; + message: string; + componentId: string | null; + field?: string; +} + +export interface FixerMessageOptions { + /** Previous conversation messages for context (to determine the current step). */ + conversationHistory?: Array<{ role: 'user' | 'assistant'; content: string }>; + /** The original system/custom prompt that describes what components should be generated. */ + promptContext?: string; +} + +/** + * Build a user message that asks the LLM to fix a broken MDMA document. + */ +export function buildFixerMessage( + markdown: string, + issues: FixerIssue[], + options?: FixerMessageOptions, +): string { + const issueLines = issues.map((issue, i) => { + const component = issue.componentId ? `#${issue.componentId}` : '(document)'; + const field = issue.field ? ` → ${issue.field}` : ''; + return `${i + 1}. [${issue.severity}] ${issue.ruleId} ${component}${field}: ${issue.message}`; + }); + + let context = ''; + if (options?.conversationHistory?.length) { + const summary = options.conversationHistory + .map((m) => { + const prefix = m.role === 'user' ? 'User' : 'Assistant'; + const short = m.content.length > 200 + ? `${m.content.slice(0, 200)}...` + : m.content; + return `${prefix}: ${short}`; + }) + .join('\n\n'); + + context += `\n\n## Conversation Context\n\nThe following conversation preceded this message. Use it to determine which step the user is on:\n\n${summary}\n`; + } + + if (options?.promptContext) { + context += `\n\n## Original Prompt Requirements\n\nThe document was generated from the following instructions. The fixed output MUST comply with these requirements — use the correct component IDs, field names, types, options, and structure specified here:\n\n${options.promptContext}\n`; + } + + return `Fix the following MDMA document. The validator found ${issues.length} issue(s) that could not be auto-fixed: + +${issueLines.join('\n')}${context} + +--- + +${markdown}`; +} diff --git a/packages/prompt-pack/tests/loader.test.ts b/packages/prompt-pack/tests/loader.test.ts index 7e0eea2..9b4b7f8 100644 --- a/packages/prompt-pack/tests/loader.test.ts +++ b/packages/prompt-pack/tests/loader.test.ts @@ -36,8 +36,9 @@ describe('listPrompts', () => { expect(names).toContain('mdma-reviewer'); }); - it('returns exactly 2 prompts', () => { + it('returns exactly 3 prompts', () => { const names = listPrompts(); - expect(names).toHaveLength(2); + expect(names).toHaveLength(3); + expect(names).toContain('mdma-fixer'); }); }); diff --git a/packages/validator/src/fixes/schema-defaults.ts b/packages/validator/src/fixes/schema-defaults.ts index c68c5da..a267ada 100644 --- a/packages/validator/src/fixes/schema-defaults.ts +++ b/packages/validator/src/fixes/schema-defaults.ts @@ -112,8 +112,12 @@ function patchFormFields(data: Record): void { f.label = keyToHeader(f.name); } - // Wrap bare bind values in {{ }} - wrapBareBinding(f, 'bind'); + // Wrap bare bind values in {{ }}, or remove empty binds + if (typeof f.bind === 'string' && f.bind.trim() === '') { + f.bind = undefined; + } else { + wrapBareBinding(f, 'bind'); + } } } diff --git a/packages/validator/src/index.ts b/packages/validator/src/index.ts index 6a9ef61..9d6e2f3 100644 --- a/packages/validator/src/index.ts +++ b/packages/validator/src/index.ts @@ -1,4 +1,5 @@ export { validate } from './validate.js'; +export { validateFlow } from './validate-flow.js'; export type { ValidationResult, ValidationIssue, @@ -7,3 +8,9 @@ export type { ValidatorOptions, ValidationRule, } from './types.js'; +export type { + FlowStepDefinition, + FlowValidationOptions, + FlowValidationResult, + FlowValidationIssue, +} from './validate-flow.js'; diff --git a/packages/validator/src/rules/flow-ordering.ts b/packages/validator/src/rules/flow-ordering.ts index 8115a89..efed1db 100644 --- a/packages/validator/src/rules/flow-ordering.ts +++ b/packages/validator/src/rules/flow-ordering.ts @@ -1,15 +1,18 @@ import type { ValidationRule } from '../types.js'; import { ACTION_REFERENCE_FIELDS } from '../constants.js'; +/** Component types that produce user interactions triggering the next step. */ +const INTERACTIVE_TYPES = new Set(['form', 'button', 'tasklist', 'approval-gate']); + export const flowOrderingRule: ValidationRule = { id: 'flow-ordering', name: 'Flow Ordering', description: - 'Checks that action targets reference components defined later in the document and detects circular references', - defaultSeverity: 'info', + 'Checks that action targets reference components defined later in the document, detects circular references, and flags multi-step flows that should be split across messages', + defaultSeverity: 'warning', validate(context) { - // Build adjacency list for cycle detection + // Build adjacency list for cycle and chain detection const graph = new Map(); for (const block of context.blocks) { @@ -45,7 +48,7 @@ export const flowOrderingRule: ValidationRule = { }); } - // Build graph for cycle detection + // Build graph for cycle and chain detection if (sourceId) { if (!graph.has(sourceId)) graph.set(sourceId, []); graph.get(sourceId)!.push(targetId); @@ -98,5 +101,117 @@ export const flowOrderingRule: ValidationRule = { } } } + + // Detect regenerated components from prior conversation turns. + // If the caller provides priorComponentIds, any component in the current + // message that reuses one of those IDs is a sign the LLM repeated a + // previous step instead of advancing to the next one. + const priorIds = context.options.priorComponentIds; + if (priorIds && priorIds.length > 0) { + const priorSet = new Set(priorIds); + for (const block of context.blocks) { + if (block.data === null) continue; + const id = block.data.id; + if (typeof id !== 'string') continue; + const type = block.data.type; + if (typeof type !== 'string') continue; + // Only flag interactive components — callouts/webhooks may legitimately repeat + if (!INTERACTIVE_TYPES.has(type)) continue; + + if (priorSet.has(id)) { + context.issues.push({ + ruleId: 'flow-ordering', + severity: 'error', + message: `Component "${id}" (${type}) was already shown in a previous message — the LLM should generate the next step, not repeat a prior one`, + componentId: id, + blockIndex: block.index, + fixed: false, + }); + } + } + } + + // Detect multi-step flows in a single document. + // + // Check 1: If an interactive component targets another interactive + // component via action fields, it's an explicit multi-step chain. + // + // Check 2: If a document contains multiple interactive components of + // different types (e.g. form + approval-gate), they represent different + // workflow stages and should be in separate messages — even without + // explicit action chains between them. Exception: form + button is OK + // since buttons often accompany forms in the same step. + const interactiveBlocks: Array<{ id: string; type: string; index: number }> = []; + + for (const block of context.blocks) { + if (block.data === null) continue; + const type = block.data.type; + if (typeof type !== 'string') continue; + if (!INTERACTIVE_TYPES.has(type)) continue; + + const sourceId = + typeof block.data.id === 'string' ? block.data.id : null; + + interactiveBlocks.push({ + id: sourceId ?? `block-${block.index}`, + type, + index: block.index, + }); + + // Check 1: explicit action chain to another interactive component + const fields = ACTION_REFERENCE_FIELDS[type]; + if (!fields) continue; + + for (const field of fields) { + const targetId = block.data[field]; + if (typeof targetId !== 'string') continue; + + const targetIndex = context.idMap.get(targetId); + if (targetIndex === undefined) continue; + + const targetBlock = context.blocks[targetIndex]; + if (!targetBlock?.data) continue; + + const targetType = targetBlock.data.type; + if (typeof targetType !== 'string') continue; + + // form ↔ button is OK (button accompanies form in the same step) + const isFormButtonPair = + (type === 'button' && targetType === 'form') || + (type === 'form' && targetType === 'button'); + if (INTERACTIVE_TYPES.has(targetType) && !isFormButtonPair) { + context.issues.push({ + ruleId: 'flow-ordering', + severity: 'error', + message: `Multi-step flow in single message: "${sourceId}" (${type}) targets "${targetId}" (${targetType}) via ${field} — each step should be a separate conversation turn`, + componentId: sourceId, + field, + blockIndex: block.index, + fixed: false, + }); + } + } + } + + // Check 2: multiple interactive types in one document + // form + button in the same message is fine (button accompanies form). + // But form + approval-gate, form + tasklist, etc. = different stages. + if (interactiveBlocks.length > 1) { + const types = new Set(interactiveBlocks.map((b) => b.type)); + const isJustFormAndButton = + types.size <= 2 && types.has('form') && types.has('button'); + + if (!isJustFormAndButton && types.size > 1) { + const typeList = [...types].join(', '); + context.issues.push({ + ruleId: 'flow-ordering', + severity: 'error', + message: `Multiple interactive component types in single message (${typeList}) — each workflow stage should be a separate conversation turn`, + componentId: null, + blockIndex: interactiveBlocks[0].index, + fixed: false, + }); + } + } }, }; diff --git a/packages/validator/src/types.ts b/packages/validator/src/types.ts index cc5f1c6..7c24f3e 100644 --- a/packages/validator/src/types.ts +++ b/packages/validator/src/types.ts @@ -105,6 +105,13 @@ export interface ValidatorOptions { customPiiPatterns?: RegExp[]; /** Custom component Zod schemas for types not in the built-in registry. */ customSchemas?: Record; + /** + * Component IDs from previous conversation messages. + * When set, the flow-ordering rule will flag any component in the current + * message that reuses an ID from a prior turn — indicating the LLM + * regenerated a previous step instead of advancing to the next one. + */ + priorComponentIds?: string[]; } export interface ValidationResult { diff --git a/packages/validator/src/validate-flow.ts b/packages/validator/src/validate-flow.ts new file mode 100644 index 0000000..63bf1ab --- /dev/null +++ b/packages/validator/src/validate-flow.ts @@ -0,0 +1,157 @@ +import { extractMdmaBlocksFromMarkdown } from './extract-blocks.js'; + +/** + * A single step definition in the expected flow. + */ +export interface FlowStepDefinition { + /** Human-readable step label (e.g. "Registration Form") */ + label: string; + /** The primary component type for this step */ + type: 'form' | 'button' | 'tasklist' | 'approval-gate' | 'webhook' | 'callout' | 'table' | 'chart'; + /** Expected component ID for the interactive component */ + id: string; +} + +export interface FlowValidationOptions { + /** Ordered list of expected flow steps. */ + steps: FlowStepDefinition[]; +} + +export interface FlowValidationResult { + /** true if no errors */ + ok: boolean; + /** All issues found across the conversation */ + issues: FlowValidationIssue[]; +} + +export interface FlowValidationIssue { + /** 0-based message index in the conversation */ + messageIndex: number; + severity: 'error' | 'warning' | 'info'; + message: string; +} + +/** + * Extract primary components from a markdown message. + * Returns all components whose type or ID matches an expected step. + */ +function extractStepComponents( + markdown: string, + expectedIds: Set, + expectedTypes: Set, +): Array<{ id: string; type: string }> { + const blocks = extractMdmaBlocksFromMarkdown(markdown); + const result: Array<{ id: string; type: string }> = []; + for (const block of blocks) { + if (!block.data) continue; + const type = block.data.type; + const id = block.data.id; + if (typeof type === 'string' && typeof id === 'string') { + if (expectedIds.has(id) || expectedTypes.has(type)) { + result.push({ id, type }); + } + } + } + return result; +} + +/** + * Validate an entire conversation flow against expected step definitions. + * + * Takes all assistant messages in order and checks: + * 1. Each message contains exactly one interactive component + * 2. Steps follow the expected order + * 3. No step is duplicated + * 4. Component IDs match the expected definitions + * + * @param assistantMessages - Assistant message contents in conversation order + * @param options - Expected flow definition + */ +export function validateFlow( + assistantMessages: string[], + options: FlowValidationOptions, +): FlowValidationResult { + const { steps } = options; + const issues: FlowValidationIssue[] = []; + const seenIds = new Set(); + let currentStepIndex = 0; + + const expectedIds = new Set(steps.map((s) => s.id)); + const expectedTypes = new Set(steps.map((s) => s.type)); + + for (let msgIdx = 0; msgIdx < assistantMessages.length; msgIdx++) { + const components = extractStepComponents(assistantMessages[msgIdx], expectedIds, expectedTypes); + + // Skip messages with no interactive components (e.g. pure text responses) + if (components.length === 0) continue; + + // Check: exactly one interactive component per message + if (components.length > 1) { + issues.push({ + messageIndex: msgIdx, + severity: 'error', + message: `Message ${msgIdx + 1} has ${components.length} interactive components (${components.map((c) => `${c.type}#${c.id}`).join(', ')}) — expected exactly 1`, + }); + } + + for (const comp of components) { + // Check: no duplicates across messages + if (seenIds.has(comp.id)) { + issues.push({ + messageIndex: msgIdx, + severity: 'error', + message: `Component "${comp.id}" (${comp.type}) was already shown in a previous message — duplicate step`, + }); + continue; + } + seenIds.add(comp.id); + + // Check: matches expected step + if (currentStepIndex < steps.length) { + const expected = steps[currentStepIndex]; + + if (comp.id !== expected.id) { + issues.push({ + messageIndex: msgIdx, + severity: 'error', + message: `Expected step ${currentStepIndex + 1} "${expected.label}" with ${expected.type}#${expected.id}, but got ${comp.type}#${comp.id}`, + }); + } else if (comp.type !== expected.type) { + issues.push({ + messageIndex: msgIdx, + severity: 'error', + message: `Step ${currentStepIndex + 1} "${expected.label}" has wrong type: expected ${expected.type}, got ${comp.type}`, + }); + } else { + issues.push({ + messageIndex: msgIdx, + severity: 'info', + message: `Step ${currentStepIndex + 1} "${expected.label}" — correct (${comp.type}#${comp.id})`, + }); + } + + currentStepIndex++; + } else { + issues.push({ + messageIndex: msgIdx, + severity: 'warning', + message: `Unexpected extra step: ${comp.type}#${comp.id} — all ${steps.length} expected steps already completed`, + }); + } + } + } + + // Check: all steps were shown + if (currentStepIndex < steps.length) { + for (let i = currentStepIndex; i < steps.length; i++) { + issues.push({ + messageIndex: assistantMessages.length - 1, + severity: 'info', + message: `Step ${i + 1} "${steps[i].label}" (${steps[i].type}#${steps[i].id}) not yet shown`, + }); + } + } + + const hasErrors = issues.some((i) => i.severity === 'error'); + return { ok: !hasErrors, issues }; +} diff --git a/packages/validator/tests/rules/flow-ordering.test.ts b/packages/validator/tests/rules/flow-ordering.test.ts index fbcebb4..268a53b 100644 --- a/packages/validator/tests/rules/flow-ordering.test.ts +++ b/packages/validator/tests/rules/flow-ordering.test.ts @@ -15,10 +15,10 @@ function createContext(blocks: ParsedBlock[]): ValidationRuleContext { } describe('flow-ordering rule', () => { - it('passes for forward-only references', () => { + it('passes when interactive component targets a non-interactive component', () => { const ctx = createContext([ - createBlock(0, { type: 'form', id: 'f', fields: [], onSubmit: 'btn' }), - createBlock(1, { type: 'button', id: 'btn', text: 'Go' }), + createBlock(0, { type: 'form', id: 'f', fields: [], onSubmit: 'c' }), + createBlock(1, { type: 'callout', id: 'c', content: 'Done!' }), ]); flowOrderingRule.validate(ctx); expect(ctx.issues).toHaveLength(0); @@ -30,8 +30,6 @@ describe('flow-ordering rule', () => { createBlock(1, { type: 'button', id: 'btn', text: 'Go', onAction: 'wh' }), ]); flowOrderingRule.validate(ctx); - // wh.trigger → btn is a forward ref (btn at index 1 > wh at index 0): OK - // btn.onAction → wh is a backward ref (wh at index 0 < btn at index 1): flagged const backwardIssues = ctx.issues.filter((i) => i.message.includes('backward')); expect(backwardIssues.length).toBeGreaterThan(0); }); @@ -54,12 +52,58 @@ describe('flow-ordering rule', () => { expect(ctx.issues).toHaveLength(0); }); - it('all issues have info severity', () => { + it('flags multi-step flow: interactive → interactive in same document', () => { const ctx = createContext([ - createBlock(0, { type: 'form', id: 'a', fields: [], onSubmit: 'b' }), - createBlock(1, { type: 'form', id: 'b', fields: [], onSubmit: 'a' }), + createBlock(0, { type: 'form', id: 'step-1', fields: [], onSubmit: 'step-2' }), + createBlock(1, { type: 'approval-gate', id: 'step-2', title: 'Approve', onApprove: 'step-3' }), + createBlock(2, { type: 'button', id: 'step-3', text: 'Done' }), + ]); + flowOrderingRule.validate(ctx); + const chainIssues = ctx.issues.filter((i) => i.message.includes('Multi-step')); + // form → approval-gate, approval-gate → button + expect(chainIssues).toHaveLength(2); + expect(chainIssues[0].severity).toBe('error'); + }); + + it('does not flag interactive → non-interactive (callout, webhook)', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'f', fields: [], onSubmit: 'wh' }), + createBlock(1, { type: 'webhook', id: 'wh', url: 'https://api.example.com', trigger: 'f' }), + ]); + flowOrderingRule.validate(ctx); + const chainIssues = ctx.issues.filter((i) => i.message.includes('Multi-step')); + expect(chainIssues).toHaveLength(0); + }); + + it('flags multiple interactive types without action chains (form + approval-gate)', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'f', fields: [] }), + createBlock(1, { type: 'approval-gate', id: 'ag', title: 'Approve' }), + createBlock(2, { type: 'callout', id: 'c', content: 'Done' }), + ]); + flowOrderingRule.validate(ctx); + const typeIssues = ctx.issues.filter((i) => i.message.includes('Multiple interactive')); + expect(typeIssues).toHaveLength(1); + expect(typeIssues[0].severity).toBe('error'); + }); + + it('allows form + button in the same message', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'f', fields: [] }), + createBlock(1, { type: 'button', id: 'btn', text: 'Submit' }), + ]); + flowOrderingRule.validate(ctx); + const typeIssues = ctx.issues.filter((i) => i.message.includes('Multiple interactive')); + expect(typeIssues).toHaveLength(0); + }); + + it('flags form + tasklist as different workflow stages', () => { + const ctx = createContext([ + createBlock(0, { type: 'form', id: 'f', fields: [] }), + createBlock(1, { type: 'tasklist', id: 'tl', items: [{ id: 't1', text: 'Do it' }] }), ]); flowOrderingRule.validate(ctx); - expect(ctx.issues.every((i) => i.severity === 'info')).toBe(true); + const typeIssues = ctx.issues.filter((i) => i.message.includes('Multiple interactive')); + expect(typeIssues).toHaveLength(1); }); }); From 82fc30b8173f0080b15628bab1c65f26c3cd50a1 Mon Sep 17 00:00:00 2001 From: gitsad Date: Wed, 1 Apr 2026 14:12:23 +0200 Subject: [PATCH 4/9] feat: working fixing tables & charts --- demo/src/ValidatorView.tsx | 20 +++- demo/src/validator-prompts.ts | 79 ++++++++++++-- .../prompt-pack/src/prompts/mdma-fixer.ts | 3 +- packages/validator/src/validate.ts | 37 ++++++- packages/validator/tests/validate.test.ts | 103 ++++++++++++++++++ 5 files changed, 224 insertions(+), 18 deletions(-) diff --git a/demo/src/ValidatorView.tsx b/demo/src/ValidatorView.tsx index 40453c3..12f6f6a 100644 --- a/demo/src/ValidatorView.tsx +++ b/demo/src/ValidatorView.tsx @@ -7,7 +7,7 @@ import { validate, validateFlow, type ValidationResult, type ValidationIssue, ty import { buildFixerPrompt, buildFixerMessage, buildSystemPrompt } from '@mobile-reality/mdma-prompt-pack'; import { chatCompletion } from './llm-client.js'; import { customizations } from './custom-components.js'; -import { VALIDATOR_PROMPT_VARIANTS, FIXER_FLOW_RULES, FLOW_STEPS } from './validator-prompts.js'; +import { VALIDATOR_PROMPT_VARIANTS, FIXER_FLOW_RULES, FIXER_CORRECT_STRUCTURE, FLOW_STEPS } from './validator-prompts.js'; function severityClass(severity: string): string { if (severity === 'error') return 'validator-severity--error'; @@ -340,7 +340,7 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { const userMessage = buildFixerMessage(result.output, unfixed, { conversationHistory: history.length > 0 ? history : undefined, - promptContext: FIXER_FLOW_RULES[promptKey] ?? variant.prompt, + promptContext: FIXER_FLOW_RULES[promptKey] ?? FIXER_CORRECT_STRUCTURE[promptKey] ?? undefined, }); const resolvedModel = fixerModel === '__custom__' ? customFixerModel : fixerModel; @@ -376,10 +376,12 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { setFixingMsgId(null); fixAbortRef.current = null; } - }, [validationResults, messages, config, isFixing, updateMessage, variant.prompt, fixerModel, customFixerModel, promptKey]); + }, [validationResults, messages, config, isFixing, updateMessage, fixerModel, customFixerModel, promptKey]); // Auto-fix with LLM when enabled and unfixed issues detected const autoFixTriggeredRef = useRef>(new Set()); + const autoFixQueueRef = useRef(null); + useEffect(() => { if (!autoFixWithLlm || isFixing || isGenerating) return; for (const [msgId, result] of validationResults) { @@ -387,11 +389,19 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { const unfixed = result.issues.filter((i) => !i.fixed && (i.severity === 'error' || i.severity === 'warning')); if (unfixed.length > 0) { autoFixTriggeredRef.current.add(msgId); - setTimeout(() => handleRequestFix(msgId), 300); + autoFixQueueRef.current = msgId; break; } } - }, [validationResults, autoFixWithLlm, isFixing, isGenerating, handleRequestFix]); + }, [validationResults, autoFixWithLlm, isFixing, isGenerating]); + + // Process the queued auto-fix in a separate effect to avoid stale closures + useEffect(() => { + if (autoFixQueueRef.current === null || isFixing || isGenerating) return; + const msgId = autoFixQueueRef.current; + autoFixQueueRef.current = null; + handleRequestFix(msgId); + }, [isFixing, isGenerating, handleRequestFix]); // Flow validation — run against all assistant messages when steps are defined const flowSteps = FLOW_STEPS[promptKey]; diff --git a/demo/src/validator-prompts.ts b/demo/src/validator-prompts.ts index af924fd..c82597b 100644 --- a/demo/src/validator-prompts.ts +++ b/demo/src/validator-prompts.ts @@ -8,7 +8,9 @@ export interface ValidatorPromptVariant { const PREAMBLE = `You are an AI assistant for testing the MDMA validator. Generate MDMA components with intentional issues so the validator can demonstrate its detection and auto-fix capabilities. -Generate real, useful-looking components — just with the specified intentional mistakes baked in.`; +Generate real, useful-looking components — just with the specified intentional mistakes baked in. + +CRITICAL: Every component MUST be wrapped in its own fenced code block using \`\`\`mdma and \`\`\`. Never output bare YAML without fences. Each component = one separate \`\`\`mdma block. Do NOT use --- separators between components — use separate fenced blocks instead.`; export const VALIDATOR_PROMPT_VARIANTS: ValidatorPromptVariant[] = [ { @@ -126,17 +128,29 @@ Generate a multi-step form (like an application form) with multiple select field rules: ['table-data-keys', 'chart-validation'], prompt: `${PREAMBLE} -Focus ONLY on table and chart data issues: +Focus ONLY on table and chart data issues. Generate a sales dashboard with these exact components, each with intentional problems: + +## Required broken components + +1. \`\`\`mdma block: **table** id: \`sales-table\` — Sales summary table + - Columns: product (header: "Product"), revenue (header: "Revenue"), units (header: "Units Sold") + - BUT use wrong data keys: \`product_name\` instead of \`product\`, \`total_revenue\` instead of \`revenue\`, \`quantity\` instead of \`units\` + - Include 3 data rows (Widget A, Widget B, Widget C) + +2. \`\`\`mdma block: **chart** id: \`sales-chart\` variant: bar — Sales bar chart + - CSV data with headers: month,sales,returns + - 4 rows of data (Jan-Apr) + - BUT set xAxis: "date" (doesn't exist, should be "month") and yAxis: ["revenue", "refunds"] (don't exist, should be "sales", "returns") + +3. \`\`\`mdma block: **table** id: \`users-table\` — User analytics table + - Columns: name (header: "Name"), email (header: "Email", sensitive: true), signups (header: "Sign-ups") + - Use data: analytics-form.results (bare binding, missing {{ }}) -1. **Table data key mismatch** — Define table columns as [name, email, role] but include data rows with keys like [full_name, mail, position] that don't match -2. **Missing column data** — Define a column that no data row populates -3. **Extra data keys** — Include data row keys that aren't in the columns -4. **Invalid chart xAxis** — Set xAxis to a header name that doesn't exist in the CSV data -5. **Invalid chart yAxis** — Set yAxis to header names that don't exist in the CSV -6. **Chart with only headers** — Provide CSV data with only a header row and no data rows -7. **Bare table data binding** — Use data: some-component.rows without wrapping in {{ }} +4. \`\`\`mdma block: **chart** id: \`users-chart\` variant: line — User growth line chart + - CSV data with ONLY a header row: week,new_users,active_users (no data rows) + - Set xAxis: "week", yAxis: ["new_users", "active_users"] -Generate a dashboard with 2 tables and 2 charts — a sales summary and a user analytics view — but with mismatched column/data keys and wrong axis references.`, +Generate all 4 components with the intentional mismatches described above.`, }, { key: 'flow', @@ -170,7 +184,17 @@ Generate exactly these components in ONE message (this is intentionally wrong 6. \`\`\`mdma block: **webhook** id: \`notify-webhook\` url: https://api.example.com/notify, method: POST - Set \`trigger: missing-component\` (invalid action target — ID doesn't exist) -Generate all 6 components in a single message. The validator should detect: multi-step flow errors, circular references, orphaned components, invalid action targets, and backward references.`, +Generate all 6 components in a single message. The validator should detect: multi-step flow errors, circular references, orphaned components, invalid action targets, and backward references. + +## Between MDMA generations + +Between generating MDMA steps, the user may: + +1. **"Continue to the next step"** — This means the fixer has already split your broken output into step 1. Now generate ONLY step 2 (the approval-gate step) with intentional issues. Do NOT regenerate step 1. +2. **Ask a normal question** ("What's that?", "How does this work?", etc.) — Respond conversationally in plain text. Do NOT generate any \`\`\`mdma blocks. Just answer their question about the workflow. +3. **Click Submit/Approve/Deny** — The system auto-sends "Continue to the next step". Generate the next step only. + +IMPORTANT: Only generate \`\`\`mdma blocks when explicitly asked to generate a step or on the first message. For all other user messages, respond with plain text only.`, }, { key: 'approval', @@ -207,6 +231,39 @@ export const FIXER_FLOW_RULES: Record = { - **Step 3:** Webhook \`notify-webhook\` (url: https://api.example.com/notify, method: POST) triggered by a button \`send-notification\` (text: "Send Notification"). Include a success callout \`workflow-complete\`. The webhook's trigger should point to the button. Each step must contain exactly ONE interactive component + its supporting callouts/webhooks. Do not include components from other steps.`, + +}; + +/** + * Describes the correct component structure for each variant. + * Used as promptContext for the LLM fixer so it knows what the fixed output + * should look like — not how to break it. + * + * Keyed by variant key. Variants without an entry fall back to no promptContext. + */ +export const FIXER_CORRECT_STRUCTURE: Record = { + 'tables-charts': `This is a sales dashboard with 4 components. The correct structure: + +**1. sales-table** (table) — Sales summary +- Columns: product (header: "Product"), revenue (header: "Revenue"), units (header: "Units Sold") +- Data rows must use keys matching column keys exactly: product, revenue, units +- 3 rows: Widget A ($50,000, 1200), Widget B ($32,000, 800), Widget C ($18,000, 450) + +**2. sales-chart** (chart, variant: bar) — Sales bar chart +- CSV data with headers: month,sales,returns +- 4 rows: Jan (12000,450), Feb (15000,380), Mar (18000,520), Apr (21000,410) +- xAxis: "month", yAxis: ["sales", "returns"] — must match actual CSV headers + +**3. users-table** (table) — User analytics +- Columns: name (header: "Name"), email (header: "Email", sensitive: true), signups (header: "Sign-ups") +- Data must be wrapped in binding syntax: data: "{{analytics-form.results}}" + +**4. users-chart** (chart, variant: line) — User growth +- CSV data with headers: week,new_users,active_users +- Must have at least 4 data rows (Week 1-4) with actual numeric data +- xAxis: "week", yAxis: ["new_users", "active_users"] + +All table data keys must exactly match their column keys. All chart axes must reference actual CSV headers. Charts must have data rows, not just headers.`, }; /** diff --git a/packages/prompt-pack/src/prompts/mdma-fixer.ts b/packages/prompt-pack/src/prompts/mdma-fixer.ts index 58364ad..4782c08 100644 --- a/packages/prompt-pack/src/prompts/mdma-fixer.ts +++ b/packages/prompt-pack/src/prompts/mdma-fixer.ts @@ -83,7 +83,8 @@ export const MDMA_FIXER_FORMS = ` |-------|-----------| | \`Missing options on select field\` | Add an \`options\` array with \`{label, value}\` objects. Generate realistic options for the field context. | | \`field is likely a typo\` | Rename the field to the suggested correct name (e.g. \`onClick\` → \`onAction\`, \`submit\` → \`onSubmit\`) | -| \`placeholder content\` | Replace placeholder text like "TODO", "TBD", "FIXME", "...", or "Lorem ipsum" with real, meaningful content appropriate to the context. NEVER keep placeholder text — always replace it. |`; +| \`placeholder content\` | Replace placeholder text like "TODO", "TBD", "FIXME", "...", or "Lorem ipsum" with real, meaningful content appropriate to the context. NEVER keep placeholder text — always replace it. | +| \`outside of a \\\`\\\`\\\`mdma fenced block\` | The YAML component is missing its fenced code block wrapper. Wrap it in \`\`\`mdma ... \`\`\`. Each component must be in its own separate fenced block. |`; /** * Extension: Table & chart fixes. diff --git a/packages/validator/src/validate.ts b/packages/validator/src/validate.ts index 9eb94c7..a871ec5 100644 --- a/packages/validator/src/validate.ts +++ b/packages/validator/src/validate.ts @@ -10,6 +10,30 @@ import { extractMdmaBlocksFromMarkdown } from './extract-blocks.js'; import { getRulesExcluding } from './rules/index.js'; import { FIX_REGISTRY, FIX_ORDER } from './fixes/index.js'; import { reconstructMarkdown } from './reserialize.js'; +import { COMPONENT_TYPES } from '@mobile-reality/mdma-spec'; + +const KNOWN_TYPES = new Set(COMPONENT_TYPES as readonly string[]); + +/** + * Detect YAML that looks like MDMA components but isn't inside ```mdma fences. + * Looks for `type: ` patterns outside of fenced code blocks. + */ +function detectUnfencedComponents(markdown: string): Array<{ type: string; line: number }> { + // Strip all fenced code blocks (any language) so we only scan prose + const stripped = markdown.replace(/```[\s\S]*?```/g, (match) => + '\n'.repeat(match.split('\n').length - 1), + ); + + const results: Array<{ type: string; line: number }> = []; + const lines = stripped.split('\n'); + for (let i = 0; i < lines.length; i++) { + const match = lines[i].match(/^type:\s*(\S+)/); + if (match && KNOWN_TYPES.has(match[1])) { + results.push({ type: match[1], line: i + 1 }); + } + } + return results; +} function buildIdMap(blocks: ParsedBlock[]): Map { const map = new Map(); @@ -29,6 +53,17 @@ export function validate( ): ValidationResult { const { exclude = [], autoFix = true } = options; + // 0. Detect MDMA-like YAML outside of fenced blocks + const unfenced = detectUnfencedComponents(markdown); + const preIssues = unfenced.map((u) => ({ + ruleId: 'yaml-correctness' as ValidationRuleId, + severity: 'error' as const, + message: `Found "type: ${u.type}" at line ${u.line} outside of a \`\`\`mdma fenced block — wrap it in \`\`\`mdma ... \`\`\``, + componentId: null, + blockIndex: -1, + fixed: false, + })); + // 1. Extract and parse all mdma blocks const blocks = extractMdmaBlocksFromMarkdown(markdown); @@ -39,7 +74,7 @@ export function validate( const context: ValidationRuleContext = { blocks, idMap, - issues: [], + issues: [...preIssues], options, }; const rules = getRulesExcluding(exclude); diff --git a/packages/validator/tests/validate.test.ts b/packages/validator/tests/validate.test.ts index ef12fef..6aedc6a 100644 --- a/packages/validator/tests/validate.test.ts +++ b/packages/validator/tests/validate.test.ts @@ -360,4 +360,107 @@ onSubmit: step-info expect(result.output).toContain('step-info'); expect(result.output).toContain('my-form'); }); + + describe('unfenced MDMA detection', () => { + it('flags MDMA-like YAML outside of fenced blocks', () => { + const md = `# Dashboard + +type: form +id: my-form +fields: + - name: email + type: email + label: Email +`; + const result = validate(md); + expect(result.ok).toBe(false); + const unfenced = result.issues.filter( + (i) => i.ruleId === 'yaml-correctness' && i.message.includes('outside of a'), + ); + expect(unfenced.length).toBe(1); + expect(unfenced[0].message).toContain('type: form'); + }); + + it('detects multiple unfenced components', () => { + const md = ` +type: form +id: my-form +fields: + - name: name + type: text + label: Name + +--- + +type: approval-gate +id: my-gate +title: Approval + +--- + +type: webhook +id: my-hook +url: https://example.com +trigger: my-form +`; + const result = validate(md); + const unfenced = result.issues.filter( + (i) => i.ruleId === 'yaml-correctness' && i.message.includes('outside of a'), + ); + expect(unfenced.length).toBe(3); + expect(unfenced.map((i) => i.message)).toEqual( + expect.arrayContaining([ + expect.stringContaining('type: form'), + expect.stringContaining('type: approval-gate'), + expect.stringContaining('type: webhook'), + ]), + ); + }); + + it('does not flag components inside fenced blocks', () => { + const md = `# Valid + +\`\`\`mdma +type: form +id: my-form +fields: + - name: email + type: email + label: Email +\`\`\` +`; + const result = validate(md); + const unfenced = result.issues.filter( + (i) => i.ruleId === 'yaml-correctness' && i.message.includes('outside of a'), + ); + expect(unfenced.length).toBe(0); + }); + + it('does not flag type fields inside non-mdma code blocks', () => { + const md = `# Example + +\`\`\`yaml +type: form +id: example +\`\`\` +`; + const result = validate(md); + const unfenced = result.issues.filter( + (i) => i.ruleId === 'yaml-correctness' && i.message.includes('outside of a'), + ); + expect(unfenced.length).toBe(0); + }); + + it('does not flag unknown types outside fenced blocks', () => { + const md = ` +type: card +id: some-card +`; + const result = validate(md); + const unfenced = result.issues.filter( + (i) => i.ruleId === 'yaml-correctness' && i.message.includes('outside of a'), + ); + expect(unfenced.length).toBe(0); + }); + }); }); From 1ef20fd28d845b2d43918ca981e873180191269b Mon Sep 17 00:00:00 2001 From: gitsad Date: Wed, 1 Apr 2026 15:17:28 +0200 Subject: [PATCH 5/9] feat: working validation in bindings & references --- demo/src/ValidatorView.tsx | 35 ++- demo/src/custom-components.tsx | 74 ++++- demo/src/styles.css | 105 ++++++- demo/src/validator-prompts.ts | 277 ++++++++++++++++-- .../parser/src/bindings/extract-bindings.ts | 2 +- .../src/components/FormRenderer.tsx | 75 +++-- .../src/components/TableRenderer.tsx | 41 ++- .../src/context/ElementOverridesContext.tsx | 6 + packages/runtime/src/core/binding-resolver.ts | 2 +- packages/runtime/src/core/document-store.ts | 16 + .../validator/src/rules/binding-syntax.ts | 8 +- 11 files changed, 572 insertions(+), 69 deletions(-) diff --git a/demo/src/ValidatorView.tsx b/demo/src/ValidatorView.tsx index 12f6f6a..b0b6610 100644 --- a/demo/src/ValidatorView.tsx +++ b/demo/src/ValidatorView.tsx @@ -3,11 +3,13 @@ import { useChat } from './chat/use-chat.js'; import { ChatSettings } from './chat/ChatSettings.js'; import { ChatMessage } from './chat/ChatMessage.js'; import { ChatInput } from './chat/ChatInput.js'; +import { ChatActionLog } from './chat/ChatActionLog.js'; +import { useChatActionLog } from './chat/use-chat-action-log.js'; import { validate, validateFlow, type ValidationResult, type ValidationIssue, type FlowValidationResult } from '@mobile-reality/mdma-validator'; import { buildFixerPrompt, buildFixerMessage, buildSystemPrompt } from '@mobile-reality/mdma-prompt-pack'; import { chatCompletion } from './llm-client.js'; import { customizations } from './custom-components.js'; -import { VALIDATOR_PROMPT_VARIANTS, FIXER_FLOW_RULES, FIXER_CORRECT_STRUCTURE, FLOW_STEPS } from './validator-prompts.js'; +import { VALIDATOR_PROMPT_VARIANTS, FIXER_FLOW_RULES, FIXER_CORRECT_STRUCTURE, FLOW_STEPS, SAMPLE_BINDING_DATA } from './validator-prompts.js'; function severityClass(severity: string): string { if (severity === 'error') return 'validator-severity--error'; @@ -226,6 +228,23 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { const flowCompleteRef = useRef(false); const [showFlowComplete, setShowFlowComplete] = useState(false); + // Seed stores with sample binding data for the active variant + const seededStores = useRef(new Set()); + useEffect(() => { + const sampleData = SAMPLE_BINDING_DATA[promptKey]; + if (!sampleData) return; + for (const msg of messages) { + if (msg.role === 'assistant' && msg.store && !seededStores.current.has(msg.store)) { + seededStores.current.add(msg.store); + for (const [componentId, fields] of Object.entries(sampleData)) { + for (const [field, value] of Object.entries(fields)) { + msg.store.dispatch({ type: 'FIELD_CHANGED', componentId, field, value }); + } + } + } + } + }, [messages, promptKey]); + useEffect(() => { const ADVANCE_EVENTS = ['ACTION_TRIGGERED', 'APPROVAL_GRANTED', 'APPROVAL_DENIED'] as const; for (const msg of messages) { @@ -299,6 +318,8 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { } }, [messages, isGenerating, updateMessage]); + const { events, isOpen: logOpen, setIsOpen: setLogOpen, clearEvents } = useChatActionLog(messages); + const chatEndRef = useRef(null); const prevMsgCountRef = useRef(messages.length); @@ -311,9 +332,10 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { const handleClear = useCallback(() => { clear(); + clearEvents(); setValidationResults(new Map()); validatedRef.current = new Set(); - }, [clear]); + }, [clear, clearEvents]); const [fixingMsgId, setFixingMsgId] = useState(null); const isFixing = fixingMsgId !== null; @@ -360,8 +382,9 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { if (fixedContent) { // Overwrite the original message with the fixed content updateMessage(msg.id, fixedContent); - // Clear old validation result so it gets re-validated + // Clear old validation and seeded stores so they re-run on the new store validatedRef.current.delete(msg.id); + seededStores.current.clear(); setValidationResults((prev) => { const next = new Map(prev); next.delete(msg.id); @@ -479,6 +502,12 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { />
+ setLogOpen((prev) => !prev)} + /> +

Fixer Settings

diff --git a/demo/src/custom-components.tsx b/demo/src/custom-components.tsx index 582f34d..a6208c6 100644 --- a/demo/src/custom-components.tsx +++ b/demo/src/custom-components.tsx @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { memo } from 'react'; +import { memo, useState } from 'react'; import { ComponentBaseSchema } from '@mobile-reality/mdma-spec'; import { ChartRenderer } from './chart-components.js'; import type { @@ -200,21 +200,36 @@ function DataSourceSelector({ // ─── Custom Form Element Overrides (scoped) ───────────────────────────────── -function GlassInput({ id, type, value, onChange, required, name }: FormInputElementProps) { +function GlassInput({ id, type, value, onChange, required, name, sensitive }: FormInputElementProps) { const edit = useEditableField(); const componentId = extractComponentId(id, name); + const [masked, setMasked] = useState(sensitive === true && value !== ''); + const displayType = masked ? 'password' : type; return ( -
+
onChange(e.target.value)} + onChange={(e) => { + onChange(e.target.value); + if (sensitive && masked) setMasked(false); + }} /> + {sensitive && value && ( + + )} {edit && }
); @@ -264,17 +279,17 @@ function ToggleCheckbox({ id, checked, onChange, label, name }: FormCheckboxElem ); } -function GlassTextarea({ id, value, onChange, required, name }: FormTextareaElementProps) { +function GlassTextarea({ id, value, onChange, required, name, sensitive }: FormTextareaElementProps) { const edit = useEditableField(); const componentId = extractComponentId(id, name); return ( -
+