From b22c8645e44238ac9c3d8f80186af8beed18244d Mon Sep 17 00:00:00 2001 From: WB Solutions Date: Fri, 5 Jun 2026 14:12:24 -0700 Subject: [PATCH] test(process): add contract-test suite for step-file workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tests/process/ — structural/contract tests for the markdown step-file workflows under _ai-memory/pov/workflows/ and the aim-model-dispatch sub-workflows. These workflows have no backing executable, so the suite validates their structural contracts rather than runtime behavior: - frontmatter schema (name/description/firstStep) - step-chain link resolution (firstStep -> nextStepFile; no dangling, cycle-guarded) - WORKFLOW-MAP routing entries resolve, with a cross-check against silent drops - embedded-procedure skills expose a valid SKILL.md - corpus-size sentinel guarding against silently-empty discovery Pure filesystem assertions (stdlib + PyYAML); no src import, no live services. Registered under a `process` pytest marker. INDEX.md documents coverage for all 30 processes (28 tested, 2 documented non-feasible). --- pyproject.toml | 1 + tests/process/INDEX.md | 90 +++++++++++ tests/process/__init__.py | 0 tests/process/conftest.py | 174 +++++++++++++++++++++ tests/process/test_corpus_sentinel.py | 36 +++++ tests/process/test_skill_procedures.py | 69 ++++++++ tests/process/test_step_chain.py | 38 +++++ tests/process/test_step_frontmatter.py | 81 ++++++++++ tests/process/test_workflow_frontmatter.py | 55 +++++++ tests/process/test_workflow_map.py | 73 +++++++++ 10 files changed, 617 insertions(+) create mode 100644 tests/process/INDEX.md create mode 100644 tests/process/__init__.py create mode 100644 tests/process/conftest.py create mode 100644 tests/process/test_corpus_sentinel.py create mode 100644 tests/process/test_skill_procedures.py create mode 100644 tests/process/test_step_chain.py create mode 100644 tests/process/test_step_frontmatter.py create mode 100644 tests/process/test_workflow_frontmatter.py create mode 100644 tests/process/test_workflow_map.py diff --git a/pyproject.toml b/pyproject.toml index 2aa7c9de..615448d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -140,6 +140,7 @@ markers = [ "requires_streamlit: Tests requiring Streamlit dashboard on port 28501", "quarantine: Quarantined flaky tests excluded from default CI runs", "regression: Regression tests requiring live Langfuse + Qdrant (excluded from default pytest run)", + "process: structural/contract tests for markdown workflow files (no live services required)", ] # F-012 companion: pytest installs catch_warnings(record=True) + simplefilter("always"), # which overrides the production filter in src/memory/config.py. Mirror the diff --git a/tests/process/INDEX.md b/tests/process/INDEX.md new file mode 100644 index 00000000..f7ecfc4e --- /dev/null +++ b/tests/process/INDEX.md @@ -0,0 +1,90 @@ +# Process Test Index — TASK-071 Phase 4 + +**Authored**: feat/task071-process-tests +**Work order**: `oversight/specs/TASK-071-PHASE4-PROCESS-TEST-WORK-ORDER.md` +**Harness authority**: `oversight/knowledge/best-practices/BP-017-pytest-contract-testing-markdown-step-file-workflows.md` + +All 30 processes from the Phase-2 Lane-5 inventory are listed below with their +coverage status. Testable processes are covered by the parametrized contract +suite in `tests/process/` — no dedicated per-process test files. + +--- + +## Coverage Notes + +### Cyclic workflows — handled gracefully + +`cycles/review-cycle` intentionally loops: `step-03→step-04→step-05→step-03`. +The exit condition is prose-controlled via `exitStepFile: ./step-07-exit-cycle.md`. +`walk_step_chain()` terminates on revisit (cycle is not an error — all referenced +files resolve). The forward link-resolution contract still holds: every +`nextStepFile` reference in the loop resolves to a real file. + +### Reverse-reachability (orphan) test — NOT implemented + +The corpus contains branch/mode steps that are reached by prose routing in step +bodies rather than the linear `firstStep`→`nextStepFile` spine: + +- `steps-e/` (edit-mode steps) +- `steps-v/` (validate-mode steps) +- `branches/branch-a..d/` (conditional branches) +- `route/step-01-resolve-backend.md` (shared routing step) + +A corpus-wide reverse-reachability check (`all step*.md − linear-reachable == ∅`) +would false-fail on all of these. The forward link-resolution contract +(`test_step_chain.py` + `test_step_frontmatter.py::test_step_nextStepFile_resolves`) +is the false-positive-free equivalent: every *referenced* path resolves; prose-routed +paths that are intentionally unreachable from the linear spine are not asserted. + +### FIRSTEP_EXEMPT set + +Two workflows have no executable step chain and are skipped for `firstStep`-presence +and chain-walk assertions (they still pass name/description/H2 checks): + +| Workflow | Reason | +|---|---| +| `session/status/workflow.md` | Single-step inline workflow — `firstStep: null` by design | +| `model-dispatch/claude-native/workflow.md` | Reference doc — no step chain | + +### aim-best-practices-researcher — core skill root + +`aim-best-practices-researcher` lives under `_ai-memory/skills/` (core skills root), +not `_ai-memory/pov/skills/` (pov skills root). `test_skill_procedures.py` uses +explicit per-skill paths for all three Section-C skills to accommodate both roots. + +--- + +## Process Coverage Table + +| # | Process ID | Root File | Section | Coverage | Test File / Notes | +|---|---|---|---|---|---| +| 1 | cycles/agent-dispatch | `_ai-memory/pov/workflows/cycles/agent-dispatch/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 2 | cycles/approval-gate | `_ai-memory/pov/workflows/cycles/approval-gate/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 3 | cycles/legitimacy-check | `_ai-memory/pov/workflows/cycles/legitimacy-check/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 4 | cycles/research-protocol | `_ai-memory/pov/workflows/cycles/research-protocol/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 5 | cycles/review-cycle | `_ai-memory/pov/workflows/cycles/review-cycle/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 6 | first-breath | `_ai-memory/pov/workflows/first-breath/workflow.md` | A | ✅ Contract suite (structural); behavioral testing non-feasible | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 7 | init/existing | `_ai-memory/pov/workflows/init/existing/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 8 | init/new | `_ai-memory/pov/workflows/init/new/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 9 | phases/architecture | `_ai-memory/pov/workflows/phases/architecture/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 10 | phases/discovery | `_ai-memory/pov/workflows/phases/discovery/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 11 | phases/execution | `_ai-memory/pov/workflows/phases/execution/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 12 | phases/integration | `_ai-memory/pov/workflows/phases/integration/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 13 | phases/maintenance | `_ai-memory/pov/workflows/phases/maintenance/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 14 | phases/planning | `_ai-memory/pov/workflows/phases/planning/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 15 | phases/release | `_ai-memory/pov/workflows/phases/release/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 16 | session/blocker | `_ai-memory/pov/workflows/session/blocker/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 17 | session/close | `_ai-memory/pov/workflows/session/close/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 18 | session/decision | `_ai-memory/pov/workflows/session/decision/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 19 | session/handoff | `_ai-memory/pov/workflows/session/handoff/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 20 | session/start | `_ai-memory/pov/workflows/session/start/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 21 | session/status | `_ai-memory/pov/workflows/session/status/workflow.md` | A | ⚠️ Partial — EXEMPT | name/description/H2 tested; `firstStep`+chain SKIPPED (`firstStep: null` — single-step inline workflow by design) | +| 22 | session/verify | `_ai-memory/pov/workflows/session/verify/workflow.md` | A | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 23 | model-dispatch/api-dispatch | `_ai-memory/pov/skills/aim-model-dispatch/workflows/api-dispatch/workflow.md` | B | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 24 | model-dispatch/bmad-dispatch | `_ai-memory/pov/skills/aim-model-dispatch/workflows/bmad-dispatch/workflow.md` | B | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 25 | model-dispatch/tmux-dispatch | `_ai-memory/pov/skills/aim-model-dispatch/workflows/tmux-dispatch/workflow.md` | B | ✅ Contract suite | `test_workflow_frontmatter.py`, `test_step_frontmatter.py`, `test_step_chain.py` | +| 26 | model-dispatch/claude-native | `_ai-memory/pov/skills/aim-model-dispatch/workflows/claude-native/workflow.md` | B | ⚠️ Partial — EXEMPT | name/description/H2 tested; `firstStep`+chain SKIPPED (reference doc, no step chain — documented non-feasible) | +| 27 | skill/aim-agent-sanctum-init | `_ai-memory/pov/skills/aim-agent-sanctum-init/SKILL.md` | C | ✅ Existing test | `tests/test_install_sanctum_preservation.py` (idempotency) — not duplicated here per work-order §5 | +| 28 | skill/aim-agent-dispatch | `_ai-memory/pov/skills/aim-agent-dispatch/SKILL.md` | C | ✅ Skill procedures | `test_skill_procedures.py` | +| 29 | skill/aim-agent-lifecycle | `_ai-memory/pov/skills/aim-agent-lifecycle/SKILL.md` | C | ✅ Skill procedures | `test_skill_procedures.py` | +| 30 | skill/aim-best-practices-researcher | `_ai-memory/skills/aim-best-practices-researcher/SKILL.md` | C | ✅ Skill procedures | `test_skill_procedures.py` — core skill root (`_ai-memory/skills/`), distinct from pov skills root | diff --git a/tests/process/__init__.py b/tests/process/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/process/conftest.py b/tests/process/conftest.py new file mode 100644 index 00000000..6279ad6a --- /dev/null +++ b/tests/process/conftest.py @@ -0,0 +1,174 @@ +"""Shared constants, helpers, and fixtures for tests/process/ contract tests. + +All tests in this package are pure filesystem operations — no src/ import, +no live services, no LLM calls. Pattern: DAG-integrity style (BP-017). + +Path anchoring: REPO_ROOT is derived from this file's location so it is +CWD-independent and safe in CI (BP-017 §10). +""" + +from pathlib import Path + +import pytest +import yaml + +# --------------------------------------------------------------------------- +# Path constants +# --------------------------------------------------------------------------- + +# tests/process/conftest.py → .parent = tests/process/ +# .parent = tests/ +# .parent = pov-work/ (repo root) +REPO_ROOT = Path(__file__).parent.parent.parent + +WORKFLOWS_ROOT = REPO_ROOT / "_ai-memory/pov/workflows" +MODEL_DISPATCH_ROOT = REPO_ROOT / "_ai-memory/pov/skills/aim-model-dispatch/workflows" +SKILLS_ROOT = ( + REPO_ROOT / "_ai-memory/pov/skills" +) # pov skills (aim-agent-dispatch, aim-agent-lifecycle, …) +CORE_SKILLS_ROOT = ( + REPO_ROOT / "_ai-memory/skills" +) # core skills (aim-best-practices-researcher, …) + +# Placeholder → absolute-path substitutions used in step frontmatter refs. +_PLACEHOLDERS = { + "{workflows_path}": str(WORKFLOWS_ROOT), + "{skills_path}": str(SKILLS_ROOT), + "{project-root}": str(REPO_ROOT), +} + +# --------------------------------------------------------------------------- +# Exempt set — workflows with no step chain (inline or reference docs). +# +# claude-native : reference doc (type: reference), no firstStep key. +# session/status: single-step inline workflow (firstStep: null). +# +# Both are skipped for firstStep-presence and chain-walk assertions only. +# name/description/h2-section assertions still run for both. +# --------------------------------------------------------------------------- +FIRSTEP_EXEMPT = frozenset( + [ + (MODEL_DISPATCH_ROOT / "claude-native/workflow.md").resolve(), + (WORKFLOWS_ROOT / "session/status/workflow.md").resolve(), + ] +) + + +# --------------------------------------------------------------------------- +# Core helpers (plain functions — available at parametrize collection time) +# --------------------------------------------------------------------------- + + +def parse_frontmatter(path: Path) -> dict: + """Return the YAML frontmatter dict from a markdown file, or {} if none.""" + text = path.read_text(encoding="utf-8") + if not text.startswith("---"): + return {} + parts = text.split("---", 2) + if len(parts) < 3: + return {} + return yaml.safe_load(parts[1]) or {} + + +def walk_step_chain(workflow_md: Path) -> list: + """Walk firstStep→nextStepFile chain from workflow_md. + + Returns the list of visited step Paths. Each nextStepFile is resolved + relative to the *step file's* parent — not the workflow root (BP-017 §12). + Raises AssertionError on dangling reference (file does not exist). + Returns [] when firstStep is absent or null. + + Cycles: some workflows deliberately loop (e.g. cycles/review-cycle, which + loops step-03→04→05→03 with an exitStepFile exit path controlled by prose + logic). A revisited step terminates the walk gracefully — the contract is + that every *referenced* file exists, not that the chain is acyclic. + """ + fm = parse_frontmatter(workflow_md) + first = fm.get("firstStep") + if not first: + return [] + + visited: list = [] + seen: set = set() + next_path = (workflow_md.parent / first).resolve() + + while next_path: + assert ( + next_path.exists() + ), f"Dangling step reference: {next_path} (from {workflow_md})" + if next_path in seen: + break # intentional loop — all links already verified; stop walking + seen.add(next_path) + visited.append(next_path) + + step_fm = parse_frontmatter(next_path) + ref = step_fm.get("nextStepFile") + if not ref: + break + next_path = (next_path.parent / ref).resolve() + + return visited + + +def resolve_template_ref(ref: str, step_path: Path) -> Path: + """Resolve a frontmatter template/path ref to an absolute Path. + + Substitutes {workflows_path}, {skills_path}, {project-root} placeholders, + then resolves relative refs against the step file's parent directory. + """ + raw = str(ref) + for placeholder, actual in _PLACEHOLDERS.items(): + raw = raw.replace(placeholder, actual) + p = Path(raw) + if p.is_absolute(): + return p.resolve() + return (step_path.parent / raw).resolve() + + +# --------------------------------------------------------------------------- +# Discovery functions (called at module level inside test files for parametrize) +# --------------------------------------------------------------------------- + + +def _all_workflow_mds() -> list: + """All workflow.md files from both workflow roots, sorted.""" + return sorted( + list(WORKFLOWS_ROOT.rglob("workflow.md")) + + list(MODEL_DISPATCH_ROOT.rglob("workflow.md")) + ) + + +def _all_step_mds() -> list: + """All step*.md files from both workflow roots, sorted.""" + return sorted( + list(WORKFLOWS_ROOT.rglob("step*.md")) + + list(MODEL_DISPATCH_ROOT.rglob("step*.md")) + ) + + +def _wf_id(p: Path) -> str: + """Human-readable pytest parametrize ID for a workflow.md path.""" + try: + return str(p.relative_to(WORKFLOWS_ROOT)) + except ValueError: + return "model-dispatch/" + str(p.relative_to(MODEL_DISPATCH_ROOT)) + + +def _step_id(p: Path) -> str: + """Human-readable pytest parametrize ID for a step file path.""" + try: + return str(p.relative_to(WORKFLOWS_ROOT)) + except ValueError: + return "model-dispatch/" + str(p.relative_to(MODEL_DISPATCH_ROOT)) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="session") +def workflows_root() -> Path: + """Session-scoped fixture: WORKFLOWS_ROOT (Section A root).""" + assert WORKFLOWS_ROOT.is_dir(), f"WORKFLOWS_ROOT not found: {WORKFLOWS_ROOT}" + return WORKFLOWS_ROOT diff --git a/tests/process/test_corpus_sentinel.py b/tests/process/test_corpus_sentinel.py new file mode 100644 index 00000000..f77647a9 --- /dev/null +++ b/tests/process/test_corpus_sentinel.py @@ -0,0 +1,36 @@ +"""Vacuous-green guard: corpus discovery must find minimum expected file counts. + +If either path anchor in conftest.py breaks (e.g. WORKFLOWS_ROOT no longer +exists), rglob returns [] → 0 tests collected across the parametrized suite → +false green. This non-parametrized sentinel catches that failure mode by +asserting hard minimums against the known corpus size. + +Minimums reflect the corpus at time of authoring (TASK-071 Phase 4): + - workflow.md : 26 (22 under WORKFLOWS_ROOT + 4 under MODEL_DISPATCH_ROOT) + - step*.md : 211 (183 under WORKFLOWS_ROOT + 28 under MODEL_DISPATCH_ROOT) +""" + +import pytest + +from .conftest import _all_step_mds, _all_workflow_mds + + +@pytest.mark.process +def test_corpus_sentinel(workflows_root): + """Discovery functions must find the minimum expected corpus size. + + Uses the workflows_root fixture to also validate that the path anchor is a + real directory. If the anchor breaks, this test — not a silent 0-collected + run — is the failure signal. + """ + wf_count = len(_all_workflow_mds()) + step_count = len(_all_step_mds()) + + assert wf_count >= 26, ( + f"workflow.md discovery returned {wf_count} files — expected >= 26. " + f"Check WORKFLOWS_ROOT / MODEL_DISPATCH_ROOT anchors in conftest.py." + ) + assert step_count >= 211, ( + f"step*.md discovery returned {step_count} files — expected >= 211. " + f"Check WORKFLOWS_ROOT / MODEL_DISPATCH_ROOT anchors in conftest.py." + ) diff --git a/tests/process/test_skill_procedures.py b/tests/process/test_skill_procedures.py new file mode 100644 index 00000000..1452ffaa --- /dev/null +++ b/tests/process/test_skill_procedures.py @@ -0,0 +1,69 @@ +"""Contract: Section-C embedded-procedure skill structural assertions. + +Tests SKILL.md files for all three Section-C skills: + 1. SKILL.md file exists + 2. Frontmatter has non-empty 'name' and 'description' + 3. Body contains at least one H2 section (## ...) + +Skills and their roots (two different roots — NOT a single skills directory): + aim-agent-dispatch → _ai-memory/pov/skills/ (pov skill) + aim-agent-lifecycle → _ai-memory/pov/skills/ (pov skill) + aim-best-practices-researcher → _ai-memory/skills/ (core skill) + +Scope notes (see INDEX.md for full coverage table): + - aim-agent-sanctum-init: covered by the existing idempotency test suite + (tests/test_install_sanctum_preservation.py); not duplicated here. +""" + +import pytest + +from .conftest import CORE_SKILLS_ROOT, SKILLS_ROOT, parse_frontmatter + +# (skill_name, skill_md_path) — explicit per-skill paths because the three +# Section-C skills live under two different roots. +_SECTION_C_SKILLS = [ + ("aim-agent-dispatch", SKILLS_ROOT / "aim-agent-dispatch/SKILL.md"), + ("aim-agent-lifecycle", SKILLS_ROOT / "aim-agent-lifecycle/SKILL.md"), + ( + "aim-best-practices-researcher", + CORE_SKILLS_ROOT / "aim-best-practices-researcher/SKILL.md", + ), +] + + +@pytest.mark.process +@pytest.mark.parametrize( + "skill_name,skill_md", + _SECTION_C_SKILLS, + ids=[name for name, _ in _SECTION_C_SKILLS], +) +def test_skill_md_exists(skill_name, skill_md): + assert skill_md.exists(), f"SKILL.md not found for {skill_name}: {skill_md}" + + +@pytest.mark.process +@pytest.mark.parametrize( + "skill_name,skill_md", + _SECTION_C_SKILLS, + ids=[name for name, _ in _SECTION_C_SKILLS], +) +def test_skill_frontmatter_schema(skill_name, skill_md): + if not skill_md.exists(): + pytest.skip(f"SKILL.md not found: {skill_name}") + fm = parse_frontmatter(skill_md) + assert fm.get("name"), f"Missing/empty 'name' in {skill_md}" + assert fm.get("description"), f"Missing/empty 'description' in {skill_md}" + + +@pytest.mark.process +@pytest.mark.parametrize( + "skill_name,skill_md", + _SECTION_C_SKILLS, + ids=[name for name, _ in _SECTION_C_SKILLS], +) +def test_skill_has_h2_section(skill_name, skill_md): + if not skill_md.exists(): + pytest.skip(f"SKILL.md not found: {skill_name}") + text = skill_md.read_text(encoding="utf-8") + h2_lines = [ln for ln in text.splitlines() if ln.startswith("## ")] + assert h2_lines, f"{skill_md}: SKILL.md has no '## ' sections" diff --git a/tests/process/test_step_chain.py b/tests/process/test_step_chain.py new file mode 100644 index 00000000..09acb435 --- /dev/null +++ b/tests/process/test_step_chain.py @@ -0,0 +1,38 @@ +"""Contract: step chain forward link resolution. + +For every workflow.md with a firstStep, walks the firstStep→nextStepFile spine +and asserts each referenced file exists (no dangling refs). + +NO reverse-reachability (orphan) test: the corpus contains branch/mode steps +(steps-e/, steps-v/, branches/branch-a..d/, route/step-01-resolve-backend.md) +that are reached by prose routing in step bodies, not by the linear nextStepFile +spine. A corpus-wide reverse-reachability check would false-fail on all of them. +The forward link-resolution contract below is the false-positive-free equivalent: +every path that IS referenced resolves; unreferenced paths are not asserted. +See INDEX.md §Coverage Notes for the full rationale. + +Parametrized over all 26 workflow.md files; FIRSTEP_EXEMPT workflows are skipped. +""" + +import pytest + +from .conftest import ( + FIRSTEP_EXEMPT, + _all_workflow_mds, + _wf_id, + walk_step_chain, +) + +_WORKFLOWS = _all_workflow_mds() + + +@pytest.mark.process +@pytest.mark.parametrize("wf_path", _WORKFLOWS, ids=_wf_id) +def test_step_chain_resolves(wf_path): + """Walk the full firstStep→nextStepFile chain; assert no dangling refs or cycles.""" + if wf_path.resolve() in FIRSTEP_EXEMPT: + pytest.skip( + "inline/reference workflow — no step chain (documented non-feasible)" + ) + chain = walk_step_chain(wf_path) + assert len(chain) >= 1, f"Workflow {wf_path} has no reachable steps" diff --git a/tests/process/test_step_frontmatter.py b/tests/process/test_step_frontmatter.py new file mode 100644 index 00000000..2d8ab4b4 --- /dev/null +++ b/tests/process/test_step_frontmatter.py @@ -0,0 +1,81 @@ +"""Contract: step file frontmatter schema and reference resolution. + +Parametrized over all step*.md files from both workflow roots (211 total). + +Three assertions per step file: + 1. name + description present and non-empty + 2. nextStepFile resolves to a real file when present + (resolved relative to the *step file's* parent — not workflow root; BP-017 §12) + 3. All template/path frontmatter keys resolve to real files + +Placeholder substitution for template refs (BP-017 §8): + {workflows_path} → WORKFLOWS_ROOT + {skills_path} → SKILLS_ROOT + {project-root} → REPO_ROOT +""" + +import pytest + +from .conftest import ( + _all_step_mds, + _step_id, + parse_frontmatter, + resolve_template_ref, +) + +# Frontmatter keys that cite paths to other files (templates, scaffolds, etc.) +_TEMPLATE_KEYS = frozenset( + { + "scaffold", + "handoffTemplate", + "correctionTemplate", + "storyTemplate", + "productionTemplate", + "phaseApprovalTemplate", + "taskApprovalTemplate", + "decisionLogTemplate", + "decisionPointTemplate", + "codeTemplate", + "codeReviewTemplate", + "analystInstructionTemplate", + "instructionTemplate", + "incompletenessTemplate", + "exitStepFile", + } +) + +_STEPS = _all_step_mds() + + +@pytest.mark.process +@pytest.mark.parametrize("step_path", _STEPS, ids=_step_id) +def test_step_name_and_description(step_path): + fm = parse_frontmatter(step_path) + assert fm.get("name"), f"Missing/empty 'name' in {step_path}" + assert fm.get("description"), f"Missing/empty 'description' in {step_path}" + + +@pytest.mark.process +@pytest.mark.parametrize("step_path", _STEPS, ids=_step_id) +def test_step_nextStepFile_resolves(step_path): + fm = parse_frontmatter(step_path) + ref = fm.get("nextStepFile") + if not ref: + pytest.skip("terminal step: no nextStepFile") + resolved = (step_path.parent / ref).resolve() + assert resolved.exists(), f"Dangling nextStepFile '{ref}' in {step_path}" + + +@pytest.mark.process +@pytest.mark.parametrize("step_path", _STEPS, ids=_step_id) +def test_step_template_refs_resolve(step_path): + fm = parse_frontmatter(step_path) + present_keys = _TEMPLATE_KEYS & fm.keys() + for key in sorted(present_keys): + ref = fm[key] + if not ref or ref is False: + continue + resolved = resolve_template_ref(str(ref), step_path) + assert ( + resolved.exists() + ), f"Step {step_path}: frontmatter key '{key}' → non-existent: {resolved}" diff --git a/tests/process/test_workflow_frontmatter.py b/tests/process/test_workflow_frontmatter.py new file mode 100644 index 00000000..3a764252 --- /dev/null +++ b/tests/process/test_workflow_frontmatter.py @@ -0,0 +1,55 @@ +"""Contract: workflow.md frontmatter schema and minimal section presence. + +Parametrized over all 26 workflow.md files from both workflow roots: + Root 1 (Section A): _ai-memory/pov/workflows/ — 22 files + Root 2 (Section B): aim-model-dispatch/workflows/ — 4 files + +Three assertions per workflow: + 1. name + description present and non-empty (all 26) + 2. firstStep present and non-null (24 of 26; 2 exempt — see FIRSTEP_EXEMPT) + 3. At least one H2 section (## ...) in the body (all 26) + +The firstStep exemption covers workflows with no step chain (BP-017 §12): + - claude-native : reference doc (type: reference) + - session/status: single-step inline workflow (firstStep: null — by design) +Both are skipped via pytest.skip so they appear in the report as explicitly +skipped, not silently omitted. +""" + +import pytest + +from .conftest import ( + FIRSTEP_EXEMPT, + _all_workflow_mds, + _wf_id, + parse_frontmatter, +) + +_WORKFLOWS = _all_workflow_mds() + + +@pytest.mark.process +@pytest.mark.parametrize("wf_path", _WORKFLOWS, ids=_wf_id) +def test_workflow_name_and_description(wf_path): + fm = parse_frontmatter(wf_path) + assert fm.get("name"), f"Missing/empty 'name' in {wf_path}" + assert fm.get("description"), f"Missing/empty 'description' in {wf_path}" + + +@pytest.mark.process +@pytest.mark.parametrize("wf_path", _WORKFLOWS, ids=_wf_id) +def test_workflow_firstStep_present(wf_path): + if wf_path.resolve() in FIRSTEP_EXEMPT: + pytest.skip( + "inline/reference workflow — no step chain (documented non-feasible)" + ) + fm = parse_frontmatter(wf_path) + assert fm.get("firstStep"), f"Missing/null 'firstStep' in {wf_path}" + + +@pytest.mark.process +@pytest.mark.parametrize("wf_path", _WORKFLOWS, ids=_wf_id) +def test_workflow_has_h2_section(wf_path): + text = wf_path.read_text(encoding="utf-8") + h2_lines = [ln for ln in text.splitlines() if ln.startswith("## ")] + assert h2_lines, f"{wf_path}: workflow.md has no '## ' sections" diff --git a/tests/process/test_workflow_map.py b/tests/process/test_workflow_map.py new file mode 100644 index 00000000..80b2b3b4 --- /dev/null +++ b/tests/process/test_workflow_map.py @@ -0,0 +1,73 @@ +"""Contract: WORKFLOW-MAP.md routing table validity. + +Every unique {workflows_path}/.../workflow.md entry in WORKFLOW-MAP.md must +resolve to a real file under WORKFLOWS_ROOT. + +The routing entries are extracted dynamically — the test count reflects the +actual number of unique entries in the current corpus (21 at time of authoring; +BP-017 §13 cited 16 which predates corpus growth). +""" + +import re + +import pytest + +from .conftest import WORKFLOWS_ROOT + +_WORKFLOW_MAP = WORKFLOWS_ROOT / "WORKFLOW-MAP.md" + +# Narrow extractor: only lowercase paths (the normal convention). +_ROUTE_RE = re.compile(r"\{workflows_path\}/([a-z][a-z0-9/_\-]*/workflow\.md)") + +# Broad extractor: case-insensitive, used only for the cross-check sentinel. +# If a malformed/uppercase entry exists that _ROUTE_RE silently drops, +# the counts will diverge and test_workflow_map_no_dropped_entries will fail. +_ROUTE_RE_BROAD = re.compile( + r"\{workflows_path\}/([^\s{}/][^\s{}]*/workflow\.md)", re.IGNORECASE +) + + +def _routing_entries() -> list: + """Return deduplicated list of relative paths from WORKFLOW-MAP routing entries.""" + text = _WORKFLOW_MAP.read_text(encoding="utf-8") + seen: set = set() + entries = [] + for m in _ROUTE_RE.finditer(text): + rel = m.group(1) + if rel not in seen: + seen.add(rel) + entries.append(rel) + return entries + + +def _routing_entries_broad() -> set: + """Broad (case-insensitive) deduplicated set of workflow.md refs.""" + text = _WORKFLOW_MAP.read_text(encoding="utf-8") + return {m.group(1).lower() for m in _ROUTE_RE_BROAD.finditer(text)} + + +@pytest.mark.process +def test_workflow_map_no_dropped_entries(): + """Narrow extractor must not silently drop any workflow.md reference. + + If the narrow _ROUTE_RE misses an entry (e.g. due to uppercase letters or + an unexpected path format), the broad grep will find a different count and + this test fails — surfacing the dropped entry rather than hiding it. + """ + narrow = set(_routing_entries()) + broad = _routing_entries_broad() + assert narrow == broad, ( + f"_ROUTE_RE dropped {len(broad) - len(narrow)} workflow.md reference(s). " + f"Entries in WORKFLOW-MAP not captured by narrow regex: {broad - narrow}" + ) + + +@pytest.mark.process +@pytest.mark.parametrize("rel_path", _routing_entries(), ids=lambda r: r) +def test_workflow_map_entry_resolves(rel_path): + """WORKFLOW-MAP routing entry must resolve to a real workflow.md.""" + resolved = (WORKFLOWS_ROOT / rel_path).resolve() + assert resolved.exists(), ( + f"WORKFLOW-MAP entry '{{workflows_path}}/{rel_path}' → " + f"non-existent: {resolved}" + )