From 781f715087cbd6ce46664d22bcf67e5ea676ee2d Mon Sep 17 00:00:00 2001 From: tpateeq Date: Mon, 13 Jul 2026 01:09:42 +0530 Subject: [PATCH] fix(cache): scope semantic cache writes to extracted files (#1757) --- graphify/cache.py | 38 +++++++++++++-- graphify/cli.py | 1 + graphify/skill-agents.md | 3 +- graphify/skill-aider.md | 3 +- graphify/skill-amp.md | 3 +- graphify/skill-claw.md | 3 +- graphify/skill-codex.md | 3 +- graphify/skill-copilot.md | 3 +- graphify/skill-devin.md | 3 +- graphify/skill-droid.md | 3 +- graphify/skill-kilo.md | 3 +- graphify/skill-kiro.md | 3 +- graphify/skill-opencode.md | 3 +- graphify/skill-pi.md | 3 +- graphify/skill-trae.md | 3 +- graphify/skill-vscode.md | 3 +- graphify/skill-windows.md | 3 +- graphify/skill.md | 3 +- tests/test_cache.py | 46 +++++++++++++++++++ tests/test_extract_cli.py | 12 +++++ tests/test_skillgen.py | 13 +++++- .../expected/graphify__skill-agents.md | 3 +- .../expected/graphify__skill-aider.md | 3 +- .../skillgen/expected/graphify__skill-amp.md | 3 +- .../skillgen/expected/graphify__skill-claw.md | 3 +- .../expected/graphify__skill-codex.md | 3 +- .../expected/graphify__skill-copilot.md | 3 +- .../expected/graphify__skill-devin.md | 3 +- .../expected/graphify__skill-droid.md | 3 +- .../skillgen/expected/graphify__skill-kilo.md | 3 +- .../skillgen/expected/graphify__skill-kiro.md | 3 +- .../expected/graphify__skill-opencode.md | 3 +- tools/skillgen/expected/graphify__skill-pi.md | 3 +- .../skillgen/expected/graphify__skill-trae.md | 3 +- .../expected/graphify__skill-vscode.md | 3 +- .../expected/graphify__skill-windows.md | 3 +- tools/skillgen/expected/graphify__skill.md | 3 +- tools/skillgen/fragments/core/aider.md | 3 +- tools/skillgen/fragments/core/core.md | 3 +- tools/skillgen/fragments/core/devin.md | 3 +- tools/skillgen/gen.py | 22 ++++++++- 41 files changed, 196 insertions(+), 41 deletions(-) diff --git a/graphify/cache.py b/graphify/cache.py index 31c945a25..34928a417 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -7,6 +7,7 @@ import os import re import tempfile +from collections.abc import Iterable from pathlib import Path # Output directory name — override with GRAPHIFY_OUT env var for worktrees or @@ -542,6 +543,7 @@ def save_semantic_cache( hyperedges: list[dict] | None = None, root: Path = Path("."), merge_existing: bool = False, + allowed_source_files: Iterable[str | Path] | None = None, ) -> int: """Save semantic extraction results to cache, keyed by source_file. @@ -553,6 +555,11 @@ def save_semantic_cache( unioned with the new results before saving instead of being overwritten. This lets callers checkpoint incrementally (e.g. once per chunk) without dropping a prior slice of a large file that was split across chunks. + + When ``allowed_source_files`` is provided, only those files may be used as + cache-write keys. Semantic nodes can legitimately mention another corpus + file, but a model must not be able to replace that file's complete cache + entry unless the file was part of the current extraction batch (#1757). Returns the number of files cached. """ from collections import defaultdict @@ -571,12 +578,37 @@ def save_semantic_cache( if src: by_file[src]["hyperedges"].append(h) + root_path = Path(root).resolve() + + def resolved_source_path(value: str | Path) -> Path: + path = Path(value) + if not path.is_absolute(): + path = root_path / path + try: + return path.resolve() + except (OSError, RuntimeError): + # Keep the cache write best-effort for inaccessible paths or a + # symlink loop emitted by an untrusted semantic result. + return Path(os.path.abspath(path)) + + allowed_paths = None + if allowed_source_files is not None: + allowed_paths = {resolved_source_path(path) for path in allowed_source_files} + saved = 0 for fpath, result in by_file.items(): - p = Path(fpath) - if not p.is_absolute(): - p = Path(root) / p + p = resolved_source_path(fpath) if p.is_file(): + if allowed_paths is not None and p not in allowed_paths: + import warnings + + warnings.warn( + "semantic cache skipped out-of-scope source_file " + f"{fpath!r}; the file was not dispatched for extraction", + RuntimeWarning, + stacklevel=2, + ) + continue if merge_existing: prev = load_cached(p, root, kind="semantic") if prev: diff --git a/graphify/cli.py b/graphify/cli.py index 25bf5499b..0bb124777 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -2354,6 +2354,7 @@ def _progress(idx: int, total: int, _result: dict) -> None: fresh.get("edges", []), fresh.get("hyperedges", []), root=out_root, + allowed_source_files=uncached_paths, ) except Exception as exc: print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 6961fa786..4969929c0 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index 5d4f6e0d8..8db8f73f2 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +uncached = [line for line in Path('.graphify_uncached.txt').read_text().splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 6961fa786..4969929c0 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 7985c508e..bafbca074 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index e6387fed1..d71f4592d 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 7985c508e..bafbca074 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index 21a4d1ad9..c444a852d 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -374,7 +374,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text().splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 25d55b23f..ae1d49325 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index da34d0d0f..6aa87a8bd 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index 7985c508e..bafbca074 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index c40f52780..8c3ce3eed 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -305,7 +305,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index 7985c508e..bafbca074 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index fc62552da..af59166ae 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -311,7 +311,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index bc303d8e3..11a2837b5 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -309,7 +309,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index b298f29ab..0dc73947a 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -335,7 +335,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/graphify/skill.md b/graphify/skill.md index 7985c508e..bafbca074 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tests/test_cache.py b/tests/test_cache.py index 9bf2cdc82..d772d1ec4 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -524,6 +524,52 @@ def test_save_semantic_cache_overwrites_by_default(tmp_path): assert ids == {"b"}, "default must overwrite, not accumulate" +def test_save_semantic_cache_rejects_out_of_scope_source_file(tmp_path): + """#1757: an undispatched file must keep its complete cache entry when a + semantic result misattributes a node to it.""" + from graphify.cache import save_semantic_cache + + intended = tmp_path / "intended.md" + intended.write_text("# Intended\n") + protected = tmp_path / "protected.md" + protected.write_text("# Protected\n") + + save_semantic_cache( + [{"id": "original", "source_file": "protected.md"}], + [], + root=tmp_path, + ) + + nodes = [ + {"id": "expected", "source_file": str(intended.resolve())}, + {"id": "stray", "source_file": "protected.md"}, + ] + edges = [ + {"source": "stray", "target": "expected", "source_file": "protected.md"}, + ] + hyperedges = [ + {"id": "stray_hyperedge", "nodes": ["stray"], "source_file": "protected.md"}, + ] + + with pytest.warns(RuntimeWarning, match="out-of-scope source_file 'protected.md'"): + saved = save_semantic_cache( + nodes, + edges, + hyperedges, + root=tmp_path, + allowed_source_files=["intended.md"], + ) + + assert saved == 1 + intended_cache = load_cached(intended, root=tmp_path, kind="semantic") + assert {node["id"] for node in intended_cache["nodes"]} == {"expected"} + + protected_cache = load_cached(protected, root=tmp_path, kind="semantic") + assert {node["id"] for node in protected_cache["nodes"]} == {"original"} + assert protected_cache["edges"] == [] + assert protected_cache["hyperedges"] == [] + + def test_save_semantic_cache_merge_existing_unions(tmp_path): """#1715: merge_existing=True unions with the prior entry so a file split across chunks (checkpointed per chunk) keeps every slice.""" diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index c301c50e5..cf5762a7e 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -102,6 +102,15 @@ def _one_chunk_succeeded(paths, **kwargs): monkeypatch.setattr( "graphify.llm.extract_corpus_parallel", _one_chunk_succeeded ) + cache_call = {} + + def _capture_semantic_cache(*args, **kwargs): + cache_call.update(kwargs) + return 0 + + monkeypatch.setattr( + "graphify.cache.save_semantic_cache", _capture_semantic_cache + ) monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -121,6 +130,9 @@ def _one_chunk_succeeded(paths, **kwargs): assert (out_dir / "graphify-out" / "graph.json").exists(), ( "graph.json must be written on the happy path" ) + assert { + str(path) for path in cache_call["allowed_source_files"] + } == {str(corpus / "README.md")} def _code_only_corpus(tmp_path): diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index 282aedfbf..9797267a9 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -487,7 +487,8 @@ def test_monoliths_change_only_sanctioned_lines(): The round-trip (multiset diff vs the pinned v8 blob) must come back clean: each added/removed line matches one of the documented sanctioned predicates in gen — the enum unification, the unified description, the chunk-cleanup - rewrite (#1172), and the four #1392 runbook fixes. Anything else is drift. + rewrite (#1172), the four #1392 runbook fixes, and semantic-cache source + scoping (#1757). Anything else is drift. """ platforms = gen.load_platforms() for key in ("aider", "devin"): @@ -534,6 +535,16 @@ def test_monoliths_carry_the_1392_runbook_fixes(): assert "if not wrote:" in body +def test_monoliths_scope_semantic_cache_writes_to_uncached_files(): + """#1757: generated monoliths pass the dispatched-file allowlist when + replacing semantic cache entries.""" + platforms = gen.load_platforms() + for key in ("aider", "devin"): + body = gen.render(platforms[key])[0].content + assert ".graphify_uncached.txt').read_text(" in body + assert "allowed_source_files=uncached" in body + + def test_generated_runbooks_pass_root_to_save_manifest(): """#1417: every save_manifest call in a shipped runbook threads root=. diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 6961fa786..4969929c0 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md index 5d4f6e0d8..8db8f73f2 100644 --- a/tools/skillgen/expected/graphify__skill-aider.md +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +uncached = [line for line in Path('.graphify_uncached.txt').read_text().splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 6961fa786..4969929c0 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index 7985c508e..bafbca074 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index e6387fed1..d71f4592d 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index 7985c508e..bafbca074 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md index 21a4d1ad9..c444a852d 100644 --- a/tools/skillgen/expected/graphify__skill-devin.md +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -374,7 +374,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text().splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index 25d55b23f..ae1d49325 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index da34d0d0f..6aa87a8bd 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index 7985c508e..bafbca074 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index c40f52780..8c3ce3eed 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -305,7 +305,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index 7985c508e..bafbca074 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index fc62552da..af59166ae 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -311,7 +311,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index bc303d8e3..11a2837b5 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -309,7 +309,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index b298f29ab..0dc73947a 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -335,7 +335,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index 7985c508e..bafbca074 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md index 5d4f6e0d8..8db8f73f2 100644 --- a/tools/skillgen/fragments/core/aider.md +++ b/tools/skillgen/fragments/core/aider.md @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +uncached = [line for line in Path('.graphify_uncached.txt').read_text().splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index 8d8aab7aa..a5e117b5c 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -248,7 +248,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md index 21a4d1ad9..c444a852d 100644 --- a/tools/skillgen/fragments/core/devin.md +++ b/tools/skillgen/fragments/core/devin.md @@ -374,7 +374,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text().splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) print(f'Cached {saved} files') " ``` diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index e6dbbd9bf..93e403f27 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -902,6 +902,22 @@ def _is_uv_from_interpreter_fix_line(line: str) -> bool: return "uv tool run" in line and "graphifyy python" in line +def _is_semantic_cache_scope_fix_line(line: str) -> bool: + """Whether a line scopes semantic cache writes to dispatched files (#1757). + + A semantic subagent can mention a corpus file outside its assigned chunk and + misattribute a node to that file. The final cache write now passes the B0 + uncached-file list as an allowlist, so an incidental mention cannot replace + another file's complete cached extraction. Both the old unscoped call + (removed) and the allowlist read/call (added) are sanctioned here. + """ + stripped = line.strip() + return ( + stripped.startswith("uncached = [line for line in Path(") + and ".graphify_uncached.txt" in stripped + ) or stripped.startswith("saved = save_semantic_cache(") + + # Every line that may differ between a rendered monolith and its pristine v8 # baseline. Each predicate documents one sanctioned change-class; a blank line is # allowed because the multi-line fix blocks insert spacing. Anything else failing @@ -919,6 +935,7 @@ def _is_uv_from_interpreter_fix_line(line: str) -> bool: _is_shebang_allowlist_fix_line, _is_obsidian_usage_comment_line, _is_uv_from_interpreter_fix_line, + _is_semantic_cache_scope_fix_line, ) @@ -935,8 +952,9 @@ def monolith_roundtrip(platform: Platform) -> list[str]: arbitrary edit (even a blessed one) from drifting them. Sanctioned changes are enumerated as predicates in ``_SANCTIONED_MONOLITH_DIFFS``: the file_type enum unification, the unified frontmatter description, the chunk-cleanup rewrite - (#1172), and the four #1392 runbook fixes (directed propagation, content-only - semantic scope, stale-cache unlink, and the zero-node/shrink-guard ordering). + (#1172), the four #1392 runbook fixes (directed propagation, content-only + semantic scope, stale-cache unlink, and the zero-node/shrink-guard ordering), + and semantic-cache source scoping (#1757). The comparison is a multiset diff, not a positional zip: a line whose text is unchanged but merely *moved* (the report-write line shifted below ``to_json``