Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 35 additions & 3 deletions graphify/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import os
import re
import tempfile
from collections.abc import Iterable
from pathlib import Path

# Output directory name — override with GRAPHIFY_OUT env var for worktrees or
Expand Down Expand Up @@ -542,6 +543,7 @@ def save_semantic_cache(
hyperedges: list[dict] | None = None,
root: Path = Path("."),
merge_existing: bool = False,
allowed_source_files: Iterable[str | Path] | None = None,
) -> int:
"""Save semantic extraction results to cache, keyed by source_file.

Expand All @@ -553,6 +555,11 @@ def save_semantic_cache(
unioned with the new results before saving instead of being overwritten.
This lets callers checkpoint incrementally (e.g. once per chunk) without
dropping a prior slice of a large file that was split across chunks.

When ``allowed_source_files`` is provided, only those files may be used as
cache-write keys. Semantic nodes can legitimately mention another corpus
file, but a model must not be able to replace that file's complete cache
entry unless the file was part of the current extraction batch (#1757).
Returns the number of files cached.
"""
from collections import defaultdict
Expand All @@ -571,12 +578,37 @@ def save_semantic_cache(
if src:
by_file[src]["hyperedges"].append(h)

root_path = Path(root).resolve()

def resolved_source_path(value: str | Path) -> Path:
path = Path(value)
if not path.is_absolute():
path = root_path / path
try:
return path.resolve()
except (OSError, RuntimeError):
# Keep the cache write best-effort for inaccessible paths or a
# symlink loop emitted by an untrusted semantic result.
return Path(os.path.abspath(path))

allowed_paths = None
if allowed_source_files is not None:
allowed_paths = {resolved_source_path(path) for path in allowed_source_files}

saved = 0
for fpath, result in by_file.items():
p = Path(fpath)
if not p.is_absolute():
p = Path(root) / p
p = resolved_source_path(fpath)
if p.is_file():
if allowed_paths is not None and p not in allowed_paths:
import warnings

warnings.warn(
"semantic cache skipped out-of-scope source_file "
f"{fpath!r}; the file was not dispatched for extraction",
RuntimeWarning,
stacklevel=2,
)
continue
if merge_existing:
prev = load_cached(p, root, kind="semantic")
if prev:
Expand Down
1 change: 1 addition & 0 deletions graphify/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2354,6 +2354,7 @@ def _progress(idx: int, total: int, _result: dict) -> None:
fresh.get("edges", []),
fresh.get("hyperedges", []),
root=out_root,
allowed_source_files=uncached_paths,
)
except Exception as exc:
print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr)
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-agents.md
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-aider.md
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []))
uncached = [line for line in Path('.graphify_uncached.txt').read_text().splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-amp.md
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-claw.md
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-codex.md
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-copilot.md
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-devin.md
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []))
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text().splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-droid.md
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-kilo.md
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-kiro.md
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-opencode.md
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-pi.md
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-trae.md
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path

new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-vscode.md
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill-windows.md
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion graphify/skill.md
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
46 changes: 46 additions & 0 deletions tests/test_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,52 @@ def test_save_semantic_cache_overwrites_by_default(tmp_path):
assert ids == {"b"}, "default must overwrite, not accumulate"


def test_save_semantic_cache_rejects_out_of_scope_source_file(tmp_path):
"""#1757: an undispatched file must keep its complete cache entry when a
semantic result misattributes a node to it."""
from graphify.cache import save_semantic_cache

intended = tmp_path / "intended.md"
intended.write_text("# Intended\n")
protected = tmp_path / "protected.md"
protected.write_text("# Protected\n")

save_semantic_cache(
[{"id": "original", "source_file": "protected.md"}],
[],
root=tmp_path,
)

nodes = [
{"id": "expected", "source_file": str(intended.resolve())},
{"id": "stray", "source_file": "protected.md"},
]
edges = [
{"source": "stray", "target": "expected", "source_file": "protected.md"},
]
hyperedges = [
{"id": "stray_hyperedge", "nodes": ["stray"], "source_file": "protected.md"},
]

with pytest.warns(RuntimeWarning, match="out-of-scope source_file 'protected.md'"):
saved = save_semantic_cache(
nodes,
edges,
hyperedges,
root=tmp_path,
allowed_source_files=["intended.md"],
)

assert saved == 1
intended_cache = load_cached(intended, root=tmp_path, kind="semantic")
assert {node["id"] for node in intended_cache["nodes"]} == {"expected"}

protected_cache = load_cached(protected, root=tmp_path, kind="semantic")
assert {node["id"] for node in protected_cache["nodes"]} == {"original"}
assert protected_cache["edges"] == []
assert protected_cache["hyperedges"] == []


def test_save_semantic_cache_merge_existing_unions(tmp_path):
"""#1715: merge_existing=True unions with the prior entry so a file split
across chunks (checkpointed per chunk) keeps every slice."""
Expand Down
12 changes: 12 additions & 0 deletions tests/test_extract_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,15 @@ def _one_chunk_succeeded(paths, **kwargs):
monkeypatch.setattr(
"graphify.llm.extract_corpus_parallel", _one_chunk_succeeded
)
cache_call = {}

def _capture_semantic_cache(*args, **kwargs):
cache_call.update(kwargs)
return 0

monkeypatch.setattr(
"graphify.cache.save_semantic_cache", _capture_semantic_cache
)
monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None)
monkeypatch.setattr(
mainmod.sys,
Expand All @@ -121,6 +130,9 @@ def _one_chunk_succeeded(paths, **kwargs):
assert (out_dir / "graphify-out" / "graph.json").exists(), (
"graph.json must be written on the happy path"
)
assert {
str(path) for path in cache_call["allowed_source_files"]
} == {str(corpus / "README.md")}


def _code_only_corpus(tmp_path):
Expand Down
13 changes: 12 additions & 1 deletion tests/test_skillgen.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,7 +487,8 @@ def test_monoliths_change_only_sanctioned_lines():
The round-trip (multiset diff vs the pinned v8 blob) must come back clean:
each added/removed line matches one of the documented sanctioned predicates
in gen — the enum unification, the unified description, the chunk-cleanup
rewrite (#1172), and the four #1392 runbook fixes. Anything else is drift.
rewrite (#1172), the four #1392 runbook fixes, and semantic-cache source
scoping (#1757). Anything else is drift.
"""
platforms = gen.load_platforms()
for key in ("aider", "devin"):
Expand Down Expand Up @@ -534,6 +535,16 @@ def test_monoliths_carry_the_1392_runbook_fixes():
assert "if not wrote:" in body


def test_monoliths_scope_semantic_cache_writes_to_uncached_files():
"""#1757: generated monoliths pass the dispatched-file allowlist when
replacing semantic cache entries."""
platforms = gen.load_platforms()
for key in ("aider", "devin"):
body = gen.render(platforms[key])[0].content
assert ".graphify_uncached.txt').read_text(" in body
assert "allowed_source_files=uncached" in body


def test_generated_runbooks_pass_root_to_save_manifest():
"""#1417: every save_manifest call in a shipped runbook threads root=.

Expand Down
3 changes: 2 additions & 1 deletion tools/skillgen/expected/graphify__skill-agents.md
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH')
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
3 changes: 2 additions & 1 deletion tools/skillgen/expected/graphify__skill-aider.md
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache
from pathlib import Path
new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []))
uncached = [line for line in Path('.graphify_uncached.txt').read_text().splitlines() if line]
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached)
print(f'Cached {saved} files')
"
```
Expand Down
Loading
Loading