Skip to content

fix: skip impossible file-anchored dedup scoring #1833

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
GautamBytes wants to merge 1 commit into
base: v8
Choose a base branch
from
Open

fix: skip impossible file-anchored dedup scoring #1833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
281 changes: 165 additions & 116 deletions graphify/dedup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,10 @@ def _numeric_tokens_differ(a: str, b: str) -> bool:
_FILE_ANCHORED_NONCODE = frozenset({"rationale", "document"})


def _is_file_anchored_noncode(node: dict) -> bool:
return node.get("file_type") in _FILE_ANCHORED_NONCODE


def _crossfile_fileanchored_blocked(node: dict, neighbor: dict) -> bool:
"""Block label-based merging of file-anchored non-code nodes across files (#1284).

Expand All @@ -136,6 +140,36 @@ def _crossfile_fileanchored_blocked(node: dict, neighbor: dict) -> bool:
return (node.get("source_file") or "") != (neighbor.get("source_file") or "")


def _fuzzy_candidate_groups(candidates: list[dict]) -> list[list[dict]]:
"""Group candidates so impossible file-anchored cross-file pairs never enter LSH.

`rationale` and `document` nodes can merge only with nodes from the same
source_file. Concepts and other non-file-anchored candidates keep the old
global fuzzy-dedup behavior.
"""
groups: list[list[dict]] = []
global_group: list[dict] = []
by_source: dict[str, list[dict]] = defaultdict(list)
anchored_sources: set[str] = set()

for node in candidates:
source_file = node.get("source_file") or ""
by_source[source_file].append(node)
if _is_file_anchored_noncode(node):
anchored_sources.add(source_file)
else:
global_group.append(node)

if len(global_group) >= 2:
groups.append(global_group)

for source_file, source_group in by_source.items():
if source_file in anchored_sources and len(source_group) >= 2:
groups.append(source_group)

return groups


# ── union-find ────────────────────────────────────────────────────────────────

class _UF:
Expand Down Expand Up @@ -299,99 +333,108 @@ def deduplicate_entities(

fuzzy_merges = 0
if len(candidates) >= 2:
lsh = MinHashLSH(threshold=_LSH_THRESHOLD, num_perm=_NUM_PERM)
minhashes: dict[str, MinHash] = {}
# Pre-build O(1) lookup structures so the query loop below doesn't scan
# the candidates list linearly for every LSH neighbor (was O(n²×B)).
candidates_by_id: dict[str, dict] = {}
norm_cache: dict[str, str] = {}

minhashes: dict[str, MinHash] = {}
for node in candidates:
node_id = node["id"]
candidates_by_id[node_id] = node
nl = _norm(node.get("label", node.get("id", "")))
norm_cache[node_id] = nl
m = _make_minhash(nl)
minhashes[node_id] = m
try:
lsh.insert(node_id, m)
except ValueError:
pass # duplicate key in LSH — already inserted

for node in candidates:
node_id = node["id"]
norm_label = norm_cache[node_id]
neighbors = lsh.query(minhashes[node_id])

for neighbor_id in neighbors:
if neighbor_id == node_id:
continue
if uf.find(node_id) == uf.find(neighbor_id):
continue

neighbor = candidates_by_id.get(neighbor_id)
if neighbor is None:
continue

neighbor_norm = norm_cache.get(neighbor_id) or _norm(neighbor.get("label", neighbor.get("id", "")))
# Cross-file long labels score on plain Jaro (no prefix bonus).
# Jaro-Winkler's leading-prefix bonus lifts pairs that share a
# prefix but diverge in a distinguishing token ("testing-library
# jest-native" vs "react-native") past threshold, fabricating
# destructive cross-file merges; on Jaro alone they fall short
# while true cross-file duplicates still clear it (#1243). Same-file
# near-duplicates keep Jaro-Winkler (low-risk, and a mid-string
# stopword insertion needs the prefix bonus to merge); short labels
# keep Jaro-Winkler too (gated by _short_label_blocked).
_xfile = (node.get("source_file") or "") != (neighbor.get("source_file") or "")
if _xfile and max(len(norm_label), len(neighbor_norm)) >= 12:
score = Jaro.normalized_similarity(norm_label, neighbor_norm) * 100
else:
score = JaroWinkler.normalized_similarity(norm_label, neighbor_norm) * 100
seen_pairs: set[tuple[str, str]] = set()
for group in _fuzzy_candidate_groups(candidates):
lsh = MinHashLSH(threshold=_LSH_THRESHOLD, num_perm=_NUM_PERM)
# Pre-build O(1) lookup structures so the query loop below doesn't scan
# the candidates list linearly for every LSH neighbor (was O(n²×B)).
candidates_by_id: dict[str, dict] = {}

if _is_variant_pair(norm_label, neighbor_norm):
continue
if _short_label_blocked(norm_label, neighbor_norm, score):
continue
# Prefix-extension pairs (getActiveSession / getActiveSessions,
# parseConfig / parseConfigFile) are almost never duplicates —
# one is a strict suffix-extension of the other. Block the merge
# regardless of JW score (#1201).
_lo, _hi = sorted((norm_label, neighbor_norm), key=len)
if _hi.startswith(_lo) and _hi != _lo:
continue
# Numbered/versioned siblings and cross-file file-anchored
# boilerplate (rationale/document) are decisively distinct
# regardless of score (#1284).
if _numeric_tokens_differ(norm_label, neighbor_norm):
continue
if _crossfile_fileanchored_blocked(node, neighbor):
continue

c1 = communities.get(node_id)
c2 = communities.get(neighbor_id)
if (c1 is not None and c2 is not None and c1 == c2
and min(len(norm_label), len(neighbor_norm)) >= 12):
score += _COMMUNITY_BOOST

if score >= _MERGE_THRESHOLD:
# Identical labels across different source files almost always
# means same-named-but-different symbols (trait impls, wrapper
# methods, common type names). Mirror Pass 1's source_file
# partition for this sub-case. (#1046, leaks #895's fix)
if norm_label == neighbor_norm:
sf_a = node.get("source_file") or ""
sf_b = neighbor.get("source_file") or ""
if sf_a != sf_b:
continue
# Pick the winner from the verified pair only. Selecting it
# from the union of both normalized-label groups pulls
# never-compared nodes (same label, different source_file)
# into the merge, bypassing the #1046/#1178 guards.
winner = _pick_winner([node, neighbor])
uf.union(winner["id"], node_id)
uf.union(winner["id"], neighbor_id)
fuzzy_merges += 1
for node in group:
node_id = node["id"]
candidates_by_id[node_id] = node
try:
lsh.insert(node_id, minhashes[node_id])
except ValueError:
pass # duplicate key in LSH — already inserted

for node in group:
node_id = node["id"]
norm_label = norm_cache[node_id]
neighbors = lsh.query(minhashes[node_id])

for neighbor_id in neighbors:
if neighbor_id == node_id:
continue
pair_key = tuple(sorted((node_id, neighbor_id)))
if pair_key in seen_pairs:
continue
seen_pairs.add(pair_key)
if uf.find(node_id) == uf.find(neighbor_id):
continue

neighbor = candidates_by_id.get(neighbor_id)
if neighbor is None:
continue

neighbor_norm = norm_cache.get(neighbor_id) or _norm(neighbor.get("label", neighbor.get("id", "")))
# Cross-file long labels score on plain Jaro (no prefix bonus).
# Jaro-Winkler's leading-prefix bonus lifts pairs that share a
# prefix but diverge in a distinguishing token ("testing-library
# jest-native" vs "react-native") past threshold, fabricating
# destructive cross-file merges; on Jaro alone they fall short
# while true cross-file duplicates still clear it (#1243). Same-file
# near-duplicates keep Jaro-Winkler (low-risk, and a mid-string
# stopword insertion needs the prefix bonus to merge); short labels
# keep Jaro-Winkler too (gated by _short_label_blocked).
_xfile = (node.get("source_file") or "") != (neighbor.get("source_file") or "")
if _xfile and max(len(norm_label), len(neighbor_norm)) >= 12:
score = Jaro.normalized_similarity(norm_label, neighbor_norm) * 100
else:
score = JaroWinkler.normalized_similarity(norm_label, neighbor_norm) * 100

if _is_variant_pair(norm_label, neighbor_norm):
continue
if _short_label_blocked(norm_label, neighbor_norm, score):
continue
# Prefix-extension pairs (getActiveSession / getActiveSessions,
# parseConfig / parseConfigFile) are almost never duplicates —
# one is a strict suffix-extension of the other. Block the merge
# regardless of JW score (#1201).
_lo, _hi = sorted((norm_label, neighbor_norm), key=len)
if _hi.startswith(_lo) and _hi != _lo:
continue
# Numbered/versioned siblings and cross-file file-anchored
# boilerplate (rationale/document) are decisively distinct
# regardless of score (#1284).
if _numeric_tokens_differ(norm_label, neighbor_norm):
continue
if _crossfile_fileanchored_blocked(node, neighbor):
continue

c1 = communities.get(node_id)
c2 = communities.get(neighbor_id)
if (c1 is not None and c2 is not None and c1 == c2
and min(len(norm_label), len(neighbor_norm)) >= 12):
score += _COMMUNITY_BOOST

if score >= _MERGE_THRESHOLD:
# Identical labels across different source files almost always
# means same-named-but-different symbols (trait impls, wrapper
# methods, common type names). Mirror Pass 1's source_file
# partition for this sub-case. (#1046, leaks #895's fix)
if norm_label == neighbor_norm:
sf_a = node.get("source_file") or ""
sf_b = neighbor.get("source_file") or ""
if sf_a != sf_b:
continue
# Pick the winner from the verified pair only. Selecting it
# from the union of both normalized-label groups pulls
# never-compared nodes (same label, different source_file)
# into the merge, bypassing the #1046/#1178 guards.
winner = _pick_winner([node, neighbor])
uf.union(winner["id"], node_id)
uf.union(winner["id"], neighbor_id)
fuzzy_merges += 1

# ── pass 3: LLM tiebreaker for ambiguous pairs (opt-in) ──────────────────
if dedup_llm_backend is not None:
Expand Down Expand Up @@ -484,38 +527,44 @@ def _llm_tiebreak(
return

ambiguous: list[tuple[dict, dict, float]] = []
for i, node in enumerate(candidates):
norm_i = _norm(node.get("label", node.get("id", "")))
for j in range(i + 1, len(candidates)):
neighbor = candidates[j]
if uf.find(node["id"]) == uf.find(neighbor["id"]):
continue
norm_j = _norm(neighbor.get("label", neighbor.get("id", "")))
# Mirror pass 2: plain Jaro for cross-file long labels (#1243).
_xfile = (node.get("source_file") or "") != (neighbor.get("source_file") or "")
if _xfile and max(len(norm_i), len(norm_j)) >= 12:
score = Jaro.normalized_similarity(norm_i, norm_j) * 100
else:
score = JaroWinkler.normalized_similarity(norm_i, norm_j) * 100
if _is_variant_pair(norm_i, norm_j):
continue
if _short_label_blocked(norm_i, norm_j, score):
continue
_lo, _hi = sorted((norm_i, norm_j), key=len)
if _hi.startswith(_lo) and _hi != _lo:
continue
# Mirror pass 2: decisively-distinct pairs never reach the LLM (#1284).
if _numeric_tokens_differ(norm_i, norm_j):
continue
if _crossfile_fileanchored_blocked(node, neighbor):
continue
c1 = communities.get(node["id"])
c2 = communities.get(neighbor["id"])
if (c1 is not None and c2 is not None and c1 == c2
and min(len(norm_i), len(norm_j)) >= 12):
score += _COMMUNITY_BOOST
if low <= score < high:
ambiguous.append((node, neighbor, score))
seen_pairs: set[tuple[str, str]] = set()
for group in _fuzzy_candidate_groups(candidates):
for i, node in enumerate(group):
norm_i = _norm(node.get("label", node.get("id", "")))
for j in range(i + 1, len(group)):
neighbor = group[j]
pair_key = tuple(sorted((node["id"], neighbor["id"])))
if pair_key in seen_pairs:
continue
seen_pairs.add(pair_key)
if uf.find(node["id"]) == uf.find(neighbor["id"]):
continue
norm_j = _norm(neighbor.get("label", neighbor.get("id", "")))
# Mirror pass 2: plain Jaro for cross-file long labels (#1243).
_xfile = (node.get("source_file") or "") != (neighbor.get("source_file") or "")
if _xfile and max(len(norm_i), len(norm_j)) >= 12:
score = Jaro.normalized_similarity(norm_i, norm_j) * 100
else:
score = JaroWinkler.normalized_similarity(norm_i, norm_j) * 100
if _is_variant_pair(norm_i, norm_j):
continue
if _short_label_blocked(norm_i, norm_j, score):
continue
_lo, _hi = sorted((norm_i, norm_j), key=len)
if _hi.startswith(_lo) and _hi != _lo:
continue
# Mirror pass 2: decisively-distinct pairs never reach the LLM (#1284).
if _numeric_tokens_differ(norm_i, norm_j):
continue
if _crossfile_fileanchored_blocked(node, neighbor):
continue
c1 = communities.get(node["id"])
c2 = communities.get(neighbor["id"])
if (c1 is not None and c2 is not None and c1 == c2
and min(len(norm_i), len(norm_j)) >= 12):
score += _COMMUNITY_BOOST
if low <= score < high:
ambiguous.append((node, neighbor, score))

if not ambiguous:
return
Expand Down
59 changes: 59 additions & 0 deletions tests/test_dedup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,65 @@ def test_dedup_does_not_merge_crossfile_document_headings():
assert len(result_nodes) == 2


def test_crossfile_fileanchored_fuzzy_candidates_skip_similarity_scoring(monkeypatch):
"""Cross-file file-anchored candidates are impossible merges, so the fuzzy
pass must not spend Jaro/Jaro-Winkler work on them."""
import graphify.dedup as dedup

def fail_similarity(*_args, **_kwargs):
raise AssertionError("cross-file file-anchored pair reached similarity scoring")

monkeypatch.setattr(dedup.Jaro, "normalized_similarity", fail_similarity)
monkeypatch.setattr(dedup.JaroWinkler, "normalized_similarity", fail_similarity)

boiler = ("Django app config for {}. No business logic here. "
"Domain services live in services.py and adapters in providers.")
nodes = [
{"id": "r1", "label": boiler.format("apps.platform.cards"),
"file_type": "rationale", "source_file": "apps/platform/cards/apps.py"},
{"id": "r2", "label": boiler.format("apps.platform.cores"),
"file_type": "rationale", "source_file": "apps/platform/cores/apps.py"},
{"id": "r3", "label": boiler.format("apps.platform.profiles"),
"file_type": "rationale", "source_file": "apps/platform/profiles/apps.py"},
]

result_nodes, _ = dedup.deduplicate_entities(nodes, [], communities={})

assert {node["id"] for node in result_nodes} == {"r1", "r2", "r3"}


def test_crossfile_fileanchored_llm_tiebreak_skips_similarity_scoring(monkeypatch):
"""The opt-in LLM tiebreaker must not reintroduce O(n²) scoring for
cross-file file-anchored candidates."""
import graphify.dedup as dedup
import graphify.llm as llm

monkeypatch.setattr(llm, "_get_backend_api_key", lambda _backend: "test-key")

def fail_similarity(*_args, **_kwargs):
raise AssertionError("cross-file file-anchored pair reached LLM tiebreak scoring")

monkeypatch.setattr(dedup.Jaro, "normalized_similarity", fail_similarity)
monkeypatch.setattr(dedup.JaroWinkler, "normalized_similarity", fail_similarity)

boiler = ("Django app config for {}. No business logic here. "
"Domain services live in services.py and adapters in providers.")
nodes = [
{"id": "r1", "label": boiler.format("apps.platform.cards"),
"file_type": "rationale", "source_file": "apps/platform/cards/apps.py"},
{"id": "r2", "label": boiler.format("apps.platform.cores"),
"file_type": "rationale", "source_file": "apps/platform/cores/apps.py"},
{"id": "r3", "label": boiler.format("apps.platform.profiles"),
"file_type": "rationale", "source_file": "apps/platform/profiles/apps.py"},
]

result_nodes, _ = dedup.deduplicate_entities(
nodes, [], communities={}, dedup_llm_backend="openai"
)

assert {node["id"] for node in result_nodes} == {"r1", "r2", "r3"}


def test_dedup_still_merges_samefile_rationale_duplicates():
"""The file-anchored guard only blocks cross-file pairs — near-identical
rationale duplicates within one file still merge (#1284 non-regression)."""
Expand Down