From 60afcf8fda401d9f5d41cddaefb5ae487dff658e Mon Sep 17 00:00:00 2001 From: Gautam Manchandani Date: Mon, 13 Jul 2026 00:51:02 +0530 Subject: [PATCH] fix: skip impossible file-anchored dedup scoring --- graphify/dedup.py | 281 ++++++++++++++++++++++++++------------------ tests/test_dedup.py | 59 ++++++++++ 2 files changed, 224 insertions(+), 116 deletions(-) diff --git a/graphify/dedup.py b/graphify/dedup.py index e0ddd2e3b..70d27f39e 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -121,6 +121,10 @@ def _numeric_tokens_differ(a: str, b: str) -> bool: _FILE_ANCHORED_NONCODE = frozenset({"rationale", "document"}) +def _is_file_anchored_noncode(node: dict) -> bool: + return node.get("file_type") in _FILE_ANCHORED_NONCODE + + def _crossfile_fileanchored_blocked(node: dict, neighbor: dict) -> bool: """Block label-based merging of file-anchored non-code nodes across files (#1284). @@ -136,6 +140,36 @@ def _crossfile_fileanchored_blocked(node: dict, neighbor: dict) -> bool: return (node.get("source_file") or "") != (neighbor.get("source_file") or "") +def _fuzzy_candidate_groups(candidates: list[dict]) -> list[list[dict]]: + """Group candidates so impossible file-anchored cross-file pairs never enter LSH. + + `rationale` and `document` nodes can merge only with nodes from the same + source_file. Concepts and other non-file-anchored candidates keep the old + global fuzzy-dedup behavior. + """ + groups: list[list[dict]] = [] + global_group: list[dict] = [] + by_source: dict[str, list[dict]] = defaultdict(list) + anchored_sources: set[str] = set() + + for node in candidates: + source_file = node.get("source_file") or "" + by_source[source_file].append(node) + if _is_file_anchored_noncode(node): + anchored_sources.add(source_file) + else: + global_group.append(node) + + if len(global_group) >= 2: + groups.append(global_group) + + for source_file, source_group in by_source.items(): + if source_file in anchored_sources and len(source_group) >= 2: + groups.append(source_group) + + return groups + + # ── union-find ──────────────────────────────────────────────────────────────── class _UF: @@ -299,99 +333,108 @@ def deduplicate_entities( fuzzy_merges = 0 if len(candidates) >= 2: - lsh = MinHashLSH(threshold=_LSH_THRESHOLD, num_perm=_NUM_PERM) - minhashes: dict[str, MinHash] = {} - # Pre-build O(1) lookup structures so the query loop below doesn't scan - # the candidates list linearly for every LSH neighbor (was O(n²×B)). - candidates_by_id: dict[str, dict] = {} norm_cache: dict[str, str] = {} - + minhashes: dict[str, MinHash] = {} for node in candidates: node_id = node["id"] - candidates_by_id[node_id] = node nl = _norm(node.get("label", node.get("id", ""))) norm_cache[node_id] = nl m = _make_minhash(nl) minhashes[node_id] = m - try: - lsh.insert(node_id, m) - except ValueError: - pass # duplicate key in LSH — already inserted - - for node in candidates: - node_id = node["id"] - norm_label = norm_cache[node_id] - neighbors = lsh.query(minhashes[node_id]) - - for neighbor_id in neighbors: - if neighbor_id == node_id: - continue - if uf.find(node_id) == uf.find(neighbor_id): - continue - - neighbor = candidates_by_id.get(neighbor_id) - if neighbor is None: - continue - neighbor_norm = norm_cache.get(neighbor_id) or _norm(neighbor.get("label", neighbor.get("id", ""))) - # Cross-file long labels score on plain Jaro (no prefix bonus). - # Jaro-Winkler's leading-prefix bonus lifts pairs that share a - # prefix but diverge in a distinguishing token ("testing-library - # jest-native" vs "react-native") past threshold, fabricating - # destructive cross-file merges; on Jaro alone they fall short - # while true cross-file duplicates still clear it (#1243). Same-file - # near-duplicates keep Jaro-Winkler (low-risk, and a mid-string - # stopword insertion needs the prefix bonus to merge); short labels - # keep Jaro-Winkler too (gated by _short_label_blocked). - _xfile = (node.get("source_file") or "") != (neighbor.get("source_file") or "") - if _xfile and max(len(norm_label), len(neighbor_norm)) >= 12: - score = Jaro.normalized_similarity(norm_label, neighbor_norm) * 100 - else: - score = JaroWinkler.normalized_similarity(norm_label, neighbor_norm) * 100 + seen_pairs: set[tuple[str, str]] = set() + for group in _fuzzy_candidate_groups(candidates): + lsh = MinHashLSH(threshold=_LSH_THRESHOLD, num_perm=_NUM_PERM) + # Pre-build O(1) lookup structures so the query loop below doesn't scan + # the candidates list linearly for every LSH neighbor (was O(n²×B)). + candidates_by_id: dict[str, dict] = {} - if _is_variant_pair(norm_label, neighbor_norm): - continue - if _short_label_blocked(norm_label, neighbor_norm, score): - continue - # Prefix-extension pairs (getActiveSession / getActiveSessions, - # parseConfig / parseConfigFile) are almost never duplicates — - # one is a strict suffix-extension of the other. Block the merge - # regardless of JW score (#1201). - _lo, _hi = sorted((norm_label, neighbor_norm), key=len) - if _hi.startswith(_lo) and _hi != _lo: - continue - # Numbered/versioned siblings and cross-file file-anchored - # boilerplate (rationale/document) are decisively distinct - # regardless of score (#1284). - if _numeric_tokens_differ(norm_label, neighbor_norm): - continue - if _crossfile_fileanchored_blocked(node, neighbor): - continue - - c1 = communities.get(node_id) - c2 = communities.get(neighbor_id) - if (c1 is not None and c2 is not None and c1 == c2 - and min(len(norm_label), len(neighbor_norm)) >= 12): - score += _COMMUNITY_BOOST - - if score >= _MERGE_THRESHOLD: - # Identical labels across different source files almost always - # means same-named-but-different symbols (trait impls, wrapper - # methods, common type names). Mirror Pass 1's source_file - # partition for this sub-case. (#1046, leaks #895's fix) - if norm_label == neighbor_norm: - sf_a = node.get("source_file") or "" - sf_b = neighbor.get("source_file") or "" - if sf_a != sf_b: - continue - # Pick the winner from the verified pair only. Selecting it - # from the union of both normalized-label groups pulls - # never-compared nodes (same label, different source_file) - # into the merge, bypassing the #1046/#1178 guards. - winner = _pick_winner([node, neighbor]) - uf.union(winner["id"], node_id) - uf.union(winner["id"], neighbor_id) - fuzzy_merges += 1 + for node in group: + node_id = node["id"] + candidates_by_id[node_id] = node + try: + lsh.insert(node_id, minhashes[node_id]) + except ValueError: + pass # duplicate key in LSH — already inserted + + for node in group: + node_id = node["id"] + norm_label = norm_cache[node_id] + neighbors = lsh.query(minhashes[node_id]) + + for neighbor_id in neighbors: + if neighbor_id == node_id: + continue + pair_key = tuple(sorted((node_id, neighbor_id))) + if pair_key in seen_pairs: + continue + seen_pairs.add(pair_key) + if uf.find(node_id) == uf.find(neighbor_id): + continue + + neighbor = candidates_by_id.get(neighbor_id) + if neighbor is None: + continue + + neighbor_norm = norm_cache.get(neighbor_id) or _norm(neighbor.get("label", neighbor.get("id", ""))) + # Cross-file long labels score on plain Jaro (no prefix bonus). + # Jaro-Winkler's leading-prefix bonus lifts pairs that share a + # prefix but diverge in a distinguishing token ("testing-library + # jest-native" vs "react-native") past threshold, fabricating + # destructive cross-file merges; on Jaro alone they fall short + # while true cross-file duplicates still clear it (#1243). Same-file + # near-duplicates keep Jaro-Winkler (low-risk, and a mid-string + # stopword insertion needs the prefix bonus to merge); short labels + # keep Jaro-Winkler too (gated by _short_label_blocked). + _xfile = (node.get("source_file") or "") != (neighbor.get("source_file") or "") + if _xfile and max(len(norm_label), len(neighbor_norm)) >= 12: + score = Jaro.normalized_similarity(norm_label, neighbor_norm) * 100 + else: + score = JaroWinkler.normalized_similarity(norm_label, neighbor_norm) * 100 + + if _is_variant_pair(norm_label, neighbor_norm): + continue + if _short_label_blocked(norm_label, neighbor_norm, score): + continue + # Prefix-extension pairs (getActiveSession / getActiveSessions, + # parseConfig / parseConfigFile) are almost never duplicates — + # one is a strict suffix-extension of the other. Block the merge + # regardless of JW score (#1201). + _lo, _hi = sorted((norm_label, neighbor_norm), key=len) + if _hi.startswith(_lo) and _hi != _lo: + continue + # Numbered/versioned siblings and cross-file file-anchored + # boilerplate (rationale/document) are decisively distinct + # regardless of score (#1284). + if _numeric_tokens_differ(norm_label, neighbor_norm): + continue + if _crossfile_fileanchored_blocked(node, neighbor): + continue + + c1 = communities.get(node_id) + c2 = communities.get(neighbor_id) + if (c1 is not None and c2 is not None and c1 == c2 + and min(len(norm_label), len(neighbor_norm)) >= 12): + score += _COMMUNITY_BOOST + + if score >= _MERGE_THRESHOLD: + # Identical labels across different source files almost always + # means same-named-but-different symbols (trait impls, wrapper + # methods, common type names). Mirror Pass 1's source_file + # partition for this sub-case. (#1046, leaks #895's fix) + if norm_label == neighbor_norm: + sf_a = node.get("source_file") or "" + sf_b = neighbor.get("source_file") or "" + if sf_a != sf_b: + continue + # Pick the winner from the verified pair only. Selecting it + # from the union of both normalized-label groups pulls + # never-compared nodes (same label, different source_file) + # into the merge, bypassing the #1046/#1178 guards. + winner = _pick_winner([node, neighbor]) + uf.union(winner["id"], node_id) + uf.union(winner["id"], neighbor_id) + fuzzy_merges += 1 # ── pass 3: LLM tiebreaker for ambiguous pairs (opt-in) ────────────────── if dedup_llm_backend is not None: @@ -484,38 +527,44 @@ def _llm_tiebreak( return ambiguous: list[tuple[dict, dict, float]] = [] - for i, node in enumerate(candidates): - norm_i = _norm(node.get("label", node.get("id", ""))) - for j in range(i + 1, len(candidates)): - neighbor = candidates[j] - if uf.find(node["id"]) == uf.find(neighbor["id"]): - continue - norm_j = _norm(neighbor.get("label", neighbor.get("id", ""))) - # Mirror pass 2: plain Jaro for cross-file long labels (#1243). - _xfile = (node.get("source_file") or "") != (neighbor.get("source_file") or "") - if _xfile and max(len(norm_i), len(norm_j)) >= 12: - score = Jaro.normalized_similarity(norm_i, norm_j) * 100 - else: - score = JaroWinkler.normalized_similarity(norm_i, norm_j) * 100 - if _is_variant_pair(norm_i, norm_j): - continue - if _short_label_blocked(norm_i, norm_j, score): - continue - _lo, _hi = sorted((norm_i, norm_j), key=len) - if _hi.startswith(_lo) and _hi != _lo: - continue - # Mirror pass 2: decisively-distinct pairs never reach the LLM (#1284). - if _numeric_tokens_differ(norm_i, norm_j): - continue - if _crossfile_fileanchored_blocked(node, neighbor): - continue - c1 = communities.get(node["id"]) - c2 = communities.get(neighbor["id"]) - if (c1 is not None and c2 is not None and c1 == c2 - and min(len(norm_i), len(norm_j)) >= 12): - score += _COMMUNITY_BOOST - if low <= score < high: - ambiguous.append((node, neighbor, score)) + seen_pairs: set[tuple[str, str]] = set() + for group in _fuzzy_candidate_groups(candidates): + for i, node in enumerate(group): + norm_i = _norm(node.get("label", node.get("id", ""))) + for j in range(i + 1, len(group)): + neighbor = group[j] + pair_key = tuple(sorted((node["id"], neighbor["id"]))) + if pair_key in seen_pairs: + continue + seen_pairs.add(pair_key) + if uf.find(node["id"]) == uf.find(neighbor["id"]): + continue + norm_j = _norm(neighbor.get("label", neighbor.get("id", ""))) + # Mirror pass 2: plain Jaro for cross-file long labels (#1243). + _xfile = (node.get("source_file") or "") != (neighbor.get("source_file") or "") + if _xfile and max(len(norm_i), len(norm_j)) >= 12: + score = Jaro.normalized_similarity(norm_i, norm_j) * 100 + else: + score = JaroWinkler.normalized_similarity(norm_i, norm_j) * 100 + if _is_variant_pair(norm_i, norm_j): + continue + if _short_label_blocked(norm_i, norm_j, score): + continue + _lo, _hi = sorted((norm_i, norm_j), key=len) + if _hi.startswith(_lo) and _hi != _lo: + continue + # Mirror pass 2: decisively-distinct pairs never reach the LLM (#1284). + if _numeric_tokens_differ(norm_i, norm_j): + continue + if _crossfile_fileanchored_blocked(node, neighbor): + continue + c1 = communities.get(node["id"]) + c2 = communities.get(neighbor["id"]) + if (c1 is not None and c2 is not None and c1 == c2 + and min(len(norm_i), len(norm_j)) >= 12): + score += _COMMUNITY_BOOST + if low <= score < high: + ambiguous.append((node, neighbor, score)) if not ambiguous: return diff --git a/tests/test_dedup.py b/tests/test_dedup.py index 267388e57..58dd2fad6 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -325,6 +325,65 @@ def test_dedup_does_not_merge_crossfile_document_headings(): assert len(result_nodes) == 2 +def test_crossfile_fileanchored_fuzzy_candidates_skip_similarity_scoring(monkeypatch): + """Cross-file file-anchored candidates are impossible merges, so the fuzzy + pass must not spend Jaro/Jaro-Winkler work on them.""" + import graphify.dedup as dedup + + def fail_similarity(*_args, **_kwargs): + raise AssertionError("cross-file file-anchored pair reached similarity scoring") + + monkeypatch.setattr(dedup.Jaro, "normalized_similarity", fail_similarity) + monkeypatch.setattr(dedup.JaroWinkler, "normalized_similarity", fail_similarity) + + boiler = ("Django app config for {}. No business logic here. " + "Domain services live in services.py and adapters in providers.") + nodes = [ + {"id": "r1", "label": boiler.format("apps.platform.cards"), + "file_type": "rationale", "source_file": "apps/platform/cards/apps.py"}, + {"id": "r2", "label": boiler.format("apps.platform.cores"), + "file_type": "rationale", "source_file": "apps/platform/cores/apps.py"}, + {"id": "r3", "label": boiler.format("apps.platform.profiles"), + "file_type": "rationale", "source_file": "apps/platform/profiles/apps.py"}, + ] + + result_nodes, _ = dedup.deduplicate_entities(nodes, [], communities={}) + + assert {node["id"] for node in result_nodes} == {"r1", "r2", "r3"} + + +def test_crossfile_fileanchored_llm_tiebreak_skips_similarity_scoring(monkeypatch): + """The opt-in LLM tiebreaker must not reintroduce O(n²) scoring for + cross-file file-anchored candidates.""" + import graphify.dedup as dedup + import graphify.llm as llm + + monkeypatch.setattr(llm, "_get_backend_api_key", lambda _backend: "test-key") + + def fail_similarity(*_args, **_kwargs): + raise AssertionError("cross-file file-anchored pair reached LLM tiebreak scoring") + + monkeypatch.setattr(dedup.Jaro, "normalized_similarity", fail_similarity) + monkeypatch.setattr(dedup.JaroWinkler, "normalized_similarity", fail_similarity) + + boiler = ("Django app config for {}. No business logic here. " + "Domain services live in services.py and adapters in providers.") + nodes = [ + {"id": "r1", "label": boiler.format("apps.platform.cards"), + "file_type": "rationale", "source_file": "apps/platform/cards/apps.py"}, + {"id": "r2", "label": boiler.format("apps.platform.cores"), + "file_type": "rationale", "source_file": "apps/platform/cores/apps.py"}, + {"id": "r3", "label": boiler.format("apps.platform.profiles"), + "file_type": "rationale", "source_file": "apps/platform/profiles/apps.py"}, + ] + + result_nodes, _ = dedup.deduplicate_entities( + nodes, [], communities={}, dedup_llm_backend="openai" + ) + + assert {node["id"] for node in result_nodes} == {"r1", "r2", "r3"} + + def test_dedup_still_merges_samefile_rationale_duplicates(): """The file-anchored guard only blocks cross-file pairs — near-identical rationale duplicates within one file still merge (#1284 non-regression)."""