From 579e1cc7442ad088b8ac42aad810b783a18e26be Mon Sep 17 00:00:00 2001 From: Safi Date: Mon, 4 May 2026 18:51:14 +0100 Subject: [PATCH] wire --dedup-llm through build pipeline and fix fresh-extract dedup bypass Co-Authored-By: Claude Sonnet 4.6 --- README.md | 6 +- docs/dedup-architecture.html | 365 +++++++++++++++++++++++++++++++++++ graphify/__main__.py | 6 +- graphify/build.py | 16 +- 4 files changed, 385 insertions(+), 8 deletions(-) create mode 100644 docs/dedup-architecture.html diff --git a/README.md b/README.md index f8cdf9b9..f5d9ef74 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ The MCP server gives your assistant structured access: `query_graph`, `get_node` - **Code files** — processed locally via tree-sitter. Nothing leaves your machine. - **Video / audio** — transcribed locally with faster-whisper. Nothing leaves your machine. -- **Docs, PDFs, images** — sent to your AI assistant for semantic extraction (via the `/graphify` skill, using whatever model your IDE session runs). Headless `graphify extract` requires `ANTHROPIC_API_KEY` or `MOONSHOT_API_KEY`. +- **Docs, PDFs, images** — sent to your AI assistant for semantic extraction (via the `/graphify` skill, using whatever model your IDE session runs). Headless `graphify extract` requires `ANTHROPIC_API_KEY` (Claude) or `MOONSHOT_API_KEY` (Kimi). The `--dedup-llm` flag uses the same key. - No telemetry, no usage tracking, no analytics. --- @@ -274,9 +274,9 @@ graphify kiro install / uninstall graphify antigravity install / uninstall graphify extract ./docs # headless LLM extraction for CI (no IDE needed) -graphify extract ./docs --backend claude # explicit backend (auto-detected from env by default) +graphify extract ./docs --backend claude # explicit backend: claude (ANTHROPIC_API_KEY) or kimi (MOONSHOT_API_KEY) graphify extract ./docs --no-cluster # raw extraction only, skip clustering -graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs +graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs (uses same API key) graphify clone https://github.com/karpathy/nanoGPT graphify merge-graphs a.json b.json --out merged.json diff --git a/docs/dedup-architecture.html b/docs/dedup-architecture.html new file mode 100644 index 00000000..74f9dfce --- /dev/null +++ b/docs/dedup-architecture.html @@ -0,0 +1,365 @@ + + + + +Graphify Deduplication Architecture + + + + +

Graphify — Deduplication Pipeline

+

graphify/dedup.py · called from build.py before graph construction · v0.7.5

+ +
+ + +
+
Entry Points
+
+
+ build() +
graphify/build.py:119
+
Merges multiple extractions, then calls deduplicate_entities(nodes, edges, communities={}) before build_from_json()
+
Flag: dedup=True (default)
+
+
+ build_merge() +
graphify/build.py:197
+
Incremental mode: loads existing graph.json, merges new chunks, calls build() with dedup=True
+
Shrink-guard skipped when dedup is active
+
+
+ __main__.py extract +
graphify/__main__.py
+
Passes --dedup-llm flag through to enable LLM tiebreaker in Pass 3
+
Also triggers via /graphify skill
+
+
+
+ +
↓nodes: list[dict], edges: list[dict], communities: dict
+ + +
+
Pre-pass — ID Deduplication
+
+ Collapse nodes with identical id fields (last-wins). Prevents AST extractors generating "UserService" and "userservice" as separate nodes (both normalize to the same id) from confusing the union-find. O(n) dict pass. +
+
+ +
↓
+ + +
+
Pass 1 — Exact Normalization
+
+ For every node, compute _norm(label): lowercase, strip all non-alphanumeric characters, collapse whitespace. + Group nodes sharing the same norm key into union-find clusters. O(n). +

+ Examples: "HTTP Client" = "http client" = "HTTPClient" = "http_client" +
+
+ +
↓unmerged pairs only
+ + +
+
Pass 2 — Fuzzy Matching (per candidate pair via MinHash/LSH blocking)
+
+ Candidate pairs are generated by MinHash LSH (not all-pairs), then scored by Jaro-Winkler. Community membership boosts the score. +
+ +
+
+ ① Entropy Gate + _entropy(label)
+ Shannon bits/char < 2.5 → skip fuzzy
+ Short/low-info labels like "A", "get", "fn" would generate false positives at scale +
+
→
+
+ ② MinHash / LSH Blocking + 3-gram shingles (spaces stripped), 128 permutations, Jaccard threshold 0.7
+ datasketch.MinHashLSH
+ Space-stripping: "graph extractor" ≡ "graphextractor" at shingling level +
+
→
+
+ ③ Jaro-Winkler Score + JaroWinkler.normalized_similarity(a,b) × 100
+ rapidfuzz.distance.JaroWinkler
+ Merge if score ≥ 92.0 after boost +
+
→
+
+ ④ Community Boost + Same community (from clustering) → +5.0 pts
+ Entities in the same module/cluster are more likely to be the same concept +
+
→
+
+ ⑤ Union-Find Merge + _UF class, path compression
+ All connected pairs → single cluster
+ Transitivity: if A~B and B~C then A,B,C all merge +
+
+
+ +
↓ambiguous zone 75–92 pts (only with --dedup-llm)
+ + +
+
Pass 3 — LLM Tiebreaker (optional, --dedup-llm)
+
+ Pairs scoring 75.0–92.0 after community boost are batched in groups of 30 and sent to Claude for a semantic judgement call. One API call per batch. LLM-approved pairs are fed back into the union-find for merging. +

+ Disabled by default — catches cases like "Synchronous HTTP client." vs "Asynchronous HTTP client." (JW=98.6) where string similarity is high but meaning differs. Without this flag, such pairs merge at Pass 2; with it, the LLM rejects the merge. +
+
+ +
↓
+ + +
+
Remap — Winner Selection & Edge Rewiring
+
+ For each cluster of merged nodes, _pick_winner(cluster) selects the canonical id: +
1. Prefer ids without chunk suffix (_c\d+) +
2. Prefer shorter id on tie +

+ All edges are rewritten: source/target remapped to winner ids. Self-loops created by the merge are dropped. Surviving nodes list uses only winners. +
+
+ +
↓
+ + +
+
Output → build_from_json()
+
+ Returns (deduped_nodes, deduped_edges) — passed directly into build_from_json() to construct the NetworkX graph. On a 17,497-node corpus: 4,938 nodes merged (3,831 exact + 1,107 fuzzy) → 12,559 nodes in final graph. +
+
+ +
+ + +

Thresholds & Constants

+
+
_ENTROPY_THRESHOLD = 2.5 bits/char — below this, skip fuzzy (short/generic labels)
+
_LSH_THRESHOLD = 0.7 Jaccard — MinHash blocking gate
+
_MERGE_THRESHOLD = 92.0 JW — auto-merge above this score
+
_COMMUNITY_BOOST = +5.0 pts — same community bonus
+
LLM tiebreak zone = 75.0–92.0 JW (only with --dedup-llm)
+
MinHash permutations = 128, shingle size = 3-gram (spaces stripped)
+
+ + + diff --git a/graphify/__main__.py b/graphify/__main__.py index 5e30492b..33d3681a 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -2126,22 +2126,24 @@ def main() -> None: # Build graph + cluster + score + write. from graphify.build import ( + build as _build, build_from_json as _build_from_json, build_merge as _build_merge, ) from graphify.cluster import cluster as _cluster, score_all as _score_all from graphify.export import to_json as _to_json from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising - + dedup_backend = backend if dedup_llm else None if incremental_mode: G = _build_merge( [merged], graph_path=existing_graph_path, prune_sources=deleted_files or None, dedup=True, + dedup_llm_backend=dedup_backend, ) else: - G = _build_from_json(merged) + G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend) if G.number_of_nodes() == 0: print( "[graphify extract] graph is empty — extraction produced no nodes. " diff --git a/graphify/build.py b/graphify/build.py index 2aa93f9d..76a83578 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -116,12 +116,20 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: return G -def build(extractions: list[dict], *, directed: bool = False, dedup: bool = True) -> nx.Graph: +def build( + extractions: list[dict], + *, + directed: bool = False, + dedup: bool = True, + dedup_llm_backend: str | None = None, +) -> nx.Graph: """Merge multiple extraction results into one graph. directed=True produces a DiGraph that preserves edge direction (source→target). directed=False (default) produces an undirected Graph for backward compatibility. dedup=True (default) runs entity deduplication before building the graph. + dedup_llm_backend: if set (e.g. "claude" or "kimi"), uses LLM to resolve + ambiguous pairs in the 75–92 Jaro-Winkler score zone. Extractions are merged in order. For nodes with the same ID, the last extraction's attributes win (NetworkX add_node overwrites). Pass AST @@ -138,7 +146,8 @@ def build(extractions: list[dict], *, directed: bool = False, dedup: bool = True combined["output_tokens"] += ext.get("output_tokens", 0) if dedup and combined["nodes"]: combined["nodes"], combined["edges"] = deduplicate_entities( - combined["nodes"], combined["edges"], communities={} + combined["nodes"], combined["edges"], communities={}, + dedup_llm_backend=dedup_llm_backend, ) return build_from_json(combined, directed=directed) @@ -201,6 +210,7 @@ def build_merge( *, directed: bool = False, dedup: bool = True, + dedup_llm_backend: str | None = None, ) -> nx.Graph: """Load existing graph.json, merge new chunks into it, and save back. @@ -226,7 +236,7 @@ def build_merge( base = [] all_chunks = base + list(new_chunks) - G = build(all_chunks, directed=directed, dedup=dedup) + G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend) # Prune nodes from deleted source files if prune_sources: