mirror of
https://github.com/safishamsi/graphify.git
synced 2026-09-22 05:25:40 +00:00
wire --dedup-llm through build pipeline and fix fresh-extract dedup bypass
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Safi
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
913feca6a6
commit
579e1cc744
@@ -221,7 +221,7 @@ The MCP server gives your assistant structured access: `query_graph`, `get_node`
|
||||
|
||||
- **Code files** — processed locally via tree-sitter. Nothing leaves your machine.
|
||||
- **Video / audio** — transcribed locally with faster-whisper. Nothing leaves your machine.
|
||||
- **Docs, PDFs, images** — sent to your AI assistant for semantic extraction (via the `/graphify` skill, using whatever model your IDE session runs). Headless `graphify extract` requires `ANTHROPIC_API_KEY` or `MOONSHOT_API_KEY`.
|
||||
- **Docs, PDFs, images** — sent to your AI assistant for semantic extraction (via the `/graphify` skill, using whatever model your IDE session runs). Headless `graphify extract` requires `ANTHROPIC_API_KEY` (Claude) or `MOONSHOT_API_KEY` (Kimi). The `--dedup-llm` flag uses the same key.
|
||||
- No telemetry, no usage tracking, no analytics.
|
||||
|
||||
---
|
||||
@@ -274,9 +274,9 @@ graphify kiro install / uninstall
|
||||
graphify antigravity install / uninstall
|
||||
|
||||
graphify extract ./docs # headless LLM extraction for CI (no IDE needed)
|
||||
graphify extract ./docs --backend claude # explicit backend (auto-detected from env by default)
|
||||
graphify extract ./docs --backend claude # explicit backend: claude (ANTHROPIC_API_KEY) or kimi (MOONSHOT_API_KEY)
|
||||
graphify extract ./docs --no-cluster # raw extraction only, skip clustering
|
||||
graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs
|
||||
graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs (uses same API key)
|
||||
|
||||
graphify clone https://github.com/karpathy/nanoGPT
|
||||
graphify merge-graphs a.json b.json --out merged.json
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Graphify Deduplication Architecture</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body {
|
||||
font-family: 'SF Mono', 'Fira Code', monospace;
|
||||
background: #0d1117;
|
||||
color: #e6edf3;
|
||||
padding: 40px 20px;
|
||||
min-height: 100vh;
|
||||
}
|
||||
h1 {
|
||||
text-align: center;
|
||||
font-size: 22px;
|
||||
font-weight: 600;
|
||||
color: #58a6ff;
|
||||
margin-bottom: 6px;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
.subtitle {
|
||||
text-align: center;
|
||||
font-size: 13px;
|
||||
color: #8b949e;
|
||||
margin-bottom: 40px;
|
||||
}
|
||||
.diagram {
|
||||
max-width: 900px;
|
||||
margin: 0 auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
/* ── generic block ── */
|
||||
.block {
|
||||
border-radius: 8px;
|
||||
padding: 14px 20px;
|
||||
position: relative;
|
||||
}
|
||||
.block-title {
|
||||
font-size: 12px;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 1px;
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
.block-body {
|
||||
font-size: 13px;
|
||||
line-height: 1.7;
|
||||
color: #c9d1d9;
|
||||
}
|
||||
.block-body code {
|
||||
background: rgba(255,255,255,0.08);
|
||||
padding: 1px 5px;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
color: #ffa657;
|
||||
}
|
||||
.block-body .dim { color: #8b949e; }
|
||||
|
||||
/* ── colours ── */
|
||||
.entry { background: #161b22; border: 1px solid #30363d; }
|
||||
.pass0 { background: #0d2840; border: 1px solid #1f6feb; }
|
||||
.pass1 { background: #1a1f2e; border: 1px solid #3d5a80; }
|
||||
.pass2 { background: #1a2d1a; border: 1px solid #3fb950; }
|
||||
.pass2b { background: #1a2d1a; border: 1px dashed #3fb950; }
|
||||
.pass3 { background: #2d1f1a; border: 1px dashed #f0883e; }
|
||||
.output { background: #1e1a2d; border: 1px solid #8957e5; }
|
||||
.callers { background: #161b22; border: 1px solid #30363d; }
|
||||
|
||||
/* ── arrows ── */
|
||||
.arrow {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
height: 28px;
|
||||
color: #8b949e;
|
||||
font-size: 20px;
|
||||
position: relative;
|
||||
}
|
||||
.arrow-label {
|
||||
position: absolute;
|
||||
right: 60px;
|
||||
font-size: 11px;
|
||||
color: #6e7681;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
/* ── pass 2 sub-steps ── */
|
||||
.subflow {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
gap: 0;
|
||||
margin: 4px 0 0 0;
|
||||
align-items: stretch;
|
||||
}
|
||||
.sub-arrow {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
color: #3fb950;
|
||||
font-size: 18px;
|
||||
padding: 0 6px;
|
||||
}
|
||||
.substep {
|
||||
background: rgba(63,185,80,0.08);
|
||||
border: 1px solid #238636;
|
||||
border-radius: 6px;
|
||||
padding: 8px 12px;
|
||||
font-size: 12px;
|
||||
flex: 1;
|
||||
line-height: 1.5;
|
||||
color: #c9d1d9;
|
||||
}
|
||||
.substep strong {
|
||||
display: block;
|
||||
color: #3fb950;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.8px;
|
||||
margin-bottom: 3px;
|
||||
}
|
||||
.substep code {
|
||||
background: rgba(255,255,255,0.07);
|
||||
padding: 1px 4px;
|
||||
border-radius: 3px;
|
||||
font-size: 11px;
|
||||
color: #ffa657;
|
||||
}
|
||||
.substep .note {
|
||||
color: #6e7681;
|
||||
font-size: 11px;
|
||||
margin-top: 3px;
|
||||
}
|
||||
|
||||
/* ── bypass branch ── */
|
||||
.bypass {
|
||||
background: rgba(240,136,62,0.08);
|
||||
border: 1px dashed #f0883e;
|
||||
border-radius: 6px;
|
||||
padding: 8px 14px;
|
||||
font-size: 12px;
|
||||
color: #c9d1d9;
|
||||
margin-top: 4px;
|
||||
}
|
||||
.bypass strong { color: #f0883e; }
|
||||
|
||||
/* ── callers row ── */
|
||||
.callers-row {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
.caller-box {
|
||||
flex: 1;
|
||||
background: #21262d;
|
||||
border: 1px solid #30363d;
|
||||
border-radius: 6px;
|
||||
padding: 10px 14px;
|
||||
font-size: 12px;
|
||||
color: #c9d1d9;
|
||||
}
|
||||
.caller-box strong {
|
||||
display: block;
|
||||
color: #58a6ff;
|
||||
margin-bottom: 4px;
|
||||
font-size: 12px;
|
||||
}
|
||||
.caller-box code {
|
||||
background: rgba(255,255,255,0.07);
|
||||
padding: 1px 4px;
|
||||
border-radius: 3px;
|
||||
font-size: 11px;
|
||||
color: #ffa657;
|
||||
}
|
||||
.caller-box .path {
|
||||
color: #8b949e;
|
||||
font-size: 11px;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
/* ── threshold legend ── */
|
||||
.legend {
|
||||
max-width: 900px;
|
||||
margin: 28px auto 0;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
}
|
||||
.legend-item {
|
||||
background: #161b22;
|
||||
border: 1px solid #30363d;
|
||||
border-radius: 6px;
|
||||
padding: 8px 14px;
|
||||
font-size: 12px;
|
||||
color: #c9d1d9;
|
||||
}
|
||||
.legend-item span {
|
||||
font-weight: 700;
|
||||
color: #58a6ff;
|
||||
}
|
||||
.legend-title {
|
||||
max-width: 900px;
|
||||
margin: 20px auto 8px;
|
||||
font-size: 12px;
|
||||
color: #8b949e;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 1px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<h1>Graphify — Deduplication Pipeline</h1>
|
||||
<p class="subtitle">graphify/dedup.py · called from build.py before graph construction · v0.7.5</p>
|
||||
|
||||
<div class="diagram">
|
||||
|
||||
<!-- CALLERS -->
|
||||
<div class="block callers">
|
||||
<div class="block-title" style="color:#8b949e">Entry Points</div>
|
||||
<div class="callers-row">
|
||||
<div class="caller-box">
|
||||
<strong>build()</strong>
|
||||
<div class="path">graphify/build.py:119</div>
|
||||
<div>Merges multiple extractions, then calls <code>deduplicate_entities(nodes, edges, communities={})</code> before <code>build_from_json()</code></div>
|
||||
<div style="margin-top:5px;color:#6e7681;font-size:11px">Flag: <code>dedup=True</code> (default)</div>
|
||||
</div>
|
||||
<div class="caller-box">
|
||||
<strong>build_merge()</strong>
|
||||
<div class="path">graphify/build.py:197</div>
|
||||
<div>Incremental mode: loads existing <code>graph.json</code>, merges new chunks, calls <code>build()</code> with <code>dedup=True</code></div>
|
||||
<div style="margin-top:5px;color:#6e7681;font-size:11px">Shrink-guard skipped when dedup is active</div>
|
||||
</div>
|
||||
<div class="caller-box">
|
||||
<strong>__main__.py extract</strong>
|
||||
<div class="path">graphify/__main__.py</div>
|
||||
<div>Passes <code>--dedup-llm</code> flag through to enable LLM tiebreaker in Pass 3</div>
|
||||
<div style="margin-top:5px;color:#6e7681;font-size:11px">Also triggers via <code>/graphify</code> skill</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="arrow">↓<span class="arrow-label">nodes: list[dict], edges: list[dict], communities: dict</span></div>
|
||||
|
||||
<!-- PASS 0 -->
|
||||
<div class="block pass0">
|
||||
<div class="block-title" style="color:#58a6ff">Pre-pass — ID Deduplication</div>
|
||||
<div class="block-body">
|
||||
Collapse nodes with identical <code>id</code> fields (last-wins). Prevents AST extractors generating <code>"UserService"</code> and <code>"userservice"</code> as separate nodes (both normalize to the same id) from confusing the union-find. <span class="dim">O(n) dict pass.</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="arrow">↓</div>
|
||||
|
||||
<!-- PASS 1 -->
|
||||
<div class="block pass1">
|
||||
<div class="block-title" style="color:#79c0ff">Pass 1 — Exact Normalization</div>
|
||||
<div class="block-body">
|
||||
For every node, compute <code>_norm(label)</code>: lowercase, strip all non-alphanumeric characters, collapse whitespace.
|
||||
Group nodes sharing the same norm key into union-find clusters. <span class="dim">O(n).</span>
|
||||
<br><br>
|
||||
Examples: <code>"HTTP Client"</code> = <code>"http client"</code> = <code>"HTTPClient"</code> = <code>"http_client"</code>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="arrow">↓<span class="arrow-label">unmerged pairs only</span></div>
|
||||
|
||||
<!-- PASS 2 header -->
|
||||
<div class="block pass2">
|
||||
<div class="block-title" style="color:#3fb950">Pass 2 — Fuzzy Matching (per candidate pair via MinHash/LSH blocking)</div>
|
||||
<div class="block-body" style="margin-bottom:10px">
|
||||
Candidate pairs are generated by MinHash LSH (not all-pairs), then scored by Jaro-Winkler. Community membership boosts the score.
|
||||
</div>
|
||||
|
||||
<div class="subflow">
|
||||
<div class="substep">
|
||||
<strong>① Entropy Gate</strong>
|
||||
<code>_entropy(label)</code><br>
|
||||
Shannon bits/char < <code>2.5</code> → skip fuzzy<br>
|
||||
<span class="note">Short/low-info labels like "A", "get", "fn" would generate false positives at scale</span>
|
||||
</div>
|
||||
<div class="sub-arrow">→</div>
|
||||
<div class="substep">
|
||||
<strong>② MinHash / LSH Blocking</strong>
|
||||
3-gram shingles (spaces stripped), 128 permutations, Jaccard threshold <code>0.7</code><br>
|
||||
<code>datasketch.MinHashLSH</code><br>
|
||||
<span class="note">Space-stripping: "graph extractor" ≡ "graphextractor" at shingling level</span>
|
||||
</div>
|
||||
<div class="sub-arrow">→</div>
|
||||
<div class="substep">
|
||||
<strong>③ Jaro-Winkler Score</strong>
|
||||
<code>JaroWinkler.normalized_similarity(a,b) × 100</code><br>
|
||||
<code>rapidfuzz.distance.JaroWinkler</code><br>
|
||||
Merge if score ≥ <code>92.0</code> after boost
|
||||
</div>
|
||||
<div class="sub-arrow">→</div>
|
||||
<div class="substep">
|
||||
<strong>④ Community Boost</strong>
|
||||
Same community (from clustering) → +<code>5.0</code> pts<br>
|
||||
<span class="note">Entities in the same module/cluster are more likely to be the same concept</span>
|
||||
</div>
|
||||
<div class="sub-arrow">→</div>
|
||||
<div class="substep">
|
||||
<strong>⑤ Union-Find Merge</strong>
|
||||
<code>_UF</code> class, path compression<br>
|
||||
All connected pairs → single cluster<br>
|
||||
<span class="note">Transitivity: if A~B and B~C then A,B,C all merge</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="arrow">↓<span class="arrow-label">ambiguous zone 75–92 pts (only with --dedup-llm)</span></div>
|
||||
|
||||
<!-- PASS 3 -->
|
||||
<div class="block pass3">
|
||||
<div class="block-title" style="color:#f0883e">Pass 3 — LLM Tiebreaker (optional, <code>--dedup-llm</code>)</div>
|
||||
<div class="block-body">
|
||||
Pairs scoring <code>75.0–92.0</code> after community boost are batched in groups of <code>30</code> and sent to Claude for a semantic judgement call. One API call per batch. LLM-approved pairs are fed back into the union-find for merging.
|
||||
<br><br>
|
||||
<span class="dim">Disabled by default — catches cases like "Synchronous HTTP client." vs "Asynchronous HTTP client." (JW=98.6) where string similarity is high but meaning differs. Without this flag, such pairs merge at Pass 2; with it, the LLM rejects the merge.</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="arrow">↓</div>
|
||||
|
||||
<!-- REMAP -->
|
||||
<div class="block pass2b">
|
||||
<div class="block-title" style="color:#3fb950">Remap — Winner Selection & Edge Rewiring</div>
|
||||
<div class="block-body">
|
||||
For each cluster of merged nodes, <code>_pick_winner(cluster)</code> selects the canonical id:
|
||||
<br>1. Prefer ids <em>without</em> chunk suffix (<code>_c\d+</code>)
|
||||
<br>2. Prefer shorter id on tie
|
||||
<br><br>
|
||||
All edges are rewritten: <code>source/target</code> remapped to winner ids. Self-loops created by the merge are dropped. Surviving nodes list uses only winners.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="arrow">↓</div>
|
||||
|
||||
<!-- OUTPUT -->
|
||||
<div class="block output">
|
||||
<div class="block-title" style="color:#8957e5">Output → build_from_json()</div>
|
||||
<div class="block-body">
|
||||
Returns <code>(deduped_nodes, deduped_edges)</code> — passed directly into <code>build_from_json()</code> to construct the NetworkX graph. On a 17,497-node corpus: <strong>4,938 nodes merged</strong> (3,831 exact + 1,107 fuzzy) → <strong>12,559 nodes</strong> in final graph.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<!-- LEGEND -->
|
||||
<p class="legend-title">Thresholds & Constants</p>
|
||||
<div class="legend">
|
||||
<div class="legend-item"><span>_ENTROPY_THRESHOLD</span> = 2.5 bits/char — below this, skip fuzzy (short/generic labels)</div>
|
||||
<div class="legend-item"><span>_LSH_THRESHOLD</span> = 0.7 Jaccard — MinHash blocking gate</div>
|
||||
<div class="legend-item"><span>_MERGE_THRESHOLD</span> = 92.0 JW — auto-merge above this score</div>
|
||||
<div class="legend-item"><span>_COMMUNITY_BOOST</span> = +5.0 pts — same community bonus</div>
|
||||
<div class="legend-item"><span>LLM tiebreak zone</span> = 75.0–92.0 JW (only with <code>--dedup-llm</code>)</div>
|
||||
<div class="legend-item"><span>MinHash permutations</span> = 128, shingle size = 3-gram (spaces stripped)</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -2126,22 +2126,24 @@ def main() -> None:
|
||||
|
||||
# Build graph + cluster + score + write.
|
||||
from graphify.build import (
|
||||
build as _build,
|
||||
build_from_json as _build_from_json,
|
||||
build_merge as _build_merge,
|
||||
)
|
||||
from graphify.cluster import cluster as _cluster, score_all as _score_all
|
||||
from graphify.export import to_json as _to_json
|
||||
from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising
|
||||
|
||||
dedup_backend = backend if dedup_llm else None
|
||||
if incremental_mode:
|
||||
G = _build_merge(
|
||||
[merged],
|
||||
graph_path=existing_graph_path,
|
||||
prune_sources=deleted_files or None,
|
||||
dedup=True,
|
||||
dedup_llm_backend=dedup_backend,
|
||||
)
|
||||
else:
|
||||
G = _build_from_json(merged)
|
||||
G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend)
|
||||
if G.number_of_nodes() == 0:
|
||||
print(
|
||||
"[graphify extract] graph is empty — extraction produced no nodes. "
|
||||
|
||||
+13
-3
@@ -116,12 +116,20 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph:
|
||||
return G
|
||||
|
||||
|
||||
def build(extractions: list[dict], *, directed: bool = False, dedup: bool = True) -> nx.Graph:
|
||||
def build(
|
||||
extractions: list[dict],
|
||||
*,
|
||||
directed: bool = False,
|
||||
dedup: bool = True,
|
||||
dedup_llm_backend: str | None = None,
|
||||
) -> nx.Graph:
|
||||
"""Merge multiple extraction results into one graph.
|
||||
|
||||
directed=True produces a DiGraph that preserves edge direction (source→target).
|
||||
directed=False (default) produces an undirected Graph for backward compatibility.
|
||||
dedup=True (default) runs entity deduplication before building the graph.
|
||||
dedup_llm_backend: if set (e.g. "claude" or "kimi"), uses LLM to resolve
|
||||
ambiguous pairs in the 75–92 Jaro-Winkler score zone.
|
||||
|
||||
Extractions are merged in order. For nodes with the same ID, the last
|
||||
extraction's attributes win (NetworkX add_node overwrites). Pass AST
|
||||
@@ -138,7 +146,8 @@ def build(extractions: list[dict], *, directed: bool = False, dedup: bool = True
|
||||
combined["output_tokens"] += ext.get("output_tokens", 0)
|
||||
if dedup and combined["nodes"]:
|
||||
combined["nodes"], combined["edges"] = deduplicate_entities(
|
||||
combined["nodes"], combined["edges"], communities={}
|
||||
combined["nodes"], combined["edges"], communities={},
|
||||
dedup_llm_backend=dedup_llm_backend,
|
||||
)
|
||||
return build_from_json(combined, directed=directed)
|
||||
|
||||
@@ -201,6 +210,7 @@ def build_merge(
|
||||
*,
|
||||
directed: bool = False,
|
||||
dedup: bool = True,
|
||||
dedup_llm_backend: str | None = None,
|
||||
) -> nx.Graph:
|
||||
"""Load existing graph.json, merge new chunks into it, and save back.
|
||||
|
||||
@@ -226,7 +236,7 @@ def build_merge(
|
||||
base = []
|
||||
|
||||
all_chunks = base + list(new_chunks)
|
||||
G = build(all_chunks, directed=directed, dedup=dedup)
|
||||
G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend)
|
||||
|
||||
# Prune nodes from deleted source files
|
||||
if prune_sources:
|
||||
|
||||
Reference in New Issue
Block a user