From df9b7ec54f2593ba9aaa86184f6ba9b1a7ca97f5 Mon Sep 17 00:00:00 2001 From: Safi Date: Thu, 23 Apr 2026 20:04:43 +0100 Subject: [PATCH] fix #479 #451: build_merge(), pre-write shrink guard, label dedup, chunk-suffix prompt block --- graphify/build.py | 121 +++++++++++++++++++++++++++++++++++++++++++++ graphify/export.py | 22 ++++++++- graphify/skill.md | 2 +- 3 files changed, 143 insertions(+), 2 deletions(-) diff --git a/graphify/build.py b/graphify/build.py index 19b5b1867..2c2c773bb 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -21,8 +21,10 @@ # before any graph construction happens. # from __future__ import annotations +import json import re import sys +from pathlib import Path import networkx as nx from .validate import validate_extraction @@ -50,6 +52,18 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: # Canonicalize legacy node/edge schema before validation. for node in extraction.get("nodes", []): if isinstance(node, dict) and "source" in node and "source_file" not in node: + # Count edges that reference this node so the warning is actionable (#479) + node_id = node.get("id", "?") + affected_edges = sum( + 1 for e in extraction.get("edges", []) + if e.get("source") == node_id or e.get("target") == node_id + ) + print( + f"[graphify] WARNING: node '{node_id}' uses field 'source' instead of " + f"'source_file' — {affected_edges} edge(s) may be misrouted. " + f"Rename the field to 'source_file' to silence this warning.", + file=sys.stderr, + ) node["source_file"] = node.pop("source") errors = validate_extraction(extraction) @@ -111,3 +125,110 @@ def build(extractions: list[dict], *, directed: bool = False) -> nx.Graph: combined["input_tokens"] += ext.get("input_tokens", 0) combined["output_tokens"] += ext.get("output_tokens", 0) return build_from_json(combined, directed=directed) + + +def _norm_label(label: str) -> str: + """Canonical dedup key — lowercase, alphanumeric only.""" + return re.sub(r"[^a-z0-9 ]", "", label.lower()).strip() + + +def deduplicate_by_label(nodes: list[dict], edges: list[dict]) -> tuple[list[dict], list[dict]]: + """Merge nodes that share a normalised label, rewriting edge references. + + Prefers IDs without chunk suffixes (_c\\d+) and shorter IDs when tied. + Drops self-loops created by the merge. Called in build() automatically. + """ + _CHUNK_SUFFIX = re.compile(r"_c\d+$") + canonical: dict[str, dict] = {} # norm_label -> surviving node + remap: dict[str, str] = {} # old_id -> surviving_id + + for node in nodes: + key = _norm_label(node.get("label", node.get("id", ""))) + if not key: + continue + existing = canonical.get(key) + if existing is None: + canonical[key] = node + else: + has_suffix = bool(_CHUNK_SUFFIX.search(node["id"])) + existing_has_suffix = bool(_CHUNK_SUFFIX.search(existing["id"])) + if has_suffix and not existing_has_suffix: + remap[node["id"]] = existing["id"] + elif existing_has_suffix and not has_suffix: + remap[existing["id"]] = node["id"] + canonical[key] = node + elif len(node["id"]) < len(existing["id"]): + remap[existing["id"]] = node["id"] + canonical[key] = node + else: + remap[node["id"]] = existing["id"] + + if not remap: + return nodes, edges + + print(f"[graphify] Deduplicated {len(remap)} duplicate node(s) by label.", file=sys.stderr) + deduped_nodes = list(canonical.values()) + deduped_edges = [] + for edge in edges: + e = dict(edge) + e["source"] = remap.get(e["source"], e["source"]) + e["target"] = remap.get(e["target"], e["target"]) + if e["source"] != e["target"]: + deduped_edges.append(e) + return deduped_nodes, deduped_edges + + +def build_merge( + new_chunks: list[dict], + graph_path: str | Path = "graphify-out/graph.json", + prune_sources: list[str] | None = None, + *, + directed: bool = False, +) -> nx.Graph: + """Load existing graph.json, merge new chunks into it, and save back. + + Never replaces — only grows (or prunes deleted-file nodes via prune_sources). + Safe to call repeatedly: existing nodes and edges are preserved. + """ + from networkx.readwrite import json_graph as _jg + + graph_path = Path(graph_path) + if graph_path.exists(): + data = json.loads(graph_path.read_text(encoding="utf-8")) + try: + existing_G = _jg.node_link_graph(data, edges="links") + except TypeError: + existing_G = _jg.node_link_graph(data) + # Reconstruct as a plain extraction dict so build() can merge it + existing_nodes = [{"id": n, **existing_G.nodes[n]} for n in existing_G.nodes] + existing_edges = [ + {"source": u, "target": v, **d} for u, v, d in existing_G.edges(data=True) + ] + base = [{"nodes": existing_nodes, "edges": existing_edges}] + else: + base = [] + + all_chunks = base + list(new_chunks) + G = build(all_chunks, directed=directed) + + # Prune nodes from deleted source files + if prune_sources: + to_remove = [ + n for n, d in G.nodes(data=True) + if d.get("source_file") in prune_sources + ] + G.remove_nodes_from(to_remove) + if to_remove: + print(f"[graphify] Pruned {len(to_remove)} node(s) from deleted sources.", file=sys.stderr) + + # Safety check: refuse to shrink the graph silently (#479) + if graph_path.exists(): + existing_n = len(existing_nodes) + new_n = G.number_of_nodes() + if new_n < existing_n: + raise ValueError( + f"graphify: build_merge would shrink graph from {existing_n} → {new_n} nodes. " + f"Pass prune_sources explicitly if you intend to remove nodes." + ) + + return G diff --git a/graphify/export.py b/graphify/export.py index 12cafb874..6eb4e0d51 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -279,7 +279,27 @@ def attach_hyperedges(G: nx.Graph, hyperedges: list) -> None: G.graph["hyperedges"] = existing -def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str) -> None: +def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, force: bool = False) -> None: + # Safety check: refuse to silently shrink an existing graph (#479) + existing_path = Path(output_path) + if not force and existing_path.exists(): + try: + existing_data = json.loads(existing_path.read_text(encoding="utf-8")) + existing_n = len(existing_data.get("nodes", [])) + new_n = G.number_of_nodes() + if new_n < existing_n: + import sys as _sys + print( + f"[graphify] WARNING: new graph has {new_n} nodes but existing " + f"graph.json has {existing_n}. Refusing to overwrite — you may be " + f"missing chunk files from a previous session. " + f"Pass force=True to override.", + file=_sys.stderr, + ) + return + except Exception: + pass # unreadable existing file — proceed with write + node_community = _node_community_map(communities) try: data = json_graph.node_link_data(G, edges="links") diff --git a/graphify/skill.md b/graphify/skill.md index 6d45b1534..60d242209 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -335,7 +335,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the filename without extension and entity is the symbol name, both normalized (lowercase, non-alphanumeric chars replaced with `_`). Example: `src/auth/session.py` + `ValidateToken` → `session_validatetoken`. This must match the ID the AST extractor generates so cross-references between code and semantic nodes connect correctly. +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the filename without extension and entity is the symbol name, both normalized (lowercase, non-alphanumeric chars replaced with `_`). Example: `src/auth/session.py` + `ValidateToken` → `session_validatetoken`. This must match the ID the AST extractor generates so cross-references between code and semantic nodes connect correctly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Output exactly this JSON (no other text): {"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0}