fix #479 #451: build_merge(), pre-write shrink guard, label dedup, chunk-suffix prompt block

graphify/build.py

@@ -21,8 +21,10 @@
 #    before any graph construction happens.
 #
 from __future__ import annotations
+import json
 import re
 import sys
+from pathlib import Path
 import networkx as nx
 from .validate import validate_extraction
 
@@ -50,6 +52,18 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph:
     # Canonicalize legacy node/edge schema before validation.
     for node in extraction.get("nodes", []):
         if isinstance(node, dict) and "source" in node and "source_file" not in node:
+            # Count edges that reference this node so the warning is actionable (#479)
+            node_id = node.get("id", "?")
+            affected_edges = sum(
+                1 for e in extraction.get("edges", [])
+                if e.get("source") == node_id or e.get("target") == node_id
+            )
+            print(
+                f"[graphify] WARNING: node '{node_id}' uses field 'source' instead of "
+                f"'source_file' — {affected_edges} edge(s) may be misrouted. "
+                f"Rename the field to 'source_file' to silence this warning.",
+                file=sys.stderr,
+            )
             node["source_file"] = node.pop("source")
 
     errors = validate_extraction(extraction)
@@ -111,3 +125,110 @@ def build(extractions: list[dict], *, directed: bool = False) -> nx.Graph:
         combined["input_tokens"] += ext.get("input_tokens", 0)
         combined["output_tokens"] += ext.get("output_tokens", 0)
     return build_from_json(combined, directed=directed)
+
+
+def _norm_label(label: str) -> str:
+    """Canonical dedup key — lowercase, alphanumeric only."""
+    return re.sub(r"[^a-z0-9 ]", "", label.lower()).strip()
+
+
+def deduplicate_by_label(nodes: list[dict], edges: list[dict]) -> tuple[list[dict], list[dict]]:
+    """Merge nodes that share a normalised label, rewriting edge references.
+
+    Prefers IDs without chunk suffixes (_c\\d+) and shorter IDs when tied.
+    Drops self-loops created by the merge. Called in build() automatically.
+    """
+    _CHUNK_SUFFIX = re.compile(r"_c\d+$")
+    canonical: dict[str, dict] = {}  # norm_label -> surviving node
+    remap: dict[str, str] = {}       # old_id -> surviving_id
+
+    for node in nodes:
+        key = _norm_label(node.get("label", node.get("id", "")))
+        if not key:
+            continue
+        existing = canonical.get(key)
+        if existing is None:
+            canonical[key] = node
+        else:
+            has_suffix = bool(_CHUNK_SUFFIX.search(node["id"]))
+            existing_has_suffix = bool(_CHUNK_SUFFIX.search(existing["id"]))
+            if has_suffix and not existing_has_suffix:
+                remap[node["id"]] = existing["id"]
+            elif existing_has_suffix and not has_suffix:
+                remap[existing["id"]] = node["id"]
+                canonical[key] = node
+            elif len(node["id"]) < len(existing["id"]):
+                remap[existing["id"]] = node["id"]
+                canonical[key] = node
+            else:
+                remap[node["id"]] = existing["id"]
+
+    if not remap:
+        return nodes, edges
+
+    print(f"[graphify] Deduplicated {len(remap)} duplicate node(s) by label.", file=sys.stderr)
+    deduped_nodes = list(canonical.values())
+    deduped_edges = []
+    for edge in edges:
+        e = dict(edge)
+        e["source"] = remap.get(e["source"], e["source"])
+        e["target"] = remap.get(e["target"], e["target"])
+        if e["source"] != e["target"]:
+            deduped_edges.append(e)
+    return deduped_nodes, deduped_edges
+
+
+def build_merge(
+    new_chunks: list[dict],
+    graph_path: str | Path = "graphify-out/graph.json",
+    prune_sources: list[str] | None = None,
+    *,
+    directed: bool = False,
+) -> nx.Graph:
+    """Load existing graph.json, merge new chunks into it, and save back.
+
+    Never replaces — only grows (or prunes deleted-file nodes via prune_sources).
+    Safe to call repeatedly: existing nodes and edges are preserved.
+    """
+    from networkx.readwrite import json_graph as _jg
+
+    graph_path = Path(graph_path)
+    if graph_path.exists():
+        data = json.loads(graph_path.read_text(encoding="utf-8"))
+        try:
+            existing_G = _jg.node_link_graph(data, edges="links")
+        except TypeError:
+            existing_G = _jg.node_link_graph(data)
+        # Reconstruct as a plain extraction dict so build() can merge it
+        existing_nodes = [{"id": n, **existing_G.nodes[n]} for n in existing_G.nodes]
+        existing_edges = [
+            {"source": u, "target": v, **d} for u, v, d in existing_G.edges(data=True)
+        ]
+        base = [{"nodes": existing_nodes, "edges": existing_edges}]
+    else:
+        base = []
+
+    all_chunks = base + list(new_chunks)
+    G = build(all_chunks, directed=directed)
+
+    # Prune nodes from deleted source files
+    if prune_sources:
+        to_remove = [
+            n for n, d in G.nodes(data=True)
+            if d.get("source_file") in prune_sources
+        ]
+        G.remove_nodes_from(to_remove)
+        if to_remove:
+            print(f"[graphify] Pruned {len(to_remove)} node(s) from deleted sources.", file=sys.stderr)
+
+    # Safety check: refuse to shrink the graph silently (#479)
+    if graph_path.exists():
+        existing_n = len(existing_nodes)
+        new_n = G.number_of_nodes()
+        if new_n < existing_n:
+            raise ValueError(
+                f"graphify: build_merge would shrink graph from {existing_n} → {new_n} nodes. "
+                f"Pass prune_sources explicitly if you intend to remove nodes."
+            )
+
+    return G

graphify/export.py

@@ -279,7 +279,27 @@ def attach_hyperedges(G: nx.Graph, hyperedges: list) -> None:
     G.graph["hyperedges"] = existing
 
 
-def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str) -> None:
+def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, force: bool = False) -> None:
+    # Safety check: refuse to silently shrink an existing graph (#479)
+    existing_path = Path(output_path)
+    if not force and existing_path.exists():
+        try:
+            existing_data = json.loads(existing_path.read_text(encoding="utf-8"))
+            existing_n = len(existing_data.get("nodes", []))
+            new_n = G.number_of_nodes()
+            if new_n < existing_n:
+                import sys as _sys
+                print(
+                    f"[graphify] WARNING: new graph has {new_n} nodes but existing "
+                    f"graph.json has {existing_n}. Refusing to overwrite — you may be "
+                    f"missing chunk files from a previous session. "
+                    f"Pass force=True to override.",
+                    file=_sys.stderr,
+                )
+                return
+        except Exception:
+            pass  # unreadable existing file — proceed with write
+
     node_community = _node_community_map(communities)
     try:
         data = json_graph.node_link_data(G, edges="links")

graphify/skill.md

@@ -335,7 +335,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d
   Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5.
 - AMBIGUOUS edges: 0.1-0.3
 
-Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the filename without extension and entity is the symbol name, both normalized (lowercase, non-alphanumeric chars replaced with `_`). Example: `src/auth/session.py` + `ValidateToken` → `session_validatetoken`. This must match the ID the AST extractor generates so cross-references between code and semantic nodes connect correctly.
+Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the filename without extension and entity is the symbol name, both normalized (lowercase, non-alphanumeric chars replaced with `_`). Example: `src/auth/session.py` + `ValidateToken` → `session_validatetoken`. This must match the ID the AST extractor generates so cross-references between code and semantic nodes connect correctly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it.
 
 Output exactly this JSON (no other text):
 {"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0}