From 5320aa8eb1e68069314dbf48627a88148d5565d6 Mon Sep 17 00:00:00 2001
From: Varun Nuthalapati <nuthalapativarun@gmail.com>
Date: Sat, 27 Jun 2026 22:18:48 -0700
Subject: [PATCH] fix(dedup): warn on cross-chunk node ID collision to surface
 silent data loss (#1504)

When two LLM extraction chunks each process a file with the same name in
different directories, they independently generate the same node IDs and
deduplicate_entities() silently drops one node (first-writer-wins). The
data loss had no indication in any log, counter, or output.

Adds a stderr WARNING when a duplicate ID comes from a different
source_file, telling the user which files collided and recommending the
per-subfolder extract + merge-graphs workflow to avoid it.
---
 graphify/dedup.py   | 22 ++++++++++++++++++++--
 tests/test_dedup.py | 40 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 2 deletions(-)

diff --git a/graphify/dedup.py b/graphify/dedup.py
index d02aedcc..e0ddd2e3 100644
--- a/graphify/dedup.py
+++ b/graphify/dedup.py
@@ -6,6 +6,7 @@ Jaro-Winkler verification → same-community boost → union-find merge.
 from __future__ import annotations
 import math
 import re
+import sys
 import unicodedata
 from collections import defaultdict
 
@@ -219,12 +220,29 @@ def deduplicate_entities(
     if len(nodes) <= 1:
         return nodes, edges
 
-    # Pre-deduplicate: keep first occurrence of each id
+    # Pre-deduplicate: keep first occurrence of each id.
+    # Warn when two nodes share an ID but originate from different source files —
+    # this indicates a cross-chunk ID collision (#1504) where silent data loss occurs.
     seen_ids: dict[str, dict] = {}
     for node in nodes:
         nid = node.get("id", "")
-        if nid and nid not in seen_ids:
+        if not nid:
+            continue
+        if nid not in seen_ids:
             seen_ids[nid] = node
+        else:
+            existing_sf = seen_ids[nid].get("source_file") or ""
+            new_sf = node.get("source_file") or ""
+            if existing_sf != new_sf:
+                print(
+                    f"[graphify] WARNING: node '{nid}' from '{new_sf}' collides with "
+                    f"node from '{existing_sf}' — the second node will be dropped. "
+                    f"This is a cross-chunk ID collision caused by two files with the "
+                    f"same name in different directories. To avoid data loss, run "
+                    f"'graphify extract' per subfolder and merge with "
+                    f"'graphify merge-graphs'.",
+                    file=sys.stderr,
+                )
     unique_nodes = list(seen_ids.values())
 
     if len(unique_nodes) <= 1:
diff --git a/tests/test_dedup.py b/tests/test_dedup.py
index 0f5d9115..267388e5 100644
--- a/tests/test_dedup.py
+++ b/tests/test_dedup.py
@@ -364,3 +364,43 @@ def test_dedup_still_merges_crossfile_true_duplicates():
     ]
     result_nodes, _ = deduplicate_entities(nodes, [], communities={})
     assert len(result_nodes) == 1
+
+
+# ── #1504: cross-chunk node ID collision warning ──────────────────────────────
+
+def test_cross_chunk_id_collision_emits_warning(capsys):
+    """When two nodes share the same ID but come from different source files
+    (a cross-chunk LLM ID collision), a WARNING must be printed to stderr
+    and only the first node survives (#1504)."""
+    nodes = [
+        {"id": "readme_booking_service", "label": "Booking Service",
+         "file_type": "concept", "source_file": "module-a/README.md"},
+        {"id": "readme_booking_service", "label": "Booking Service",
+         "file_type": "concept", "source_file": "module-b/README.md"},
+    ]
+    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
+
+    assert len(result_nodes) == 1
+    assert result_nodes[0]["source_file"] == "module-a/README.md"
+
+    captured = capsys.readouterr()
+    assert "WARNING" in captured.err
+    assert "readme_booking_service" in captured.err
+    assert "module-b/README.md" in captured.err
+    assert "module-a/README.md" in captured.err
+
+
+def test_same_id_same_source_file_no_warning(capsys):
+    """When two nodes share both ID and source_file (same-file dedup),
+    no collision warning should be emitted."""
+    nodes = [
+        {"id": "readme_booking_service", "label": "Booking Service",
+         "file_type": "concept", "source_file": "module-a/README.md"},
+        {"id": "readme_booking_service", "label": "Booking Service (dupe)",
+         "file_type": "concept", "source_file": "module-a/README.md"},
+    ]
+    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
+
+    assert len(result_nodes) == 1
+    captured = capsys.readouterr()
+    assert "WARNING" not in captured.err