diff --git a/graphify/dedup.py b/graphify/dedup.py index d02aedcc..e0ddd2e3 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -6,6 +6,7 @@ Jaro-Winkler verification → same-community boost → union-find merge. from __future__ import annotations import math import re +import sys import unicodedata from collections import defaultdict @@ -219,12 +220,29 @@ def deduplicate_entities( if len(nodes) <= 1: return nodes, edges - # Pre-deduplicate: keep first occurrence of each id + # Pre-deduplicate: keep first occurrence of each id. + # Warn when two nodes share an ID but originate from different source files — + # this indicates a cross-chunk ID collision (#1504) where silent data loss occurs. seen_ids: dict[str, dict] = {} for node in nodes: nid = node.get("id", "") - if nid and nid not in seen_ids: + if not nid: + continue + if nid not in seen_ids: seen_ids[nid] = node + else: + existing_sf = seen_ids[nid].get("source_file") or "" + new_sf = node.get("source_file") or "" + if existing_sf != new_sf: + print( + f"[graphify] WARNING: node '{nid}' from '{new_sf}' collides with " + f"node from '{existing_sf}' — the second node will be dropped. " + f"This is a cross-chunk ID collision caused by two files with the " + f"same name in different directories. To avoid data loss, run " + f"'graphify extract' per subfolder and merge with " + f"'graphify merge-graphs'.", + file=sys.stderr, + ) unique_nodes = list(seen_ids.values()) if len(unique_nodes) <= 1: diff --git a/tests/test_dedup.py b/tests/test_dedup.py index 0f5d9115..267388e5 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -364,3 +364,43 @@ def test_dedup_still_merges_crossfile_true_duplicates(): ] result_nodes, _ = deduplicate_entities(nodes, [], communities={}) assert len(result_nodes) == 1 + + +# ── #1504: cross-chunk node ID collision warning ────────────────────────────── + +def test_cross_chunk_id_collision_emits_warning(capsys): + """When two nodes share the same ID but come from different source files + (a cross-chunk LLM ID collision), a WARNING must be printed to stderr + and only the first node survives (#1504).""" + nodes = [ + {"id": "readme_booking_service", "label": "Booking Service", + "file_type": "concept", "source_file": "module-a/README.md"}, + {"id": "readme_booking_service", "label": "Booking Service", + "file_type": "concept", "source_file": "module-b/README.md"}, + ] + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + + assert len(result_nodes) == 1 + assert result_nodes[0]["source_file"] == "module-a/README.md" + + captured = capsys.readouterr() + assert "WARNING" in captured.err + assert "readme_booking_service" in captured.err + assert "module-b/README.md" in captured.err + assert "module-a/README.md" in captured.err + + +def test_same_id_same_source_file_no_warning(capsys): + """When two nodes share both ID and source_file (same-file dedup), + no collision warning should be emitted.""" + nodes = [ + {"id": "readme_booking_service", "label": "Booking Service", + "file_type": "concept", "source_file": "module-a/README.md"}, + {"id": "readme_booking_service", "label": "Booking Service (dupe)", + "file_type": "concept", "source_file": "module-a/README.md"}, + ] + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + + assert len(result_nodes) == 1 + captured = capsys.readouterr() + assert "WARNING" not in captured.err