graphify/tests/test_dedup.py

"""Tests for graphify/dedup.py entity deduplication pipeline."""
from __future__ import annotations
import pytest
from graphify.dedup import (
    deduplicate_entities,
    _collision_rank,
    _defines_id,
    _entropy,
    _lifecycle_penalty,
    _shingles,
)


# ── entropy gate ─────────────────────────────────────────────────────────────

def test_entropy_short_label_low():
    assert _entropy("AI") < 2.5

def test_entropy_normal_label_high():
    assert _entropy("AuthenticationManager") >= 2.5

def test_entropy_empty_string():
    assert _entropy("") == 0.0


# ── shingles ─────────────────────────────────────────────────────────────────

def test_shingles_produces_trigrams():
    s = _shingles("hello")
    assert "hel" in s
    assert "ell" in s
    assert "llo" in s

def test_shingles_short_string():
    # strings shorter than 3 chars return single shingle of the string itself
    assert _shingles("ab") == {"ab"}


# ── full pipeline ─────────────────────────────────────────────────────────────

def _make_nodes(*labels):
    return [{"id": label.lower().replace(" ", "_"), "label": label, "source_file": "test.md"} for label in labels]

def _make_edges(src, tgt, relation="relates_to"):
    return [{"source": src, "target": tgt, "relation": relation}]


def test_exact_duplicates_merged():
    nodes = _make_nodes("UserService", "userservice", "User Service")
    edges = []
    result_nodes, result_edges = deduplicate_entities(nodes, edges, communities={})
    # All three are the same concept — only one survives
    assert len(result_nodes) == 1


def test_typo_merged():
    # "GraphExtractor" vs "Graph Extractor" — Jaro-Winkler >= 0.92
    nodes = _make_nodes("GraphExtractor", "Graph Extractor")
    edges = []
    result_nodes, _ = deduplicate_entities(nodes, edges, communities={})
    assert len(result_nodes) == 1


def test_unrelated_not_merged():
    nodes = _make_nodes("UserService", "OrderService")
    edges = []
    result_nodes, _ = deduplicate_entities(nodes, edges, communities={})
    assert len(result_nodes) == 2


def test_short_low_entropy_not_merged():
    # "AI" and "ML" are low-entropy — entropy gate skips them
    nodes = _make_nodes("AI", "ML")
    edges = []
    result_nodes, _ = deduplicate_entities(nodes, edges, communities={})
    assert len(result_nodes) == 2


def test_edges_rewired_after_merge():
    nodes = _make_nodes("GraphExtractor", "Graph Extractor", "Parser")
    # edge from loser to Parser should be rewired to winner
    edges = [{"source": "graph_extractor", "target": "parser", "relation": "uses"}]
    result_nodes, result_edges = deduplicate_entities(nodes, edges, communities={})
    assert len(result_nodes) == 2  # merged + Parser
    # edge should still exist (rewired to winner)
    assert len(result_edges) == 1


def test_self_loops_dropped_after_merge():
    # If both endpoints of an edge get merged into same node, drop the edge
    nodes = _make_nodes("GraphExtractor", "Graph Extractor")
    edges = [{"source": "graphextractor", "target": "graph_extractor", "relation": "same"}]
    _, result_edges = deduplicate_entities(nodes, edges, communities={})
    assert result_edges == []


def test_community_boost_aids_merge():
    # Two nodes in same community with score in 0.75-0.85 zone get boosted
    nodes = _make_nodes("AuthManager", "Auth Manager")
    edges = []
    # Same community → boost → merge
    communities = {"authmanager": 1, "auth_manager": 1}
    result_with, _ = deduplicate_entities(nodes, edges, communities=communities)
    # Different community → no boost
    communities_diff = {"authmanager": 1, "auth_manager": 2}
    result_without, _ = deduplicate_entities(nodes, edges, communities=communities_diff)
    assert len(result_with) <= len(result_without)


def test_empty_inputs():
    result_nodes, result_edges = deduplicate_entities([], [], communities={})
    assert result_nodes == []
    assert result_edges == []


def test_single_node_no_crash():
    nodes = _make_nodes("UserService")
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1


def test_dedup_llm_flag_accepted():
    """deduplicate_entities accepts dedup_llm_backend without crashing when no ambiguous pairs exist."""
    nodes = _make_nodes("UserService", "OrderService")
    edges = []
    result_nodes, _ = deduplicate_entities(nodes, edges, communities={}, dedup_llm_backend=None)
    assert len(result_nodes) == 2


# ── build integration ─────────────────────────────────────────────────────────

def test_build_calls_dedup():
    """build() should deduplicate near-identical nodes across extractions."""
    from graphify.build import build
    chunk1 = {
        "nodes": [{"id": "graphextractor", "label": "GraphExtractor", "source_file": "a.py"}],
        "edges": [],
    }
    chunk2 = {
        "nodes": [{"id": "graph_extractor", "label": "Graph Extractor", "source_file": "b.py"}],
        "edges": [],
    }
    G = build([chunk1, chunk2])
    assert G.number_of_nodes() == 1


def test_build_dedup_preserves_semantic_attributes():
    """The default build path must not discard semantic enrichment (#2091)."""
    from graphify.build import build
    ast = {
        "nodes": [{"id": "src_auth_login", "label": "login", "file_type": "code",
                   "source_file": "src/auth.py", "_origin": "ast",
                   "source_location": "L42"}],
        "edges": [],
    }
    semantic = {
        "nodes": [{"id": "src_auth_login", "label": "User login handler",
                   "file_type": "code", "source_file": "src/auth.py",
                   "summary": "Authenticates a user.", "confidence_score": 0.9}],
        "edges": [],
    }

    node = dict(build([ast, semantic], directed=True, dedup=True).nodes["src_auth_login"])

    assert node["label"] == "login"
    assert node["source_location"] == "L42"
    assert node["summary"] == "Authenticates a user."
    assert node["confidence_score"] == 0.9


# --- #878: fuzzy dedup false merges on short/variant labels ---

def test_dedup_does_not_merge_numeric_variants(tmp_path):
    """Chip SKU variants (ASR1603 vs ASR1605) must not be merged (#878)."""
    nodes = _make_nodes("ASR1603", "ASR1605")
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2, "ASR1603 and ASR1605 are distinct chip models, not duplicates"


def test_dedup_does_not_merge_short_insertion_variants(tmp_path):
    """Short labels differing by an insertion (cranel vs cranelr) must not merge (#878)."""
    nodes = _make_nodes("cranel", "cranelr")
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2, "cranel and cranelr are distinct, not a typo"


def test_dedup_does_not_merge_model_with_suffix(tmp_path):
    """M1 vs M1 Pro must not merge (#878)."""
    nodes = _make_nodes("M1", "M1 Pro")
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2, "M1 and M1 Pro are distinct Apple chip variants"


def test_dedup_still_merges_real_typos():
    """Genuine same-length single-char typos should still merge (#878 non-regression)."""
    from graphify.dedup import _is_variant_pair, _short_label_blocked
    from rapidfuzz.distance import JaroWinkler
    a, b = "graphextractor", "graphextractar"
    score = JaroWinkler.normalized_similarity(a, b) * 100
    assert not _is_variant_pair(a, b), "not a variant pair"
    assert not _short_label_blocked(a, b, score), "long-enough label, should not be blocked"


def test_variant_pair_helper():
    """_is_variant_pair correctly identifies chip-model variant pairs (#878)."""
    from graphify.dedup import _is_variant_pair
    assert _is_variant_pair("asr1603", "asr1605")
    assert _is_variant_pair("cortex a55", "cortex a55x")
    assert not _is_variant_pair("graphextractor", "graphextracter")
    assert not _is_variant_pair("foo", "foo")


def test_prefix_extension_symbols_not_merged():
    """Distinct symbols whose name is a strict prefix-extension of another must not
    be merged (#1201). getActiveSession / getActiveSessions score ~98.82 JW but are
    different functions; parseConfig / parseConfigFile likewise."""
    import networkx as nx
    from graphify.dedup import deduplicate_entities

    pairs = [
        ("getActiveSession", "getActiveSessions"),
        ("parseConfig", "parseConfigFile"),
        ("load", "loadAll"),
        ("handleRequest", "handleRequestTimeout"),
    ]
    for a, b in pairs:
        nodes = [
            {"id": f"{a}_id", "label": a, "type": "CODE", "src_file": "api.py"},
            {"id": f"{b}_id", "label": b, "type": "CODE", "src_file": "api.py"},
        ]
        edges = [{"src": f"{a}_id", "tgt": f"{b}_id", "relation": "calls",
                  "c": 1.0, "weight": 1.0}]
        out_nodes, _ = deduplicate_entities(
            nodes, edges, communities={f"{a}_id": 0, f"{b}_id": 0}
        )
        labels = {n["label"] for n in out_nodes}
        assert a in labels and b in labels, (
            f"#1201 regression: '{a}' and '{b}' were merged — they are distinct symbols"
        )


def test_pass2_winner_union_does_not_pull_in_uncompared_same_label_nodes():
    """Pass 2's winner selection must consider only the verified pair (#1247).

    Picking the winner from the union of both normalized-label groups pulls
    never-compared nodes into the merge: here A ("Session Manager", auth.md)
    and B ("Session Manager", billing.md) are deliberately kept distinct by
    the cross-file identical-label guards (#1046, #1178), yet when the
    A-C fuzzy match ("Session Managr" typo) fires, _pick_winner over
    [A, B, C] selects B (shortest id) and unions B with both A and C —
    merging B although it was never compared against anything.
    """
    nodes = [
        {"id": "session_manager_auth", "label": "Session Manager",
         "source_file": "auth.md"},
        {"id": "sm", "label": "Session Manager",
         "source_file": "billing.md"},
        {"id": "session_managr_notes", "label": "Session Managr",
         "source_file": "notes.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    ids = {n["id"] for n in result_nodes}
    # B must survive as a distinct node: identical label across different
    # source files is exactly what the #1046/#1178 guards keep separate.
    assert "sm" in ids, (
        "uncompared cross-file node 'sm' was absorbed via pass-2 winner-union"
    )
    # The verified fuzzy pair (A, C) still merges — only one of them survives.
    assert len(result_nodes) == 2


def test_prefix_guard_does_not_block_same_length_typos():
    """The prefix-extension guard must not fire for same-length pairs — only strict
    prefix-extensions (one is a substring of the other) should be blocked (#1201).
    graphextractor / graphextractar have the same length, so neither starts-with the
    other, and the guard must not fire."""
    from graphify.dedup import _norm
    a = _norm("GraphExtractor")   # "graphextractor" — 14 chars
    b = _norm("GraphExtractar")   # "graphextractar" — 14 chars
    lo, hi = sorted((a, b), key=len)
    # Same-length pair: startswith only holds when strings are identical
    assert not (hi.startswith(lo) and hi != lo), (
        f"Prefix guard fires on same-length pair ({a!r}, {b!r}) — should not"
    )


def test_prefix_guard_fires_for_extension_pairs():
    """The prefix-extension guard must fire for pairs where one is a strict prefix
    of the other, preventing false merges (#1201)."""
    from graphify.dedup import _norm
    pairs = [
        ("getActiveSession", "getActiveSessions"),
        ("parseConfig", "parseConfigFile"),
        ("load", "loadAll"),
    ]
    for a_raw, b_raw in pairs:
        a, b = _norm(a_raw), _norm(b_raw)
        lo, hi = sorted((a, b), key=len)
        assert hi.startswith(lo) and hi != lo, (
            f"Prefix guard should fire for ({a!r}, {b!r}) but did not"
        )


# ── #1284: numbered siblings + cross-file file-anchored boilerplate ──────────

def test_numeric_tokens_differ_helper():
    """_numeric_tokens_differ compares digit runs as zero-padding-insensitive
    multisets (#1284)."""
    from graphify.dedup import _numeric_tokens_differ
    assert _numeric_tokens_differ("adr 0011 d5 pipeline placement", "adr 0013 d4 pipeline placement")
    assert _numeric_tokens_differ("3 1 product goals", "1 1 product goals")
    assert _numeric_tokens_differ("code block3", "code block13")
    assert not _numeric_tokens_differ("phase 09 overview", "phase 9 overview")  # zero-padding
    assert not _numeric_tokens_differ("module layout wave 3", "module layouts wave 3")
    assert not _numeric_tokens_differ("graph extractor", "graph extractar")  # digitless


def test_dedup_does_not_merge_numbered_siblings():
    """Long labels differing only in embedded numbers (ADR/section/issue ids)
    must not merge — numbered siblings, not duplicates (#1284)."""
    nodes = [
        {"id": "n1", "label": "Pipeline placement — 4 call sites (ADR 0013 D4)",
         "file_type": "document", "source_file": "docs/index-activity.md"},
        {"id": "n2", "label": "Pipeline placement — 4 call sites (ADR 0011 §D5)",
         "file_type": "document", "source_file": "docs/schema-matcher.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2


def test_dedup_does_not_merge_crossfile_rationale_boilerplate():
    """Rationale nodes are file-anchored like code (#1205): parallel modules'
    boilerplate docstrings differing by one word must not merge (#1284)."""
    boiler = ("Django app config for {}. No business logic here. "
              "Domain services live in services.py and adapters in providers.")
    nodes = [
        {"id": "r1", "label": boiler.format("apps.platform.cards"),
         "file_type": "rationale", "source_file": "apps/platform/cards/apps.py"},
        {"id": "r2", "label": boiler.format("apps.platform.cores"),
         "file_type": "rationale", "source_file": "apps/platform/cores/apps.py"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2


def test_dedup_does_not_merge_crossfile_document_headings():
    """Document nodes are file-anchored too: near-identical headings in different
    files are distinct sections, not duplicates (#1284, extends the rationale guard)."""
    nodes = [
        {"id": "d1", "label": "Getting Started Installation Guide",
         "file_type": "document", "source_file": "docs/a.md"},
        {"id": "d2", "label": "Getting Started Installation Setup",
         "file_type": "document", "source_file": "docs/b.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2


def test_dedup_still_merges_samefile_rationale_duplicates():
    """The file-anchored guard only blocks cross-file pairs — near-identical
    rationale duplicates within one file still merge (#1284 non-regression)."""
    nodes = [
        {"id": "r1", "label": "Counts-only metrics export, a read-only aggregation service.",
         "file_type": "rationale", "source_file": "apps/schemas/metrics.py"},
        {"id": "r2", "label": "Counts-only metrics export, the read-only aggregation service.",
         "file_type": "rationale", "source_file": "apps/schemas/metrics.py"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1


# ── #1243: JaroWinkler prefix-bonus over-merge (cross-file) ──────────────────

def test_dedup_does_not_merge_crossfile_shared_prefix_divergence():
    """Cross-file labels sharing a long prefix but diverging in a distinguishing
    token ("…jest native" vs "…react native") get JaroWinkler's prefix bonus past
    threshold but are distinct entities; scoring them on plain Jaro blocks the
    merge (#1243)."""
    nodes = [
        {"id": "p1", "label": "testing library jest native",
         "file_type": "concept", "source_file": "pkg-a/package.json"},
        {"id": "p2", "label": "testing library react native",
         "file_type": "concept", "source_file": "pkg-b/package.json"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2


def test_dedup_still_merges_crossfile_true_duplicates():
    """The #1243 guard only drops the prefix bonus — a genuine cross-file
    duplicate (high similarity on Jaro alone) must still merge."""
    nodes = [
        {"id": "g1", "label": "GraphExtractor", "file_type": "concept", "source_file": "a.md"},
        {"id": "g2", "label": "Graph Extractor", "file_type": "concept", "source_file": "b.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1


# ── #1504: cross-chunk node ID collision warning ──────────────────────────────

def test_cross_chunk_id_collision_emits_warning(capsys):
    """When two nodes share the same ID but come from different source files
    (a cross-chunk LLM ID collision), a WARNING must be printed to stderr
    and only the first node survives (#1504)."""
    nodes = [
        {"id": "readme_booking_service", "label": "Booking Service",
         "file_type": "concept", "source_file": "module-a/README.md"},
        {"id": "readme_booking_service", "label": "Booking Service",
         "file_type": "concept", "source_file": "module-b/README.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})

    assert len(result_nodes) == 1
    assert result_nodes[0]["source_file"] == "module-a/README.md"

    captured = capsys.readouterr()
    assert "WARNING" in captured.err
    assert "readme_booking_service" in captured.err
    assert "module-b/README.md" in captured.err
    assert "module-a/README.md" in captured.err


def test_same_id_same_source_file_no_warning(capsys):
    """When two nodes share both ID and source_file (same-file dedup),
    no collision warning should be emitted."""
    nodes = [
        {"id": "readme_booking_service", "label": "Booking Service",
         "file_type": "concept", "source_file": "module-a/README.md"},
        {"id": "readme_booking_service", "label": "Booking Service (dupe)",
         "file_type": "concept", "source_file": "module-a/README.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})

    assert len(result_nodes) == 1
    captured = capsys.readouterr()
    assert "WARNING" not in captured.err


# ── #1857: dedup summary log breakdown ────────────────────────────────────────

def test_dedup_summary_prints_fuzzy_count_when_no_exact_merges(capsys):
    """A fuzzy-only run must still report the fuzzy count (#1857).

    Two long, high-entropy, non-code labels on different files. Pass 1 (exact,
    same-file) finds nothing; Pass 2 (Jaro-Winkler cross-file) merges them.
    Previously the fuzzy count was nested inside `if exact_merges`, so this
    line printed as `Deduplicated 1 node(s).` with no breakdown.
    """
    nodes = [
        {"id": "g1", "label": "GraphExtractor", "file_type": "concept", "source_file": "a.md"},
        {"id": "g2", "label": "Graph Extractor", "file_type": "concept", "source_file": "b.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1
    captured = capsys.readouterr()
    assert "Deduplicated" in captured.out
    assert "fuzzy" in captured.out
    assert "exact" not in captured.out


def test_dedup_summary_still_reports_exact_only(capsys):
    """Non-regression: an exact-only run still prints `(N exact)` and no fuzzy."""
    # Same file + same normalized label → Pass 1 exact merge.
    nodes = [
        {"id": "u1", "label": "User Service", "file_type": "concept", "source_file": "svc.md"},
        {"id": "u2", "label": "user service", "file_type": "concept", "source_file": "svc.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1
    captured = capsys.readouterr()
    assert "exact" in captured.out
    assert "fuzzy" not in captured.out


# ── ID collisions: definition vs cross-reference ──────────────────────────────

# The defining node and a doc that merely mentions the entity. Both mint the ID
# encoded from the *defining* file's path, so they collide by construction.
_DEFINING = {"id": "agents_make_batch_fixtures_make_batch_fixtures",
             "label": "make-batch-fixtures agent", "file_type": "concept",
             "source_file": "agents/make-batch-fixtures.md"}
_REFERENCING = {"id": "agents_make_batch_fixtures_make_batch_fixtures",
                "label": "make-batch-fixtures", "file_type": "concept",
                "source_file": "available/diagnose-issue/SKILL.md"}


@pytest.mark.parametrize("nodes", [
    [_DEFINING, _REFERENCING],
    [_REFERENCING, _DEFINING],
], ids=["definition-first", "reference-first"])
def test_defining_file_wins_over_referencing_file(nodes, capsys):
    """The node whose source_file is the file its ID encodes survives, whichever
    chunk order the nodes arrive in — the survivor must not depend on it."""
    result_nodes, _ = deduplicate_entities(list(nodes), [], communities={})

    assert len(result_nodes) == 1
    assert result_nodes[0]["source_file"] == "agents/make-batch-fixtures.md"
    assert result_nodes[0]["label"] == "make-batch-fixtures agent"


def test_reference_collision_is_silent(capsys):
    """A cross-reference rewires silently without importing foreign-file metadata."""
    edges = _make_edges("agents_make_batch_fixtures_make_batch_fixtures", "other")
    referencing = dict(_REFERENCING, summary="Reference-local description.")
    result_nodes, result_edges = deduplicate_entities(
        [_DEFINING, referencing], edges, communities={})

    assert len(result_nodes) == 1
    assert len(result_edges) == 1
    assert "summary" not in result_nodes[0]
    captured = capsys.readouterr()
    assert "WARNING" not in captured.err
    assert "note:" not in captured.err


def test_absolute_source_path_still_defines_id(capsys):
    """source_file is absolute in some pipelines and repo-relative in others; the
    defining file is recognised either way."""
    absolute = dict(_DEFINING, source_file="/home/u/proj/agents/make-batch-fixtures.md")
    result_nodes, _ = deduplicate_entities([_REFERENCING, absolute], [], communities={})

    assert len(result_nodes) == 1
    assert result_nodes[0]["label"] == "make-batch-fixtures agent"
    assert "WARNING" not in capsys.readouterr().err


# ── #3352: non-Latin source paths must still define their own id ─────────────
# _id_prefixes reimplemented id slugification with an ASCII-only regex instead
# of the canonical normalize_id every real extractor mints an id with, so a
# path segment made of Korean, CJK, or Cyrillic characters collapsed to
# nothing instead of being preserved. The reconstructed prefix then never
# matched the id actually minted for that file, so the defining page lost the
# definer-wins tiebreak to a page that merely references the same entity.

def test_id_prefixes_preserves_non_latin_segments():
    from graphify.dedup import _id_prefixes
    prefixes = _id_prefixes("concepts/작업 단위 폴더 + README 진입점 컨벤션.md")
    assert "concepts_작업_단위_폴더_readme_진입점_컨벤션" in prefixes
    assert "작업_단위_폴더_readme_진입점_컨벤션" in prefixes


_KOREAN_DEFINING = {
    "id": "concepts_작업_단위_폴더_readme_진입점_컨벤션",
    "label": "README 진입점 컨벤션", "file_type": "concept",
    "source_file": "concepts/작업 단위 폴더 + README 진입점 컨벤션.md",
}
_KOREAN_REFERENCING = {
    "id": "concepts_작업_단위_폴더_readme_진입점_컨벤션",
    "label": "README 진입점 컨벤션", "file_type": "concept",
    "source_file": "concepts/LLM Schema - CLAUDE.md를 가이드로 활용하기.md",
}


def test_defines_id_recognizes_a_non_latin_path():
    assert _defines_id(_KOREAN_DEFINING)
    assert not _defines_id(_KOREAN_REFERENCING)


@pytest.mark.parametrize("nodes", [
    [_KOREAN_DEFINING, _KOREAN_REFERENCING],
    [_KOREAN_REFERENCING, _KOREAN_DEFINING],
], ids=["definition-first", "reference-first"])
def test_korean_defining_file_wins_over_referencing_file(nodes):
    """The issue's own repro shape: the page that defines a concept must keep
    its own node instead of losing it to a page that merely mentions it."""
    result_nodes, _ = deduplicate_entities(list(nodes), [], communities={})

    assert len(result_nodes) == 1
    assert result_nodes[0]["source_file"] == (
        "concepts/작업 단위 폴더 + README 진입점 컨벤션.md"
    )


def test_id_prefixes_ascii_path_unchanged():
    """Negative control: an ordinary ASCII path's reconstructed prefixes must
    be identical to what the old ASCII only regex produced."""
    from graphify.dedup import _id_prefixes
    assert _id_prefixes("docs/v1/api/README.md") == {
        "readme", "api_readme", "v1_api_readme", "docs_v1_api_readme",
    }


def test_same_file_relabel_is_noted(capsys):
    """Two labels for one ID from one file: the loser's label is discarded, which is
    the one drop that used to be silent. It is a note, not a collision warning."""
    nodes = [
        {"id": "agents_make_batch_fixtures_make_batch_fixtures",
         "label": "make-batch-fixtures agent", "file_type": "concept",
         "source_file": "agents/make-batch-fixtures.md"},
        {"id": "agents_make_batch_fixtures_make_batch_fixtures",
         "label": "make-batch-fixtures helper agent", "file_type": "concept",
         "source_file": "agents/make-batch-fixtures.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})

    assert len(result_nodes) == 1
    captured = capsys.readouterr()
    assert "note:" in captured.err
    assert "make-batch-fixtures helper agent" in captured.err
    assert "WARNING" not in captured.err


@pytest.mark.parametrize("nodes", [
    [
        {"id": "src_auth_login", "label": "login", "file_type": "code",
         "source_file": "src/auth.py", "_origin": "ast", "source_location": "L42"},
        {"id": "src_auth_login", "label": "User login handler", "file_type": "code",
         "source_file": "src/auth.py", "summary": "Authenticates a user.",
         "confidence_score": 0.9},
    ],
    [
        {"id": "src_auth_login", "label": "User login handler", "file_type": "code",
         "source_file": "src/auth.py", "summary": "Authenticates a user.",
         "confidence_score": 0.9},
        {"id": "src_auth_login", "label": "login", "file_type": "code",
         "source_file": "src/auth.py", "_origin": "ast", "source_location": "L42"},
    ],
], ids=["ast-first", "semantic-first"])
def test_same_id_same_entity_retains_complementary_attributes(nodes):
    """Exact-ID dedup combines AST precision with semantic enrichment (#2091)."""
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})

    assert len(result_nodes) == 1
    assert result_nodes[0] == {
        "id": "src_auth_login",
        "label": "login",
        "file_type": "code",
        "source_file": "src/auth.py",
        "_origin": "ast",
        "source_location": "L42",
        "summary": "Authenticates a user.",
        "confidence_score": 0.9,
    }


def test_cross_file_id_collision_does_not_mix_attributes(capsys):
    """Two files that both mint one ID remain isolated despite exact-ID dedup."""
    nodes = [
        {"id": "pkg_service_run", "label": "run", "file_type": "code",
         "source_file": "pkg/service.py", "source_location": "L10"},
        {"id": "pkg_service_run", "label": "run helper", "file_type": "code",
         "source_file": "pkg_service.py", "summary": "Different function."},
    ]

    result_nodes, _ = deduplicate_entities(nodes, [], communities={})

    assert len(result_nodes) == 1
    assert result_nodes[0]["source_file"] == "pkg/service.py"
    assert "summary" not in result_nodes[0]
    assert "WARNING" in capsys.readouterr().err


def test_collision_survivor_is_order_independent():
    """#1851: definer + same-file relabel + cross-file reference. Across every
    insertion order the SAME node (source_file AND label) must survive — the
    definer heuristic alone left the label order-dependent among co-definers."""
    import itertools
    nid = "agents_make_batch_fixtures_make_batch_fixtures"
    definer = {"id": nid, "label": "make-batch-fixtures agent",
               "file_type": "concept", "source_file": "agents/make-batch-fixtures.md"}
    relabel = {"id": nid, "label": "make-batch-fixtures helper agent",
               "file_type": "concept", "source_file": "agents/make-batch-fixtures.md"}
    xref = {"id": nid, "label": "make-batch-fixtures", "file_type": "concept",
            "source_file": "available/diagnose-issue/SKILL.md"}
    survivors = set()
    for perm in itertools.permutations([definer, relabel, xref]):
        out, _ = deduplicate_entities([dict(n) for n in perm], [], communities={})
        assert len(out) == 1
        survivors.add((out[0]["source_file"], out[0]["label"]))
    assert survivors == {("agents/make-batch-fixtures.md", "make-batch-fixtures agent")}, (
        f"non-deterministic collision survivor: {survivors}"
    )


def test_bare_file_node_defines_its_own_id():
    """A file-level semantic node whose id is exactly the slugified path (no
    `_entity` suffix) must be recognised as defining its id (#1851 tweak)."""
    assert _defines_id({"id": "agents_make_batch_fixtures",
                        "source_file": "agents/make-batch-fixtures.md"})


def test_defines_id_helper():
    assert _defines_id(_DEFINING)
    assert not _defines_id(_REFERENCING)
    # Pre-#1504 IDs keyed off the bare filename stem.
    assert _defines_id({"id": "readme_booking_service",
                        "source_file": "module-a/README.md"})
    # A path that is merely a string-prefix of the ID's path does not define it.
    assert not _defines_id({"id": "agents_foo", "source_file": "agent/foo.md"})
    assert not _defines_id({"id": "docs_intro_foo", "source_file": ""})


# ── #2532: lifecycle-aware, environment-stable collision ranking ──────────────
# Active-vs-archived path markers adapted from @michaelxer's PR #2540, judged on
# root-relative segments with a reversed-segment final tiebreak so the survivor
# cannot flip with path form or checkout location.

_DONE_PLAN = {"id": "plans_binding_doctrine", "label": "binding doctrine",
              "file_type": "concept",
              "source_file": "plans/_done/binding-doctrine.md"}
_ACTIVE_PLAN = {"id": "plans_binding_doctrine", "label": "binding doctrine",
                "file_type": "concept",
                "source_file": "plans/in-progress/binding-doctrine.md"}


def test_lifecycle_penalty_on_relative_segments():
    assert _lifecycle_penalty("plans/_done/x.md") == 2
    assert _lifecycle_penalty("plans/in-progress/x.md") == 0
    assert _lifecycle_penalty("docs/x.md") == 1
    # A FILE named after a marker is not a marker — only directory segments count.
    assert _lifecycle_penalty("done.md") == 1
    assert _lifecycle_penalty("plans/done.md") == 1
    # Mixed markers: the best (most active) marked segment wins.
    assert _lifecycle_penalty("archive/in-progress/x.md") == 0


def test_archived_path_ranks_below_active_despite_ascii_order():
    """#2532: `plans/_done` sorts before `plans/in-progress` lexically ('_' < 'i'),
    so a raw-string tiebreak picks the archived copy — assert the ASCII trap is
    real, then that the rank inverts it."""
    assert _DONE_PLAN["source_file"] < _ACTIVE_PLAN["source_file"]  # the trap
    assert _collision_rank(_ACTIVE_PLAN) < _collision_rank(_DONE_PLAN)


def test_active_plan_survives_archived_copy_order_independent():
    """#2532 reported case: the in-progress copy survives, whichever order the
    colliding nodes arrive in."""
    import itertools
    for perm in itertools.permutations([_DONE_PLAN, _ACTIVE_PLAN]):
        out, _ = deduplicate_entities([dict(n) for n in perm], [], communities={})
        assert len(out) == 1
        assert out[0]["source_file"] == "plans/in-progress/binding-doctrine.md"


def test_checkout_dir_lifecycle_name_does_not_mark_paths(tmp_path):
    """Gap in #2540: markers must be matched on ROOT-RELATIVE segments only. An
    absolute source_file stored under a checkout directory named `wip` must not
    be scored active by the checkout location — the archived copy still loses."""
    import itertools
    root = tmp_path / "wip" / "repo"
    archived = {"id": "plans_doc", "label": "doc", "file_type": "concept",
                "source_file": str(root / "plans" / "_done" / "doc.md")}
    active = {"id": "plans_doc", "label": "doc", "file_type": "concept",
              "source_file": "plans/roadmap/doc.md"}
    for perm in itertools.permutations([archived, active]):
        out, _ = deduplicate_entities([dict(n) for n in perm], [],
                                      communities={}, root=root)
        assert len(out) == 1
        assert out[0]["source_file"] == "plans/roadmap/doc.md"


def test_neutral_collision_survivor_is_stable_across_path_forms(tmp_path):
    """#2532: with no lifecycle markers, the tiebreak must not depend on whether
    either path is stored absolute or repo-relative — one root-relative file
    identity survives across all four form combinations and both insertion
    orders (8 runs)."""
    import itertools
    root = tmp_path / "repo"
    rel_a, rel_b = "plans/q3/doc.md", "plans/roadmap/doc.md"
    survivors = set()
    for form_a in (rel_a, str(root / rel_a)):
        for form_b in (rel_b, str(root / rel_b)):
            a = {"id": "plans_doc", "label": "doc", "file_type": "concept",
                 "source_file": form_a}
            b = {"id": "plans_doc", "label": "doc", "file_type": "concept",
                 "source_file": form_b}
            for perm in itertools.permutations([a, b]):
                out, _ = deduplicate_entities([dict(n) for n in perm], [],
                                              communities={}, root=root)
                assert len(out) == 1
                sf = out[0]["source_file"].replace("\\", "/")
                prefix = str(root).replace("\\", "/") + "/"
                survivors.add(sf.removeprefix(prefix))
    assert survivors == {"plans/q3/doc.md"}, (
        f"survivor depends on path form or order: {survivors}"
    )


def test_archived_definer_beats_active_reference():
    """The definer flag outranks the lifecycle penalty: a node that owns its ID
    survives even from an archived folder against a live cross-reference."""
    import itertools
    nid = "plans_done_spec_spec"
    definer = {"id": nid, "label": "spec doc", "file_type": "concept",
               "source_file": "plans/_done/spec.md"}
    reference = {"id": nid, "label": "spec", "file_type": "concept",
                 "source_file": "plans/in-progress/roadmap.md"}
    assert _defines_id(definer) and not _defines_id(reference)
    for perm in itertools.permutations([definer, reference]):
        out, _ = deduplicate_entities([dict(n) for n in perm], [], communities={})
        assert len(out) == 1
        assert out[0]["source_file"] == "plans/_done/spec.md"


# ── #2091 review: attribute-merge correctness (fixes A-D) ─────────────────────

def test_dedup_gapfill_is_order_independent_with_multiple_losers():
    """(fix A) With 3+ same-ID same-source records, the merged attributes must not
    depend on arrival order — the best loser by collision rank supplies each
    missing key deterministically, preserving the #1851 order-independence."""
    import itertools
    base = [
        {"id": "f", "label": "f", "file_type": "code", "source_file": "m.py",
         "source_location": "L1"},                                  # shortest label -> survivor
        {"id": "f", "label": "f helper beta", "file_type": "code",
         "source_file": "m.py", "summary": "BETA"},
        {"id": "f", "label": "f helper alpha", "file_type": "code",
         "source_file": "m.py", "summary": "ALPHA"},
    ]
    seen = set()
    for perm in itertools.permutations(base):
        nodes, _ = deduplicate_entities([dict(n) for n in perm], [], communities={})
        assert len(nodes) == 1
        seen.add(nodes[0].get("summary"))
    assert len(seen) == 1, f"merged summary depends on arrival order: {seen}"


def test_dedup_no_attribute_merge_when_source_file_missing():
    """(fix B) Two provenance-less records sharing an ID must NOT cross-pollinate
    attributes — '' == '' is not proof of the same symbol (#1178)."""
    nodes = [
        {"id": "c", "label": "c", "file_type": "concept", "summary": "A"},
        {"id": "c", "label": "c", "file_type": "concept", "notes": "B"},
    ]
    result, _ = deduplicate_entities([dict(n) for n in nodes], [], communities={})
    assert len(result) == 1
    surv = result[0]
    assert not ("summary" in surv and "notes" in surv), (
        "provenance-less same-id records must not merge attributes"
    )


def test_dedup_survivor_does_not_inherit_false_origin_ast():
    """(fix C) An LLM survivor must not inherit _origin='ast' from a dropped
    same-source AST record — a false authority tag is read by ghost-merge/watch."""
    nodes = [
        {"id": "x", "label": "run", "file_type": "code", "source_file": "m.py",
         "source_location": "L9"},                                   # shorter label -> survivor (LLM)
        {"id": "x", "label": "run() [ast]", "file_type": "code", "source_file": "m.py",
         "source_location": "L2", "_origin": "ast"},                 # loser carries _origin=ast
    ]
    result, _ = deduplicate_entities([dict(n) for n in nodes], [], communities={})
    assert len(result) == 1
    assert result[0].get("_origin") != "ast", "survivor must not inherit a false _origin=ast"


def test_dedup_fills_explicit_none_attribute():
    """(fix D) An explicit source_location=None on the survivor is treated as
    absent and filled from a same-source record that has a real line (#2091)."""
    nodes = [
        {"id": "y", "label": "y", "file_type": "code", "source_file": "m.py",
         "source_location": None},                                   # survivor, explicit None
        {"id": "y", "label": "y helper", "file_type": "code", "source_file": "m.py",
         "source_location": "L7"},
    ]
    result, _ = deduplicate_entities([dict(n) for n in nodes], [], communities={})
    assert len(result) == 1
    assert result[0].get("source_location") == "L7", "explicit-None must be filled from the loser"


# ── #2182: cross-file exact-duplicate concepts must merge ─────────────────────

def test_crossfile_identical_concepts_merge_and_rewire():
    """Two `concept` nodes whose labels are byte-identical after _norm() but
    live in different files must merge (#2182). Pass 1 used to defer them to
    Pass 2, whose norm-unique candidate filter (`seen_norms`) structurally
    cannot form an equal-norm pair — so exact cross-file duplicates were the
    one class of duplicate that never merged, while a one-char-different
    fuzzy pair did."""
    nodes = [
        {"id": "sz_intl", "label": "SHENZHEN INTERNATIONAL",
         "file_type": "concept", "source_file": "doc1.md"},
        {"id": "shenzhen_international_holdings", "label": "Shenzhen international",
         "file_type": "concept", "source_file": "doc2.md"},
        {"id": "port_ops", "label": "Port Operations",
         "file_type": "concept", "source_file": "doc2.md"},
    ]
    edges = [{"source": "shenzhen_international_holdings", "target": "port_ops",
              "relation": "operates"}]
    result_nodes, result_edges = deduplicate_entities(nodes, edges, communities={})
    ids = {n["id"] for n in result_nodes}
    assert len(result_nodes) == 2
    # _pick_winner prefers the shorter, non-chunk-suffixed id.
    assert "sz_intl" in ids
    assert "shenzhen_international_holdings" not in ids
    # The loser's edge is rewired to the winner.
    assert result_edges == [
        {"source": "sz_intl", "target": "port_ops", "relation": "operates"}]


def test_crossfile_one_char_typo_concepts_still_merge():
    """Non-regression: the near-identical (one-char-different) cross-file pair
    that already merged via Pass 2 fuzzy matching must keep merging (#2182)."""
    nodes = [
        {"id": "g1", "label": "Authentication Manager",
         "file_type": "concept", "source_file": "a.md"},
        {"id": "g2", "label": "Authentication Managr",
         "file_type": "concept", "source_file": "b.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1


_RATIONALE_BOILER = ("Django app config for apps.platform.cards. No business "
                     "logic here. Domain services live in services.py.")


@pytest.mark.parametrize("a,b", [
    # #296 narrowed the document/rationale rows here to their boundary case: an
    # identical label on two nodes that are provably NOT their files' own nodes
    # now merges (that is the bug the concept-only gate was causing). What must
    # still never merge is a file's OWN node, so these two rows now carry the
    # ids the file nodes themselves would be minted with.
    ({"id": "docs_a", "label": "Getting Started Installation Guide",
      "file_type": "document", "source_file": "docs/a.md"},
     {"id": "docs_b", "label": "Getting Started Installation Guide",
      "file_type": "document", "source_file": "docs/b.md"}),
    ({"id": "apps_platform_cards_apps", "label": _RATIONALE_BOILER,
      "file_type": "rationale", "source_file": "apps/platform/cards/apps.py"},
     {"id": "apps_platform_cores_apps", "label": _RATIONALE_BOILER,
      "file_type": "rationale", "source_file": "apps/platform/cores/apps.py"}),
    ({"id": "backend_a_render_frame", "label": "render_frame",
      "file_type": "code", "source_file": "backend_a.py"},
     {"id": "backend_b_render_frame", "label": "render_frame",
      "file_type": "code", "source_file": "backend_b.py"}),
    ({"id": "web_logo", "label": "logo.png",
      "file_type": "image", "source_file": "web/assets/logo.png"},
     {"id": "docs_logo", "label": "logo.png",
      "file_type": "image", "source_file": "docs/img/logo.png"}),
    ({"id": "logo_concept", "label": "logo.png",
      "file_type": "concept", "source_file": "doc1.md"},
     {"id": "logo_image", "label": "logo.png",
      "file_type": "image", "source_file": "assets/logo.png"}),
    ({"id": "shenzhen_a", "label": "Shenzhen International",
      "file_type": "concept", "source_file": ""},
     {"id": "shenzhen_b", "label": "Shenzhen International",
      "file_type": "concept", "source_file": ""}),
    ({"id": "api_a", "label": "API",
      "file_type": "concept", "source_file": "doc1.md"},
     {"id": "api_b", "label": "API",
      "file_type": "concept", "source_file": "doc2.md"}),
], ids=["document-own-file-node", "rationale-own-file-node", "code",
        "image-basename", "concept-image-mixed", "empty-source-file",
        "low-entropy-concept"])
def test_crossfile_identical_labels_stay_distinct_for_guarded_types(a, b):
    """The #2182 cross-file merge requires provenance and high entropy on BOTH
    sides, and #296 widened its type gate only as far as nodes provably not
    their file's own node. Identical labels must still NOT merge for: a
    document/rationale node that IS its file's own node (#296, the boundary of
    #1284's file-anchored guard), code (#1205), images sharing a basename in
    different dirs, mixed concept+image pairs, provenance-less nodes (#1178),
    and low-entropy generic labels."""
    result_nodes, _ = deduplicate_entities([dict(a), dict(b)], [], communities={})
    assert len(result_nodes) == 2, (
        f"guarded pair ({a['id']}, {b['id']}) was merged — #2182 fix leaked "
        f"past its concept-only gate"
    )


def test_cross_repo_guard_still_raises():
    """The cross-repo guard is untouched by #2182: identical concepts from
    different repos must still raise, never merge."""
    nodes = [
        {"id": "c1", "label": "Shenzhen International", "file_type": "concept",
         "source_file": "doc1.md", "repo": "repo-a"},
        {"id": "c2", "label": "Shenzhen International", "file_type": "concept",
         "source_file": "doc2.md", "repo": "repo-b"},
    ]
    with pytest.raises(ValueError, match="multiple repos"):
        deduplicate_entities(nodes, [], communities={})


def test_crossfile_concept_merge_is_order_independent():
    """Three identical-norm concepts across three files: every input order must
    yield the same single survivor (#2182). Winner ids differ in length so
    _pick_winner has a unique minimum."""
    import itertools
    base = [
        {"id": "shenzhen", "label": "SHENZHEN INTERNATIONAL",
         "file_type": "concept", "source_file": "doc1.md"},
        {"id": "shenzhen_intl", "label": "Shenzhen international",
         "file_type": "concept", "source_file": "doc2.md"},
        {"id": "shenzhen_international", "label": "shenzhen-international",
         "file_type": "concept", "source_file": "doc3.md"},
    ]
    survivors = set()
    for perm in itertools.permutations(base):
        out, _ = deduplicate_entities([dict(n) for n in perm], [], communities={})
        assert len(out) == 1
        survivors.add(out[0]["id"])
    assert survivors == {"shenzhen"}, f"non-deterministic survivor: {survivors}"


def test_crossfile_concept_merge_deterministic_across_hash_seeds():
    """#2182 determinism, #1753/#2074 precedent: the survivor must not depend on
    PYTHONHASHSEED. pytest fixes the seed per process, so run out-of-process
    with shuffled input."""
    import os
    import subprocess
    import sys
    script = (
        "import random, sys\n"
        "from graphify.dedup import deduplicate_entities\n"
        "nodes = [\n"
        "    {'id': 'shenzhen', 'label': 'SHENZHEN INTERNATIONAL',\n"
        "     'file_type': 'concept', 'source_file': 'doc1.md'},\n"
        "    {'id': 'shenzhen_intl', 'label': 'Shenzhen international',\n"
        "     'file_type': 'concept', 'source_file': 'doc2.md'},\n"
        "    {'id': 'shenzhen_international', 'label': 'shenzhen-international',\n"
        "     'file_type': 'concept', 'source_file': 'doc3.md'},\n"
        "]\n"
        "random.Random(int(sys.argv[1])).shuffle(nodes)\n"
        "out, _ = deduplicate_entities(nodes, [], communities={})\n"
        "print(len(out), sorted(n['id'] for n in out)[0])\n"
    )
    results = set()
    for seed in ("0", "1", "2", "3"):
        env = {**os.environ, "PYTHONHASHSEED": seed}
        r = subprocess.run(
            [sys.executable, "-c", script, seed],
            capture_output=True, text=True, env=env,
        )
        assert r.returncode == 0, r.stderr
        results.add(r.stdout.strip().splitlines()[-1])
    assert results == {"1 shenzhen"}, (
        f"non-deterministic dedup across hash seeds: {results}"
    )


def test_crossfile_concept_merge_is_transitive():
    """Exact cross-file matches and a punctuation variant collapse to one
    survivor: {'Acme Corp' doc1, 'Acme Corp' doc2, 'Acme Corp.' doc3} all
    normalize to 'acme corp' and must transitively union (#2182)."""
    nodes = [
        {"id": "acme_corp_one", "label": "Acme Corp",
         "file_type": "concept", "source_file": "doc1.md"},
        {"id": "acme_corp_two", "label": "Acme Corp",
         "file_type": "concept", "source_file": "doc2.md"},
        {"id": "acme_corp_three", "label": "Acme Corp.",
         "file_type": "concept", "source_file": "doc3.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1


# ── #2576: same-file labels differing by a content-word swap ──────────────────
# Guard adopted from @wilyan09007's PR #2587, hardened: any same-position
# distinct-content-word pair blocks (not just exactly one), and the differing
# tokens are judged by _same_word_variant instead of bare token-level JW.

def test_dedup_does_not_merge_samefile_sibling_pipeline_stages():
    """Sibling sections of one document, named from a template, share both
    affixes and differ in a single content word, so Jaro-Winkler's prefix bonus
    clears the threshold on the same-file path (92.74 with the bonus, 87.90 on
    plain Jaro). The absorbed node's edges are re-pointed at the survivor, so
    the graph asserts relationships the document never states (#2576)."""
    src = "docs/pipeline.md"
    nodes = [
        {"id": "pipeline_asset_contribution_flow", "label": "Asset Contribution Flow",
         "file_type": "concept", "source_file": src},
        {"id": "pipeline_asset_consumption_flow", "label": "Asset Consumption Flow",
         "file_type": "concept", "source_file": src},
        {"id": "pipeline_producer_personas", "label": "Producer Personas",
         "file_type": "concept", "source_file": src},
        {"id": "pipeline_asset_review_flow", "label": "Asset Review & Approval Flow",
         "file_type": "concept", "source_file": src},
    ]
    edges = [
        {"source": "pipeline_producer_personas",
         "target": "pipeline_asset_contribution_flow", "relation": "references"},
        {"source": "pipeline_asset_contribution_flow",
         "target": "pipeline_asset_review_flow", "relation": "references"},
    ]
    result_nodes, result_edges = deduplicate_entities(nodes, edges, communities={})
    assert len(result_nodes) == 4, (
        "contribution and consumption are distinct pipeline stages"
    )
    assert {(e["source"], e["target"]) for e in result_edges} == {
        ("pipeline_producer_personas", "pipeline_asset_contribution_flow"),
        ("pipeline_asset_contribution_flow", "pipeline_asset_review_flow"),
    }, "edges must stay on the nodes the document actually connects"


def test_dedup_still_merges_samefile_stopword_insertion():
    """The pair #1243 was scoped around: swapping a function word is how a
    restatement of the same entity differs, so a stopword in either differing
    position exempts the pair and the merge still happens (#2576)."""
    nodes = [
        {"id": "m1", "file_type": "concept", "source_file": "docs/metrics.md",
         "label": "Counts-only metrics export, a read-only aggregation service"},
        {"id": "m2", "file_type": "concept", "source_file": "docs/metrics.md",
         "label": "Counts-only metrics export, the read-only aggregation service"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1


def test_dedup_still_merges_samefile_normalization_variants():
    """Punct/case variants norm-equal in one file (pass 1), and a fused
    camelCase symbol vs its spaced spelling (token counts differ, so the
    #2576 guard defers to whole-label scoring), both still merge."""
    nodes = [
        {"id": "am1", "label": "Authentication Manager",
         "file_type": "concept", "source_file": "docs/auth.md"},
        {"id": "am2", "label": "authentication-manager",
         "file_type": "concept", "source_file": "docs/auth.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1

    nodes = [
        {"id": "g1", "label": "getUserById",
         "file_type": "concept", "source_file": "docs/api.md"},
        {"id": "g2", "label": "get user by id",
         "file_type": "concept", "source_file": "docs/api.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1


def test_dedup_still_merges_samefile_typo_in_one_token():
    """A typo inside the differing token leaves it the same word: manager and
    managr score 97.14 against each other, so the pair is not a content-word
    swap and still merges (#2576)."""
    nodes = [
        {"id": "a1", "label": "Authentication Manager",
         "file_type": "concept", "source_file": "docs/auth.md"},
        {"id": "a2", "label": "Authentication Managr",
         "file_type": "concept", "source_file": "docs/auth.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1


def test_dedup_still_merges_single_token_transposition_at_length_boundary():
    """A single-token label at the 12-char _short_label_blocked boundary with a
    trailing transposition is a typo: same length, Damerau-Levenshtein 1, so
    _same_word_variant exempts it and the merge survives (#2576)."""
    nodes = [
        {"id": "gb1", "label": "GraphBuilder",
         "file_type": "concept", "source_file": "docs/build.md"},
        {"id": "gb2", "label": "GraphBuildre",
         "file_type": "concept", "source_file": "docs/build.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1


def test_dedup_recovers_samefile_first_letter_typo():
    """DELTA over #2587: token-level JW scores manager/nanager at 84.92 (the
    prefix bonus is zero at position 0), so the PR's bare-JW token test would
    have blocked a genuine typo. The same-length Damerau-Levenshtein<=1 branch
    of _same_word_variant reads it as one word and the merge happens (#2576).
    The label carries a third token so the pair clears MinHash/LSH candidacy
    (the two-token pair's shingle Jaccard sits at 0.727, on the 0.7 blocking
    threshold, and never reaches the comparator either way)."""
    nodes = [
        {"id": "a1", "label": "Authentication Session Manager",
         "file_type": "concept", "source_file": "docs/auth.md"},
        {"id": "a2", "label": "Authentication Session Nanager",
         "file_type": "concept", "source_file": "docs/auth.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1


def test_dedup_does_not_merge_samefile_short_content_word_swap():
    """DELTA over #2587: pane/plane scores 94.0 on token-level JW, above the
    merge threshold, so the PR's bare-JW token test read two distinct short
    words as a typo. Tokens under 6 chars never pass the JW fallback (#2576)."""
    nodes = [
        {"id": "p1", "label": "User Profile Pane",
         "file_type": "concept", "source_file": "docs/ui.md"},
        {"id": "p2", "label": "User Profile Plane",
         "file_type": "concept", "source_file": "docs/ui.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2


def test_dedup_does_not_merge_two_position_content_swap():
    """DELTA over #2587: the PR guard required exactly one differing token, so
    a sibling pair that also drifts in a second position (Flow vs Flows) slid
    back to whole-label JW (95.44) and merged. Any same-position distinct
    content-word pair now blocks (#2576)."""
    nodes = [
        {"id": "h1", "label": "Customer Asset Contribution Flow Handler",
         "file_type": "concept", "source_file": "docs/handlers.md"},
        {"id": "h2", "label": "Customer Asset Consumption Flows Handler",
         "file_type": "concept", "source_file": "docs/handlers.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2


def test_dedup_does_not_merge_samefile_sibling_packages():
    """#1243's own first example, with the source file its repro used: two
    dependencies both come from one package.json, so the cross-file scoring
    change never applied to them and they still merged (#2576)."""
    nodes = [
        {"id": "jest_native", "label": "@testing-library/jest-native",
         "file_type": "concept", "source_file": "package.json"},
        {"id": "react_native", "label": "@testing-library/react-native",
         "file_type": "concept", "source_file": "package.json"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2


def test_dedup_does_not_merge_same_community_dated_doc_slugs():
    """#1243's fourth example, with the communities its repro used: the pair is
    cross-file and scores 87.68 on plain Jaro, but both nodes sit in one
    community and the +5 boost lifts it back to 92.68. Comparing the differing
    tokens (fe/mob) blocks it before the boost applies (#2576)."""
    nodes = [
        {"id": "fe_doc", "label": "2026-05-17-phase4-fe-api-integration.md",
         "file_type": "concept", "source_file": "a.md"},
        {"id": "mob_doc", "label": "2026-05-17-phase4-mob-api-integration.md",
         "file_type": "concept", "source_file": "b.md"},
    ]
    result_nodes, _ = deduplicate_entities(
        nodes, [], communities={"fe_doc": 1, "mob_doc": 1}
    )
    assert len(result_nodes) == 2


def test_content_token_swap_helper():
    """_content_token_swap fires when any same-position pair is a swap of two
    distinct content words (#2576)."""
    from graphify.dedup import _content_token_swap
    assert _content_token_swap("asset contribution flow", "asset consumption flow")
    assert _content_token_swap("testing library jest native",
                               "testing library react native")
    # A function word is what a restatement swaps, so either side exempts.
    assert not _content_token_swap("export a read only", "export the read only")
    # A typo leaves the differing token the same word.
    assert not _content_token_swap("graph extractor", "graph extractar")
    # DELTA over #2587: two differing content-word positions block too (the PR
    # exempted anything but exactly one).
    assert _content_token_swap("alpha beta gamma", "alpha delta epsilon")
    assert _content_token_swap("customer asset contribution flow handler",
                               "customer asset consumption flows handler")
    # Differing token counts belong to the prefix-extension guard.
    assert not _content_token_swap("graph extractor", "graph extractor service")
    assert not _content_token_swap("asset flow", "asset flow")


def test_same_word_variant_helper():
    """_same_word_variant separates one-word misspellings from two words
    (#2576 DELTA over #2587's bare token-JW test)."""
    from graphify.dedup import _same_word_variant
    assert _same_word_variant("manager", "managr")     # JW 97.14, min len 6
    assert _same_word_variant("manager", "nanager")    # JW 84.92 but same-len DL 1
    assert _same_word_variant("builder", "buildre")    # trailing transposition
    assert not _same_word_variant("pane", "plane")     # JW 94.0 but short
    assert not _same_word_variant("flow", "flows")     # short inflection: under-merge
    assert not _same_word_variant("contribution", "consumption")
    assert not _same_word_variant("jest", "react")
    # Accepted trade: a length-differing 5-char spelling variant reads as two
    # words and stays unmerged, per the never-merge-distinct-entities bar.
    assert not _same_word_variant("colour", "color")


# -- #296: cross-file merge for entity nodes typed by their file's extension --
#
# Reported shape: per-file extraction over a note vault mints one node per
# mention of the same person/project, and they never merge. Normalization was
# never the problem -- `_norm` already buckets the variants together -- the
# Pass 1 cross-file union was gated to `file_type == "concept"`, and an entity
# pulled out of a `.md` note inherits `document` from the file's extension, not
# from anything about the entity. The @cyrilXBT variant family below is the
# reported corpus's shape, reduced to the spellings that occur in it.

_CYRIL_VARIANTS = ["@cyrilXBT", "@cyrilxbt", "cyrilXBT"]


def test_reads_as_file_entity_helper():
    """A file's own node, a stamped `page`/`heading` node, and any node missing
    an id or provenance all stay treated as file-anchored (#296). The own-node
    half is a reconstruction and holds for every stored-path spelling; the
    structural half rests on the producer stamping `node_kind`."""
    from graphify.dedup import _reads_as_file_entity
    # An entity extracted from a note: `<path>_<entity>` never equals the path.
    assert _reads_as_file_entity(
        {"id": "journal_2024_03_01_cyrilxbt", "label": "@cyrilXBT",
         "source_file": "journal/2024-03-01.md"})
    # The file's own node, in each spelling a stored source_file may take.
    assert not _reads_as_file_entity(
        {"id": "journal_2024_03_01", "source_file": "journal/2024-03-01.md"})
    assert not _reads_as_file_entity(
        {"id": "2024_03_01", "source_file": "journal/2024-03-01.md"})  # pre-#1504 bare stem
    assert not _reads_as_file_entity(
        {"id": "vault_journal_2024_03_01", "source_file": 'C:\\vault\\journal\\2024-03-01.md'})
    # The markdown extractor states it outright, for the file and its sections.
    assert not _reads_as_file_entity(
        {"id": "anything", "node_kind": "page", "source_file": "docs/a.md"})
    assert not _reads_as_file_entity(
        {"id": "docs_a_decisions", "node_kind": "heading",
         "label": "Decisions", "source_file": "docs/a.md"})
    # Unprovable -- no id, or no provenance.
    assert not _reads_as_file_entity({"id": "", "source_file": "docs/a.md"})
    assert not _reads_as_file_entity({"id": "docs_a_thing", "source_file": ""})


def test_dedup_merges_crossfile_document_entity_variants():
    """The reported bug (#296): case/prefix variants of one entity, extracted
    from three different notes and typed `document` by extension, must collapse
    to a single node."""
    nodes = [
        {"id": "journal_2024_03_0%d_cyrilxbt" % i, "label": variant,
         "file_type": "document", "source_file": "journal/2024-03-0%d.md" % i}
        for i, variant in enumerate(_CYRIL_VARIANTS, start=1)
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1, (
        "cross-file `document` entity variants did not merge -- the #296 gate "
        "widening is not reaching Pass 1's cross-file residue"
    )


def test_dedup_merges_crossfile_rationale_entity_variants():
    """`rationale` rides the same gate as `document` (#296): entity nodes of
    that type, provably not their files' own nodes, merge on an exact label."""
    nodes = [
        {"id": "svc_alpha_py_retention_window", "label": "Retention Window",
         "file_type": "rationale", "source_file": "svc/alpha.py"},
        {"id": "svc_beta_py_retention_window", "label": "retention window",
         "file_type": "rationale", "source_file": "svc/beta.py"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 1


def test_dedup_never_merges_a_files_own_node_away():
    """The boundary the widened gate is defined against (#296): a curated note
    whose OWN node carries the entity's label must survive as its own node, even
    when a same-label entity node exists in another file."""
    nodes = [
        # The curated page: its id is exactly the slugified source path.
        {"id": "people_cyrilxbt", "label": "@cyrilXBT",
         "file_type": "document", "source_file": "people/@cyrilXBT.md"},
        # A mention of the same entity, extracted from a journal note.
        {"id": "journal_2024_03_01_cyrilxbt", "label": "cyrilXBT",
         "file_type": "document", "source_file": "journal/2024-03-01.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2, (
        "a file's own node merged away -- the #296 widening must never reach it"
    )
    assert {n["id"] for n in result_nodes} == {
        "people_cyrilxbt", "journal_2024_03_01_cyrilxbt"}


def test_dedup_never_merges_two_files_own_nodes():
    """Two README.md in different folders are genuinely two documents (#1284's
    original reasoning), and stay two under #296."""
    nodes = [
        {"id": "web_readme", "label": "README.md", "file_type": "document",
         "node_kind": "page", "source_file": "web/README.md"},
        {"id": "api_readme", "label": "README.md", "file_type": "document",
         "node_kind": "page", "source_file": "api/README.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2


def test_dedup_crossfile_entity_merge_keeps_the_entropy_gate():
    """Entropy guard untouched (#296): a short generic label stays distinct for
    `document` entity nodes exactly as it does for `concept`."""
    nodes = [
        {"id": "docs_a_api", "label": "API", "file_type": "document",
         "source_file": "docs/a.md"},
        {"id": "docs_b_api", "label": "API", "file_type": "document",
         "source_file": "docs/b.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2


def test_dedup_crossfile_entity_merge_keeps_the_provenance_gate():
    """Provenance guard untouched (#296, #1178): without a source_file the node
    cannot be proven to be an entity, so it stays out of the merge."""
    nodes = [
        {"id": "orphan_cyrilxbt", "label": "@cyrilXBT",
         "file_type": "document", "source_file": ""},
        {"id": "journal_2024_03_01_cyrilxbt", "label": "cyrilXBT",
         "file_type": "document", "source_file": "journal/2024-03-01.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2


def test_dedup_crossfile_fuzzy_fileanchored_block_is_untouched():
    """#296 widens only the exact-normalization pass. Pass 2's fuzzy
    `_crossfile_fileanchored_blocked` is unchanged, so near-identical (not
    identical) document labels in different files still stay distinct -- the
    #1284 guard keeps doing its job on entity nodes too."""
    nodes = [
        {"id": "docs_a_guide", "label": "Getting Started Installation Guide",
         "file_type": "document", "source_file": "docs/a.md"},
        {"id": "docs_b_setup", "label": "Getting Started Installation Setup",
         "file_type": "document", "source_file": "docs/b.md"},
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 2


def test_dedup_never_merges_repeated_headings_across_files():
    """#3094's verification case must keep holding under #296: sibling documents
    that each carry the same `## Decisions` / `## Next steps` sections keep one
    heading node per file, attributed to that file."""
    nodes = [
        {"id": "docs_%s_%s" % (doc, slug), "label": heading,
         "file_type": "document", "node_kind": "heading",
         "source_file": "docs/%s.md" % doc}
        for doc in ("a", "b", "c")
        for slug, heading in (("decisions", "Decisions"),
                              ("next_steps", "Next steps"))
    ]
    result_nodes, _ = deduplicate_entities(nodes, [], communities={})
    assert len(result_nodes) == 6, (
        "repeated headings merged across files -- #296 must not reach section "
        "nodes (#1284, #3094)"
    )
    assert {n["source_file"] for n in result_nodes} == {
        "docs/a.md", "docs/b.md", "docs/c.md"}


def test_reads_as_file_entity_trusts_the_node_kind_stamp_only():
    """The documented limit of the structural half (#296): `node_kind` is what
    marks a sub-file node, so an UNSTAMPED structural node reads as an entity.
    Pinned deliberately — the fix for such a producer is to stamp `node_kind`,
    and this test is what fails if the predicate is ever quietly changed to
    guess instead."""
    from graphify.dedup import _reads_as_file_entity
    stamped = {"id": "book_xlsx_summary", "label": "Summary (sheet)",
               "node_kind": "heading", "source_file": "book.xlsx"}
    unstamped = dict(stamped)
    del unstamped["node_kind"]
    assert not _reads_as_file_entity(stamped)
    assert _reads_as_file_entity(unstamped)