"""Tests for graphify/dedup.py entity deduplication pipeline.""" from __future__ import annotations import pytest from graphify.dedup import deduplicate_entities, _entropy, _shingles # ── entropy gate ───────────────────────────────────────────────────────────── def test_entropy_short_label_low(): assert _entropy("AI") < 2.5 def test_entropy_normal_label_high(): assert _entropy("AuthenticationManager") >= 2.5 def test_entropy_empty_string(): assert _entropy("") == 0.0 # ── shingles ───────────────────────────────────────────────────────────────── def test_shingles_produces_trigrams(): s = _shingles("hello") assert "hel" in s assert "ell" in s assert "llo" in s def test_shingles_short_string(): # strings shorter than 3 chars return single shingle of the string itself assert _shingles("ab") == {"ab"} # ── full pipeline ───────────────────────────────────────────────────────────── def _make_nodes(*labels): return [{"id": label.lower().replace(" ", "_"), "label": label, "source_file": "test.md"} for label in labels] def _make_edges(src, tgt, relation="relates_to"): return [{"source": src, "target": tgt, "relation": relation}] def test_exact_duplicates_merged(): nodes = _make_nodes("UserService", "userservice", "User Service") edges = [] result_nodes, result_edges = deduplicate_entities(nodes, edges, communities={}) # All three are the same concept — only one survives assert len(result_nodes) == 1 def test_typo_merged(): # "GraphExtractor" vs "Graph Extractor" — Jaro-Winkler >= 0.92 nodes = _make_nodes("GraphExtractor", "Graph Extractor") edges = [] result_nodes, _ = deduplicate_entities(nodes, edges, communities={}) assert len(result_nodes) == 1 def test_unrelated_not_merged(): nodes = _make_nodes("UserService", "OrderService") edges = [] result_nodes, _ = deduplicate_entities(nodes, edges, communities={}) assert len(result_nodes) == 2 def test_short_low_entropy_not_merged(): # "AI" and "ML" are low-entropy — entropy gate skips them nodes = _make_nodes("AI", "ML") edges = [] result_nodes, _ = deduplicate_entities(nodes, edges, communities={}) assert len(result_nodes) == 2 def test_edges_rewired_after_merge(): nodes = _make_nodes("GraphExtractor", "Graph Extractor", "Parser") # edge from loser to Parser should be rewired to winner edges = [{"source": "graph_extractor", "target": "parser", "relation": "uses"}] result_nodes, result_edges = deduplicate_entities(nodes, edges, communities={}) assert len(result_nodes) == 2 # merged + Parser # edge should still exist (rewired to winner) assert len(result_edges) == 1 def test_self_loops_dropped_after_merge(): # If both endpoints of an edge get merged into same node, drop the edge nodes = _make_nodes("GraphExtractor", "Graph Extractor") edges = [{"source": "graphextractor", "target": "graph_extractor", "relation": "same"}] _, result_edges = deduplicate_entities(nodes, edges, communities={}) assert result_edges == [] def test_community_boost_aids_merge(): # Two nodes in same community with score in 0.75-0.85 zone get boosted nodes = _make_nodes("AuthManager", "Auth Manager") edges = [] # Same community → boost → merge communities = {"authmanager": 1, "auth_manager": 1} result_with, _ = deduplicate_entities(nodes, edges, communities=communities) # Different community → no boost communities_diff = {"authmanager": 1, "auth_manager": 2} result_without, _ = deduplicate_entities(nodes, edges, communities=communities_diff) assert len(result_with) <= len(result_without) def test_empty_inputs(): result_nodes, result_edges = deduplicate_entities([], [], communities={}) assert result_nodes == [] assert result_edges == [] def test_single_node_no_crash(): nodes = _make_nodes("UserService") result_nodes, _ = deduplicate_entities(nodes, [], communities={}) assert len(result_nodes) == 1 def test_dedup_llm_flag_accepted(): """deduplicate_entities accepts dedup_llm_backend without crashing when no ambiguous pairs exist.""" nodes = _make_nodes("UserService", "OrderService") edges = [] result_nodes, _ = deduplicate_entities(nodes, edges, communities={}, dedup_llm_backend=None) assert len(result_nodes) == 2 # ── build integration ───────────────────────────────────────────────────────── def test_build_calls_dedup(): """build() should deduplicate near-identical nodes across extractions.""" from graphify.build import build chunk1 = { "nodes": [{"id": "graphextractor", "label": "GraphExtractor", "source_file": "a.py"}], "edges": [], } chunk2 = { "nodes": [{"id": "graph_extractor", "label": "Graph Extractor", "source_file": "b.py"}], "edges": [], } G = build([chunk1, chunk2]) assert G.number_of_nodes() == 1