From 8bc477f2e2728782f0f42f29eea2bb8adb6cd992 Mon Sep 17 00:00:00 2001 From: safishamsi Date: Fri, 17 Jul 2026 00:09:02 +0100 Subject: [PATCH] fix(llm): move the unverifiable-node flag to a real, consumed field (follow-up to #1949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PR flagged unverifiable code nodes as confidence="UNVERIFIED", but node confidence is read by nothing and UNVERIFIED isn't in the validated (edge-only) confidence vocabulary {EXTRACTED,INFERRED,AMBIGUOUS} — a dead, colliding field. - Move the flag to a dedicated verification="unverified" node field, off the confidence key. A node the model already hedged (INFERRED/AMBIGUOUS) is left untouched, as before. - Also check the node id's identifiers (not just the label): ids carry the verbatim symbol, cutting false flags on prettified labels. - Skip the (potentially PDF-re-extracting) source read entirely when a result has no code-typed node with a source_file. - Wire a consumer: diagnose_extraction now counts and reports unverified nodes, so the persisted flag is actually surfaced. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/diagnostics.py | 10 ++++++ graphify/llm.py | 46 ++++++++++++++++--------- tests/test_evidence_binding.py | 61 +++++++++++++++++++++++----------- 3 files changed, 82 insertions(+), 35 deletions(-) diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index ff66fa95..fcb9a11c 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -168,6 +168,14 @@ def diagnose_extraction( raw_edges = _edge_list(extraction) canonical_edges = [_canonical_edge(edge) for edge in raw_edges] + # Code-typed semantic nodes the extractor could not verify against the source + # it read (#1949): likely-inferred (or hallucinated) symbols surfaced from a + # document. Count them so the flag on graph.json nodes is actually surfaced. + unverified_node_count = sum( + 1 for n in extraction.get("nodes", []) + if isinstance(n, dict) and n.get("verification") == "unverified" + ) + exact_counts: Counter[str] = Counter(_exact_signature(edge) for edge in raw_edges) directed_pairs: Counter[tuple[str, str]] = Counter() undirected_pairs: Counter[tuple[str, str]] = Counter() @@ -239,6 +247,7 @@ def diagnose_extraction( return { "node_count": len(node_ids), + "unverified_node_count": unverified_node_count, "raw_edge_count": len(raw_edges), "non_object_edges": non_object_edges, "missing_endpoint_edges": missing_endpoint_edges, @@ -344,6 +353,7 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: "input_stage: provided JSON (normal graph.json is post-build)", f"effective_directed: {summary.get('effective_directed', '')}", f"nodes: {summary['node_count']}", + f"unverified_code_nodes: {summary.get('unverified_node_count', 0)}", f"raw_edges: {summary['raw_edge_count']}", f"valid_candidate_edges: {summary['valid_candidate_edges']}", f"missing_endpoint_edges: {summary['missing_endpoint_edges']}", diff --git a/graphify/llm.py b/graphify/llm.py index e94e2001..ef094b53 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -565,13 +565,18 @@ def _read_files(units: "list[Path | FileSlice]", root: Path) -> str: # `_out_of_scope` (#1895) only rejects a node attributed to a real file that was # NOT dispatched; a fabricated symbol attributed to a file that WAS dispatched # slips through it. This closes that intra-file gap with a lenient substring -# check and DOWNGRADES (never drops) an unverifiable node's confidence to -# UNVERIFIED, surfaced by the caller (stderr) and left on the node in graph.json. +# check and FLAGS (never drops) an unverifiable node with ``verification = +# "unverified"``, surfaced by the caller (stderr), reported by the diagnostics, +# and left on the node in graph.json. # Short tokens (len < 3) are ignored: they match too readily to be evidence and # their absence is not a reliable fabrication signal, so skipping them avoids # false positives. _LABEL_IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") -_UNVERIFIED_CONFIDENCE = "UNVERIFIED" +# A dedicated node field — deliberately NOT the ``confidence`` key, whose +# validated vocabulary ({EXTRACTED, INFERRED, AMBIGUOUS}, and only on edges) +# this value does not belong to. Downstream (diagnostics) counts it. +_VERIFICATION_FIELD = "verification" +_UNVERIFIED_VALUE = "unverified" def _label_identifiers(label: str) -> list[str]: @@ -611,24 +616,33 @@ def _bind_node_evidence(result: dict, text_units: "list[Path | FileSlice]", root For every ``file_type == "code"`` node whose ``source_file`` resolves to one of the (document/paper/image) files sent in THIS call, verify that at least - one identifier from its label occurs in that file's source bytes. If none - does, mark the node ``confidence = "UNVERIFIED"`` rather than dropping it. + one identifier from its label OR id occurs in that file's source bytes. If + none does, set ``verification = "unverified"`` rather than dropping it. Precision-first, to avoid false-positives on legitimately-derived names: - Only ``code`` nodes are checked — code labels are verbatim symbol names, whereas document/paper/concept labels are prose and would false-positive. + - Both the label AND the id are checked: the id (``stem_entityname``) + usually carries the verbatim symbol even when the label is prettified, + cutting false flags on human-readable labels. - Nodes without a ``source_file``, and nodes attributed to a file not dispatched in this call (left to #1895), are never touched. - - Verification is lenient: any label identifier occurring as a substring + - Verification is lenient: any identifier occurring as a substring (case-insensitive) passes; a node is flagged only when NONE occur. - - A label with no checkable identifier (all short / non-ASCII) is left as-is. - - The action is a reversible confidence downgrade, never a drop. A code - symbol a document only describes in prose (no verbatim occurrence) is - legitimately UNVERIFIED — the model inferred it rather than read it. + - A node with no checkable identifier (all short / non-ASCII) is left as-is. + - The action is a reversible flag, never a drop. A code symbol a document + only describes in prose (no verbatim occurrence) is legitimately + unverified — the model inferred it rather than read it. """ nodes = result.get("nodes") if not nodes: return 0 + # Perf: skip the (potentially expensive, e.g. PDF re-extraction) source read + # entirely when the result has no code-typed node with a source_file — the + # common case for a document/paper batch. + if not any(isinstance(n, dict) and n.get("file_type") == "code" and n.get("source_file") + for n in nodes): + return 0 source_by_path = _dispatched_source_text(text_units, root) if not source_by_path: return 0 @@ -649,14 +663,16 @@ def _bind_node_evidence(result: dict, text_units: "list[Path | FileSlice]", root src = source_by_path.get(key) if src is None: continue # not dispatched in this call — #1895's out-of-scope domain - idents = _label_identifiers(str(n.get("label", ""))) + idents = _label_identifiers(str(n.get("label", ""))) + _label_identifiers(str(n.get("id", ""))) if not idents: continue # nothing checkable — do not flag if any(ident.lower() in src for ident in idents): continue # symbol name is present in the source — verified - # No evidence: downgrade unless the model already flagged it lower. - if n.get("confidence") in (None, "", "EXTRACTED"): - n["confidence"] = _UNVERIFIED_CONFIDENCE + # No evidence. Flag only a node the model itself presented as solid + # (EXTRACTED/unset) — one it already hedged (INFERRED/AMBIGUOUS) needs no + # second flag. Idempotent: never overwrites an existing verification. + if n.get("confidence") in (None, "", "EXTRACTED") and not n.get(_VERIFICATION_FIELD): + n[_VERIFICATION_FIELD] = _UNVERIFIED_VALUE downgraded += 1 return downgraded @@ -1638,7 +1654,7 @@ def extract_files_direct( if _n_unverified: print( f"[graphify] {_n_unverified} semantic node(s) had no evidence in " - "the source and were marked confidence=UNVERIFIED", + "the source and were flagged verification=unverified", file=sys.stderr, ) except Exception as _exc: # noqa: BLE001 — evidence-binding is advisory diff --git a/tests/test_evidence_binding.py b/tests/test_evidence_binding.py index 2be1e614..6df06818 100644 --- a/tests/test_evidence_binding.py +++ b/tests/test_evidence_binding.py @@ -1,7 +1,7 @@ """Tests for semantic evidence-binding in graphify.llm. A code node the model returns whose symbol name has no evidence in the dispatched -source is downgraded to ``confidence = "UNVERIFIED"`` (never dropped). This closes +source is flagged ``verification = "unverified"`` (never dropped). This closes the intra-file hallucination gap that ``_out_of_scope`` (#1895) — which only rejects nodes attributed to a file that was NOT dispatched — cannot see. """ @@ -50,11 +50,11 @@ def test_fabricated_code_symbol_is_downgraded(tmp_path): ] out = _by_label(_run([src], nodes, tmp_path)) # The fabricated symbol has no evidence in the source -> flagged. - assert out["totally_fabricated_symbol()"]["confidence"] == "UNVERIFIED" + assert out["totally_fabricated_symbol()"]["verification"] == "unverified" # A symbol that IS in the source is verified -> untouched (no confidence key). - assert "confidence" not in out["real_function()"] + assert "verification" not in out["real_function()"] # A concept node is prose, never checked. - assert "confidence" not in out["Payments Overview"] + assert "verification" not in out["Payments Overview"] def test_qualified_and_prettified_labels_do_not_false_positive(tmp_path): @@ -66,8 +66,8 @@ def test_qualified_and_prettified_labels_do_not_false_positive(tmp_path): ] out = _by_label(_run([src], nodes, tmp_path)) # Any label identifier present in the source verifies the whole label. - assert "confidence" not in out["PaymentProcessor.charge_card()"] - assert "confidence" not in out["charge_card(amount, token)"] + assert "verification" not in out["PaymentProcessor.charge_card()"] + assert "verification" not in out["charge_card(amount, token)"] def test_document_and_sourceless_nodes_are_never_flagged(tmp_path): @@ -78,8 +78,8 @@ def test_document_and_sourceless_nodes_are_never_flagged(tmp_path): {"id": "b", "label": "orphan_symbol()", "file_type": "code"}, # no source_file ] out = _by_label(_run([src], nodes, tmp_path)) - assert "confidence" not in out["Nonexistent Heading"] - assert "confidence" not in out["orphan_symbol()"] + assert "verification" not in out["Nonexistent Heading"] + assert "verification" not in out["orphan_symbol()"] def test_node_attributed_to_undispatched_file_is_left_to_out_of_scope(tmp_path): @@ -92,7 +92,7 @@ def test_node_attributed_to_undispatched_file_is_left_to_out_of_scope(tmp_path): ] out = _by_label(_run([src], nodes, tmp_path)) # Not in the dispatched set -> #1895's domain, not evidence-binding's. - assert "confidence" not in out["ghost_func()"] + assert "verification" not in out["ghost_func()"] def test_uncheckable_short_label_is_not_flagged(tmp_path): @@ -103,7 +103,7 @@ def test_uncheckable_short_label_is_not_flagged(tmp_path): ] out = _by_label(_run([src], nodes, tmp_path)) # "id" is < 3 chars, so there is no checkable identifier -> leave as-is. - assert "confidence" not in out["id()"] + assert "verification" not in out["id()"] def test_existing_lower_confidence_is_not_overwritten(tmp_path): @@ -113,8 +113,10 @@ def test_existing_lower_confidence_is_not_overwritten(tmp_path): {"id": "a", "label": "made_up()", "file_type": "code", "source_file": "mod.py", "confidence": "INFERRED"}, ] out = _by_label(_run([src], nodes, tmp_path)) - # The model already flagged it lower; the downgrade never clobbers that. + # The model already hedged it (INFERRED); evidence-binding leaves it entirely + # alone — no verification flag added, and its confidence is never clobbered. assert out["made_up()"]["confidence"] == "INFERRED" + assert "verification" not in out["made_up()"] def test_label_identifiers_helper(): @@ -156,8 +158,8 @@ def test_evidence_binding_handles_file_slice(tmp_path): n = llm._bind_node_evidence(result, [fs], tmp_path) by = {x["label"]: x for x in result["nodes"]} assert n == 1 - assert "confidence" not in by["real_function()"] - assert by["ghost_symbol()"]["confidence"] == "UNVERIFIED" + assert "verification" not in by["real_function()"] + assert by["ghost_symbol()"]["verification"] == "unverified" def test_evidence_binding_handles_absolute_source_file(tmp_path): @@ -169,7 +171,7 @@ def test_evidence_binding_handles_absolute_source_file(tmp_path): ]} n = llm._bind_node_evidence(result, [src], tmp_path) assert n == 1 - assert result["nodes"][0]["confidence"] == "UNVERIFIED" + assert result["nodes"][0]["verification"] == "unverified" def test_downgrade_emits_stderr_summary(tmp_path, capsys): @@ -178,18 +180,37 @@ def test_downgrade_emits_stderr_summary(tmp_path, capsys): nodes = [{"id": "b", "label": "totally_made_up_symbol()", "file_type": "code", "source_file": "mod.py"}] _run([src], nodes, tmp_path) err = capsys.readouterr().err - assert "UNVERIFIED" in err + assert "unverified" in err -def test_unverified_confidence_does_not_fail_validation(): - # The downgrade must never make an otherwise-valid node fail validation - # (node-level confidence is not part of the validated schema). +def test_unverified_flag_does_not_fail_validation(): + # The flag lives on its own ``verification`` field, deliberately NOT on the + # validated ``confidence`` key (whose vocabulary is edge-only), so it must + # never make an otherwise-valid node fail validation. from graphify.validate import validate_extraction extraction = { "nodes": [{"id": "n1", "label": "foo", "file_type": "code", - "source_file": "a.md", "confidence": "UNVERIFIED"}], + "source_file": "a.md", "verification": "unverified"}], "edges": [], } errors = validate_extraction(extraction) - assert not any("confidence" in str(e).lower() for e in errors) + assert not any("verification" in str(e).lower() for e in errors) + + +def test_diagnostics_reports_unverified_node_count(): + # The consumer: diagnose_extraction surfaces the persisted verification flag + # so it is not a dead field. + from graphify.diagnostics import diagnose_extraction, format_diagnostic_report + + extraction = { + "nodes": [ + {"id": "n1", "label": "ghost", "file_type": "code", + "source_file": "a.md", "verification": "unverified"}, + {"id": "n2", "label": "real", "file_type": "code", "source_file": "a.md"}, + ], + "edges": [], + } + summary = diagnose_extraction(extraction) + assert summary["unverified_node_count"] == 1 + assert "unverified_code_nodes: 1" in format_diagnostic_report(summary)