fix(llm): move the unverifiable-node flag to a real, consumed field (follow-up to #1949)

The PR flagged unverifiable code nodes as confidence="UNVERIFIED", but node
confidence is read by nothing and UNVERIFIED isn't in the validated (edge-only)
confidence vocabulary {EXTRACTED,INFERRED,AMBIGUOUS} — a dead, colliding field.

- Move the flag to a dedicated verification="unverified" node field, off the
  confidence key. A node the model already hedged (INFERRED/AMBIGUOUS) is left
  untouched, as before.
- Also check the node id's identifiers (not just the label): ids carry the
  verbatim symbol, cutting false flags on prettified labels.
- Skip the (potentially PDF-re-extracting) source read entirely when a result
  has no code-typed node with a source_file.
- Wire a consumer: diagnose_extraction now counts and reports unverified nodes,
  so the persisted flag is actually surfaced.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

graphify/diagnostics.py

@@ -168,6 +168,14 @@ def diagnose_extraction(
     raw_edges = _edge_list(extraction)
     canonical_edges = [_canonical_edge(edge) for edge in raw_edges]
 
+    # Code-typed semantic nodes the extractor could not verify against the source
+    # it read (#1949): likely-inferred (or hallucinated) symbols surfaced from a
+    # document. Count them so the flag on graph.json nodes is actually surfaced.
+    unverified_node_count = sum(
+        1 for n in extraction.get("nodes", [])
+        if isinstance(n, dict) and n.get("verification") == "unverified"
+    )
+
     exact_counts: Counter[str] = Counter(_exact_signature(edge) for edge in raw_edges)
     directed_pairs: Counter[tuple[str, str]] = Counter()
     undirected_pairs: Counter[tuple[str, str]] = Counter()
@@ -239,6 +247,7 @@ def diagnose_extraction(
 
     return {
         "node_count": len(node_ids),
+        "unverified_node_count": unverified_node_count,
         "raw_edge_count": len(raw_edges),
         "non_object_edges": non_object_edges,
         "missing_endpoint_edges": missing_endpoint_edges,
@@ -344,6 +353,7 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str:
         "input_stage: provided JSON (normal graph.json is post-build)",
         f"effective_directed: {summary.get('effective_directed', '<direct-call>')}",
         f"nodes: {summary['node_count']}",
+        f"unverified_code_nodes: {summary.get('unverified_node_count', 0)}",
         f"raw_edges: {summary['raw_edge_count']}",
         f"valid_candidate_edges: {summary['valid_candidate_edges']}",
         f"missing_endpoint_edges: {summary['missing_endpoint_edges']}",

graphify/llm.py

@@ -565,13 +565,18 @@ def _read_files(units: "list[Path | FileSlice]", root: Path) -> str:
 # `_out_of_scope` (#1895) only rejects a node attributed to a real file that was
 # NOT dispatched; a fabricated symbol attributed to a file that WAS dispatched
 # slips through it. This closes that intra-file gap with a lenient substring
-# check and DOWNGRADES (never drops) an unverifiable node's confidence to
-# UNVERIFIED, surfaced by the caller (stderr) and left on the node in graph.json.
+# check and FLAGS (never drops) an unverifiable node with ``verification =
+# "unverified"``, surfaced by the caller (stderr), reported by the diagnostics,
+# and left on the node in graph.json.
 # Short tokens (len < 3) are ignored: they match too readily to be evidence and
 # their absence is not a reliable fabrication signal, so skipping them avoids
 # false positives.
 _LABEL_IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
-_UNVERIFIED_CONFIDENCE = "UNVERIFIED"
+# A dedicated node field — deliberately NOT the ``confidence`` key, whose
+# validated vocabulary ({EXTRACTED, INFERRED, AMBIGUOUS}, and only on edges)
+# this value does not belong to. Downstream (diagnostics) counts it.
+_VERIFICATION_FIELD = "verification"
+_UNVERIFIED_VALUE = "unverified"
 
 
 def _label_identifiers(label: str) -> list[str]:
@@ -611,24 +616,33 @@ def _bind_node_evidence(result: dict, text_units: "list[Path | FileSlice]", root
 
     For every ``file_type == "code"`` node whose ``source_file`` resolves to one
     of the (document/paper/image) files sent in THIS call, verify that at least
-    one identifier from its label occurs in that file's source bytes. If none
-    does, mark the node ``confidence = "UNVERIFIED"`` rather than dropping it.
+    one identifier from its label OR id occurs in that file's source bytes. If
+    none does, set ``verification = "unverified"`` rather than dropping it.
 
     Precision-first, to avoid false-positives on legitimately-derived names:
       - Only ``code`` nodes are checked — code labels are verbatim symbol names,
         whereas document/paper/concept labels are prose and would false-positive.
+      - Both the label AND the id are checked: the id (``stem_entityname``)
+        usually carries the verbatim symbol even when the label is prettified,
+        cutting false flags on human-readable labels.
       - Nodes without a ``source_file``, and nodes attributed to a file not
         dispatched in this call (left to #1895), are never touched.
-      - Verification is lenient: any label identifier occurring as a substring
+      - Verification is lenient: any identifier occurring as a substring
         (case-insensitive) passes; a node is flagged only when NONE occur.
-      - A label with no checkable identifier (all short / non-ASCII) is left as-is.
-      - The action is a reversible confidence downgrade, never a drop. A code
-        symbol a document only describes in prose (no verbatim occurrence) is
-        legitimately UNVERIFIED — the model inferred it rather than read it.
+      - A node with no checkable identifier (all short / non-ASCII) is left as-is.
+      - The action is a reversible flag, never a drop. A code symbol a document
+        only describes in prose (no verbatim occurrence) is legitimately
+        unverified — the model inferred it rather than read it.
     """
     nodes = result.get("nodes")
     if not nodes:
         return 0
+    # Perf: skip the (potentially expensive, e.g. PDF re-extraction) source read
+    # entirely when the result has no code-typed node with a source_file — the
+    # common case for a document/paper batch.
+    if not any(isinstance(n, dict) and n.get("file_type") == "code" and n.get("source_file")
+               for n in nodes):
+        return 0
     source_by_path = _dispatched_source_text(text_units, root)
     if not source_by_path:
         return 0
@@ -649,14 +663,16 @@ def _bind_node_evidence(result: dict, text_units: "list[Path | FileSlice]", root
         src = source_by_path.get(key)
         if src is None:
             continue  # not dispatched in this call — #1895's out-of-scope domain
-        idents = _label_identifiers(str(n.get("label", "")))
+        idents = _label_identifiers(str(n.get("label", ""))) + _label_identifiers(str(n.get("id", "")))
         if not idents:
             continue  # nothing checkable — do not flag
         if any(ident.lower() in src for ident in idents):
             continue  # symbol name is present in the source — verified
-        # No evidence: downgrade unless the model already flagged it lower.
-        if n.get("confidence") in (None, "", "EXTRACTED"):
-            n["confidence"] = _UNVERIFIED_CONFIDENCE
+        # No evidence. Flag only a node the model itself presented as solid
+        # (EXTRACTED/unset) — one it already hedged (INFERRED/AMBIGUOUS) needs no
+        # second flag. Idempotent: never overwrites an existing verification.
+        if n.get("confidence") in (None, "", "EXTRACTED") and not n.get(_VERIFICATION_FIELD):
+            n[_VERIFICATION_FIELD] = _UNVERIFIED_VALUE
             downgraded += 1
     return downgraded
 
@@ -1638,7 +1654,7 @@ def extract_files_direct(
             if _n_unverified:
                 print(
                     f"[graphify] {_n_unverified} semantic node(s) had no evidence in "
-                    "the source and were marked confidence=UNVERIFIED",
+                    "the source and were flagged verification=unverified",
                     file=sys.stderr,
                 )
         except Exception as _exc:  # noqa: BLE001 — evidence-binding is advisory

tests/test_evidence_binding.py

@@ -1,7 +1,7 @@
 """Tests for semantic evidence-binding in graphify.llm.
 
 A code node the model returns whose symbol name has no evidence in the dispatched
-source is downgraded to ``confidence = "UNVERIFIED"`` (never dropped). This closes
+source is flagged ``verification = "unverified"`` (never dropped). This closes
 the intra-file hallucination gap that ``_out_of_scope`` (#1895) — which only
 rejects nodes attributed to a file that was NOT dispatched — cannot see.
 """
@@ -50,11 +50,11 @@ def test_fabricated_code_symbol_is_downgraded(tmp_path):
     ]
     out = _by_label(_run([src], nodes, tmp_path))
     # The fabricated symbol has no evidence in the source -> flagged.
-    assert out["totally_fabricated_symbol()"]["confidence"] == "UNVERIFIED"
+    assert out["totally_fabricated_symbol()"]["verification"] == "unverified"
     # A symbol that IS in the source is verified -> untouched (no confidence key).
-    assert "confidence" not in out["real_function()"]
+    assert "verification" not in out["real_function()"]
     # A concept node is prose, never checked.
-    assert "confidence" not in out["Payments Overview"]
+    assert "verification" not in out["Payments Overview"]
 
 
 def test_qualified_and_prettified_labels_do_not_false_positive(tmp_path):
@@ -66,8 +66,8 @@ def test_qualified_and_prettified_labels_do_not_false_positive(tmp_path):
     ]
     out = _by_label(_run([src], nodes, tmp_path))
     # Any label identifier present in the source verifies the whole label.
-    assert "confidence" not in out["PaymentProcessor.charge_card()"]
-    assert "confidence" not in out["charge_card(amount, token)"]
+    assert "verification" not in out["PaymentProcessor.charge_card()"]
+    assert "verification" not in out["charge_card(amount, token)"]
 
 
 def test_document_and_sourceless_nodes_are_never_flagged(tmp_path):
@@ -78,8 +78,8 @@ def test_document_and_sourceless_nodes_are_never_flagged(tmp_path):
         {"id": "b", "label": "orphan_symbol()", "file_type": "code"},  # no source_file
     ]
     out = _by_label(_run([src], nodes, tmp_path))
-    assert "confidence" not in out["Nonexistent Heading"]
-    assert "confidence" not in out["orphan_symbol()"]
+    assert "verification" not in out["Nonexistent Heading"]
+    assert "verification" not in out["orphan_symbol()"]
 
 
 def test_node_attributed_to_undispatched_file_is_left_to_out_of_scope(tmp_path):
@@ -92,7 +92,7 @@ def test_node_attributed_to_undispatched_file_is_left_to_out_of_scope(tmp_path):
     ]
     out = _by_label(_run([src], nodes, tmp_path))
     # Not in the dispatched set -> #1895's domain, not evidence-binding's.
-    assert "confidence" not in out["ghost_func()"]
+    assert "verification" not in out["ghost_func()"]
 
 
 def test_uncheckable_short_label_is_not_flagged(tmp_path):
@@ -103,7 +103,7 @@ def test_uncheckable_short_label_is_not_flagged(tmp_path):
     ]
     out = _by_label(_run([src], nodes, tmp_path))
     # "id" is < 3 chars, so there is no checkable identifier -> leave as-is.
-    assert "confidence" not in out["id()"]
+    assert "verification" not in out["id()"]
 
 
 def test_existing_lower_confidence_is_not_overwritten(tmp_path):
@@ -113,8 +113,10 @@ def test_existing_lower_confidence_is_not_overwritten(tmp_path):
         {"id": "a", "label": "made_up()", "file_type": "code", "source_file": "mod.py", "confidence": "INFERRED"},
     ]
     out = _by_label(_run([src], nodes, tmp_path))
-    # The model already flagged it lower; the downgrade never clobbers that.
+    # The model already hedged it (INFERRED); evidence-binding leaves it entirely
+    # alone — no verification flag added, and its confidence is never clobbered.
     assert out["made_up()"]["confidence"] == "INFERRED"
+    assert "verification" not in out["made_up()"]
 
 
 def test_label_identifiers_helper():
@@ -156,8 +158,8 @@ def test_evidence_binding_handles_file_slice(tmp_path):
     n = llm._bind_node_evidence(result, [fs], tmp_path)
     by = {x["label"]: x for x in result["nodes"]}
     assert n == 1
-    assert "confidence" not in by["real_function()"]
-    assert by["ghost_symbol()"]["confidence"] == "UNVERIFIED"
+    assert "verification" not in by["real_function()"]
+    assert by["ghost_symbol()"]["verification"] == "unverified"
 
 
 def test_evidence_binding_handles_absolute_source_file(tmp_path):
@@ -169,7 +171,7 @@ def test_evidence_binding_handles_absolute_source_file(tmp_path):
     ]}
     n = llm._bind_node_evidence(result, [src], tmp_path)
     assert n == 1
-    assert result["nodes"][0]["confidence"] == "UNVERIFIED"
+    assert result["nodes"][0]["verification"] == "unverified"
 
 
 def test_downgrade_emits_stderr_summary(tmp_path, capsys):
@@ -178,18 +180,37 @@ def test_downgrade_emits_stderr_summary(tmp_path, capsys):
     nodes = [{"id": "b", "label": "totally_made_up_symbol()", "file_type": "code", "source_file": "mod.py"}]
     _run([src], nodes, tmp_path)
     err = capsys.readouterr().err
-    assert "UNVERIFIED" in err
+    assert "unverified" in err
 
 
-def test_unverified_confidence_does_not_fail_validation():
-    # The downgrade must never make an otherwise-valid node fail validation
-    # (node-level confidence is not part of the validated schema).
+def test_unverified_flag_does_not_fail_validation():
+    # The flag lives on its own ``verification`` field, deliberately NOT on the
+    # validated ``confidence`` key (whose vocabulary is edge-only), so it must
+    # never make an otherwise-valid node fail validation.
     from graphify.validate import validate_extraction
 
     extraction = {
         "nodes": [{"id": "n1", "label": "foo", "file_type": "code",
-                   "source_file": "a.md", "confidence": "UNVERIFIED"}],
+                   "source_file": "a.md", "verification": "unverified"}],
         "edges": [],
     }
     errors = validate_extraction(extraction)
-    assert not any("confidence" in str(e).lower() for e in errors)
+    assert not any("verification" in str(e).lower() for e in errors)
+
+
+def test_diagnostics_reports_unverified_node_count():
+    # The consumer: diagnose_extraction surfaces the persisted verification flag
+    # so it is not a dead field.
+    from graphify.diagnostics import diagnose_extraction, format_diagnostic_report
+
+    extraction = {
+        "nodes": [
+            {"id": "n1", "label": "ghost", "file_type": "code",
+             "source_file": "a.md", "verification": "unverified"},
+            {"id": "n2", "label": "real", "file_type": "code", "source_file": "a.md"},
+        ],
+        "edges": [],
+    }
+    summary = diagnose_extraction(extraction)
+    assert summary["unverified_node_count"] == 1
+    assert "unverified_code_nodes: 1" in format_diagnostic_report(summary)