From 3bbf420bc3d944e21c3d0065313466296b06fb23 Mon Sep 17 00:00:00 2001 From: abhay-codes07 <182421137+abhay-codes07@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:02:02 +0100 Subject: [PATCH] fix(extract): give AST INFERRED edges a rubric confidence_score (#2813) AST-emitted INFERRED edges landed at the rubric-forbidden 0.5 (or a hardcoded 0.8). Each AST INFERRED emission site now sets a discrete rubric score keyed to the relation (uses -> 0.95 direct structural evidence, indirect_call and unresolved cross-file calls -> 0.85), and the INFERRED write-time default moves 0.5 -> 0.55 so any score-less INFERRED edge is on the rubric set. EXTRACTED / AMBIGUOUS tiers are unchanged; uses stays INFERRED (not promoted to EXTRACTED) to keep audit percentages honest. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/export.py | 10 +- graphify/extract.py | 10 +- graphify/extractors/engine.py | 6 ++ graphify/extractors/resolution.py | 6 ++ graphify/symbol_resolution.py | 4 +- tests/test_confidence.py | 6 +- tests/test_inferred_confidence_rubric.py | 115 +++++++++++++++++++++++ tests/test_symbol_resolution.py | 2 +- 8 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 tests/test_inferred_confidence_rubric.py diff --git a/graphify/export.py b/graphify/export.py index 7ff216bd..24150668 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -157,7 +157,15 @@ from graphify.exporters.base import COMMUNITY_COLORS # noqa: E402,F401 from graphify.exporters.html import to_html # noqa: E402,F401 -_CONFIDENCE_SCORE_DEFAULTS = {"EXTRACTED": 1.0, "INFERRED": 0.5, "AMBIGUOUS": 0.2} +# Fallback scores for an edge that carries a confidence tier but no +# confidence_score. The INFERRED default was 0.5, which references/extraction-spec.md +# rules out in as many words — "never omit it, never use 0.5 as a default" — and +# which is not in the discrete INFERRED set {0.55, 0.65, 0.75, 0.85, 0.95} either. +# It is now the bottom of that set: a missing score is an absence of evidence +# about strength, so the honest fallback is the weakest value the rubric allows, +# not a midpoint that reads as a coin flip (#2813). Every AST emission site now +# supplies its own score, so this is a backstop rather than a routine path. +_CONFIDENCE_SCORE_DEFAULTS = {"EXTRACTED": 1.0, "INFERRED": 0.55, "AMBIGUOUS": 0.2} def attach_hyperedges(G: nx.Graph, hyperedges: list) -> None: diff --git a/graphify/extract.py b/graphify/extract.py index a113d4a7..00aea9f2 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -6494,7 +6494,11 @@ def extract( "relation": "indirect_call", "context": rc.get("context", "argument"), "confidence": "INFERRED", - "confidence_score": 0.8, + # 0.85, not 0.8: the rubric in references/extraction-spec.md + # is a discrete set {0.55, 0.65, 0.75, 0.85, 0.95} and 0.8 is + # not in it. Same tier, same meaning ("strong inference"), + # now a value the documented scale actually contains (#2813). + "confidence_score": 0.85, "source_file": rc.get("source_file", ""), "source_location": rc.get("source_location"), "weight": 1.0, @@ -6523,7 +6527,9 @@ def extract( confidence_score = 1.0 else: confidence = "INFERRED" - confidence_score = 0.8 + # 0.85 rather than 0.8 — the rubric's INFERRED set is discrete + # and does not contain 0.8 (#2813). + confidence_score = 0.85 all_edges.append({ "source": caller, "target": tgt, diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index eef67268..8359d38c 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -4686,6 +4686,12 @@ def _extract_generic( "relation": "indirect_call", "context": context, "confidence": "INFERRED", + # 0.85 = "strong inference" on the extraction-spec rubric. The symbol + # link is direct — the function is named right here — but that it is + # ever INVOKED is the inference, which is why this is not the 0.95 + # tier. Previously no score was emitted at all and the edge inherited + # the 0.5 default the rubric forbids (#2813). + "confidence_score": 0.85, "source_file": str_path, "source_location": f"L{loc_node.start_point[0] + 1}", "weight": 1.0, diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 454c9904..a75b450f 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -2062,6 +2062,12 @@ def _resolve_cross_file_imports( "target": tgt_nid, "relation": "uses", "confidence": "INFERRED", + # 0.95 = "direct structural evidence (named cross-file + # reference)" from the extraction-spec rubric, which is + # exactly what this edge is: a name this file imports, then + # references. Omitting the score entirely fell through to the + # 0.5 default the same rubric forbids outright (#2813). + "confidence_score": 0.95, "source_file": str_path, "source_location": f"L{line}", "weight": 0.8, diff --git a/graphify/symbol_resolution.py b/graphify/symbol_resolution.py index 892f3106..6adb4a2a 100644 --- a/graphify/symbol_resolution.py +++ b/graphify/symbol_resolution.py @@ -367,7 +367,9 @@ def resolve_cross_file_raw_calls( "relation": "calls", "context": "call", "confidence": "INFERRED", - "confidence_score": 0.8, + # 0.85, not 0.8 — the extraction-spec rubric's INFERRED values + # are a discrete set and 0.8 is not one of them (#2813). + "confidence_score": 0.85, "source_file": raw_call.get("source_file", ""), "source_location": raw_call.get("source_location"), "weight": 1.0, diff --git a/tests/test_confidence.py b/tests/test_confidence.py index 299548ac..dfec94ca 100644 --- a/tests/test_confidence.py +++ b/tests/test_confidence.py @@ -130,7 +130,11 @@ def test_to_json_defaults_missing_confidence_score(): links_by_conf[conf] = link.get("confidence_score") assert links_by_conf.get("EXTRACTED") == 1.0, "EXTRACTED default should be 1.0" - assert links_by_conf.get("INFERRED") == 0.5, "INFERRED default should be 0.5" + # references/extraction-spec.md forbids 0.5 outright ("never use 0.5 as a + # default") and its INFERRED set is {0.55, 0.65, 0.75, 0.85, 0.95}. A missing + # score is an absence of evidence about strength, so the fallback is the + # weakest value the rubric allows rather than a midpoint (#2813). + assert links_by_conf.get("INFERRED") == 0.55, "INFERRED default should be 0.55" def test_report_shows_avg_confidence_for_inferred(): diff --git a/tests/test_inferred_confidence_rubric.py b/tests/test_inferred_confidence_rubric.py new file mode 100644 index 00000000..b9c47c93 --- /dev/null +++ b/tests/test_inferred_confidence_rubric.py @@ -0,0 +1,115 @@ +"""Every INFERRED edge the AST extractor emits must carry a rubric score. + +`references/extraction-spec.md` is explicit: + + confidence_score is REQUIRED on every edge - never omit it, never use 0.5 + as a default + - INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 / 0.85 / 0.75 / 0.65 / 0.55 + +The AST extractor honoured neither half. Some sites emitted no +`confidence_score` at all, which fell through to `_CONFIDENCE_SCORE_DEFAULTS` +and landed on exactly the 0.5 the rubric rules out; others hardcoded 0.8, which +is not in the discrete set. On graphify's own package that was 128 of 128 +INFERRED edges — 54 missing a score and 74 at 0.8 (#2813). + +The tiers are deliberately NOT changed here. The reporter's other option was to +promote AST `uses` edges to EXTRACTED/1.0, which is a semantic judgement about +what the extractor knows; snapping the scores onto the documented scale fixes +the stated violation without making that call. +""" +from pathlib import Path + +import pytest + +from graphify.export import _CONFIDENCE_SCORE_DEFAULTS + +# The discrete INFERRED set from references/extraction-spec.md. +RUBRIC = {0.55, 0.65, 0.75, 0.85, 0.95} + +SRC = Path(__file__).resolve().parent.parent / "graphify" + + +def _extract(tmp_path, name, body): + from graphify.extract import extract + f = tmp_path / name + f.write_text(body, encoding="utf-8") + return extract([f], root=tmp_path) + + +def _inferred(res): + return [e for e in res.get("edges", []) if e.get("confidence") == "INFERRED"] + + +# --------------------------------------------------------------------------- +# The default +# --------------------------------------------------------------------------- + +def test_the_inferred_default_is_not_the_forbidden_value(): + assert _CONFIDENCE_SCORE_DEFAULTS["INFERRED"] != 0.5 + + +def test_every_default_that_can_apply_to_an_inferred_edge_is_on_the_rubric(): + assert _CONFIDENCE_SCORE_DEFAULTS["INFERRED"] in RUBRIC + + +def test_extracted_and_ambiguous_defaults_are_unchanged(): + """The fix is scoped to INFERRED; the other two tiers keep their values.""" + assert _CONFIDENCE_SCORE_DEFAULTS["EXTRACTED"] == 1.0 + assert _CONFIDENCE_SCORE_DEFAULTS["AMBIGUOUS"] == 0.2 + + +# --------------------------------------------------------------------------- +# No emission site ships an off-rubric literal +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("rel_path", [ + "extract.py", + "symbol_resolution.py", + "extractors/engine.py", + "extractors/resolution.py", +]) +def test_no_module_hardcodes_an_off_rubric_inferred_score(rel_path): + """0.8 was the value in the tree and is not on the scale. Catch it and the + forbidden 0.5 as literals, so a future edit cannot reintroduce either.""" + text = (SRC / rel_path).read_text(encoding="utf-8") + for forbidden in ('"confidence_score": 0.8,', '"confidence_score": 0.5,', + "confidence_score = 0.8\n", "confidence_score = 0.5\n"): + assert forbidden not in text, f"{rel_path} still emits {forbidden.strip()}" + + +# --------------------------------------------------------------------------- +# End to end +# --------------------------------------------------------------------------- + +def test_python_indirect_call_edges_carry_a_rubric_score(tmp_path): + """A function passed by name as an argument — the indirect_call path.""" + res = _extract(tmp_path, "m.py", ( + "def handler():\n return 1\n\n" + "def register(cb):\n return cb\n\n" + "def wire():\n return register(handler)\n" + )) + inferred = _inferred(res) + if not inferred: + pytest.skip("this build produced no INFERRED edges for the fixture") + for e in inferred: + assert e.get("confidence_score") in RUBRIC, e + + +def test_no_inferred_edge_is_left_without_a_score(tmp_path): + res = _extract(tmp_path, "m.py", ( + "def handler():\n return 1\n\n" + "def register(cb):\n return cb\n\n" + "def wire():\n return register(handler)\n" + )) + for e in _inferred(res): + assert e.get("confidence_score") is not None, ( + f"edge would inherit the default instead of stating a score: {e}") + + +def test_extracted_edges_still_score_one(tmp_path): + """The other half of the rubric must not drift while fixing this one.""" + res = _extract(tmp_path, "m.py", "def a():\n return 1\n\ndef b():\n return a()\n") + for e in res.get("edges", []): + if e.get("confidence") == "EXTRACTED" and "confidence_score" in e: + assert e["confidence_score"] == 1.0, e diff --git a/tests/test_symbol_resolution.py b/tests/test_symbol_resolution.py index faa3e52a..d8737d23 100644 --- a/tests/test_symbol_resolution.py +++ b/tests/test_symbol_resolution.py @@ -70,7 +70,7 @@ def test_resolve_cross_file_raw_calls_emits_unique_unqualified_call() -> None: "relation": "calls", "context": "call", "confidence": "INFERRED", - "confidence_score": 0.8, + "confidence_score": 0.85, # rubric value; 0.8 is not on the scale (#2813) "source_file": "caller.py", "source_location": "L2", "weight": 1.0,