diff --git a/graphify/llm.py b/graphify/llm.py index 97e199b3..421fe25b 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -1036,6 +1036,12 @@ def _json_object_candidates(text: str) -> list[int]: model that narrates before answering — "Here's a thinking process: 1. **Analyze User Input:** …" with braces in the narration — does not have its real answer masked by the first brace in the text (#2882). + + Known limit: each bucket is capped at ``_MAX_OBJECT_CANDIDATES`` from the + front, so a reply with more than that many *keyed* braces before the real + answer (a very verbose model that emits a ``{"nodes": …}`` sketch per file) + could drop the true answer's brace. This needs an implausibly chatty + preamble and is left as a known gap rather than complicating the scan. """ preferred: list[int] = [] rest: list[int] = [] @@ -1130,10 +1136,18 @@ def _parse_llm_json(raw: str) -> dict: if not isinstance(parsed, dict): continue if any(k in parsed for k in _FRAGMENT_KEYS): - if any(parsed.get(k) for k in _FRAGMENT_KEYS): - return _sanitize_fragment(parsed) + # Gate on the SANITIZED content, not the raw value. A reasoning + # sketch commonly lists ids as bare strings — `{"nodes": ["A", "B"]}` + # — whose arrays are truthy but hold no edge/node objects. Testing + # the raw value would let that sketch win and then sanitize down to + # empty, shadowing the real answer that follows and re-triggering the + # #2880 hollow-response bisection. Sanitizing first demotes it to the + # empty-fragment tier so the genuine fragment below still wins. + cand = _sanitize_fragment(parsed) + if any(cand.get(k) for k in _FRAGMENT_KEYS): + return cand if empty_fragment is None: - empty_fragment = parsed + empty_fragment = cand elif fallback is None: fallback = parsed diff --git a/tests/test_llm_parser_reasoning.py b/tests/test_llm_parser_reasoning.py index 5c16c7ba..d63bab99 100644 --- a/tests/test_llm_parser_reasoning.py +++ b/tests/test_llm_parser_reasoning.py @@ -136,3 +136,30 @@ def test_empty_fragment_outranks_a_non_fragment_object(): result = llm._parse_llm_json(raw) assert "description" not in result assert result["nodes"] == [] + + +def test_string_list_sketch_does_not_shadow_the_real_answer(): + """A reasoning sketch that lists ids as bare strings (`{"nodes": ["A","B"]}`) + has truthy arrays but no node/edge objects. Gating on the raw value would let + it win and then sanitize to empty, shadowing the real fragment that follows + and re-triggering the #2880 hollow-response bisection. The sanitized-content + gate must demote the sketch and return the real answer.""" + raw = ( + "Planning the graph. First a rough sketch of what I will emit:\n" + '{"nodes": ["FileA", "FileB"], "edges": ["FileA->FileB"]}\n\n' + "Now the actual fragment:\n" + '{"nodes": [{"id": "a", "label": "A"}], ' + '"edges": [{"source": "a", "target": "b"}], "hyperedges": []}' + ) + result = llm._parse_llm_json(raw) + assert result["nodes"] == [{"id": "a", "label": "A"}], result + assert result["edges"] == [{"source": "a", "target": "b"}] + + +def test_string_list_sketch_alone_sanitizes_to_empty(): + """With no real answer following, a string-list sketch must not masquerade as + content: its non-dict entries are stripped, leaving an empty fragment (which + then reads as hollow downstream).""" + result = llm._parse_llm_json('{"nodes": ["FileA", "FileB"], "edges": ["x"]}') + assert result["nodes"] == [] + assert result["edges"] == []