From 479f1af4554a8a07fd6869636b652a7591327edc Mon Sep 17 00:00:00 2001 From: safishamsi Date: Fri, 17 Jul 2026 00:02:15 +0100 Subject: [PATCH] fix(cache): mark files partial on empty-parse truncation (follow-up to #1950) The _partial item-marker approach couldn't fire on the most common truncation shape: a mid-JSON cut parses to zero items, so a sliced document whose second slice truncated empty was still stamped complete (the PR's own headline case). - The adaptive-retry give-up sites now record the chunk's own source files in a result-level _partial_files list, independent of parsed items; it propagates through _merge_two / the recursion merges / _merge_into so it reaches both the per-chunk checkpoint and the run-level manifest stamp. - _partial_source_files unions _partial_files with the item markers. - save_semantic_cache seeds an empty group for a named partial file with no items so its entry is stamped partial, and carries a partial prev entry's flag forward so a later clean slice merging over it can't re-promote it to complete. - the CLI final save now passes partial_source_files (computed before the save) so an empty-parse file isn't written back as a complete entry. Co-Authored-By: Claude Opus 4.8 (1M context) --- graphify/cache.py | 27 ++++++++++++++++++---- graphify/cli.py | 23 +++++++++++-------- graphify/llm.py | 44 ++++++++++++++++++++++++++++++++--- tests/test_partial_cache.py | 46 +++++++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 18 deletions(-) diff --git a/graphify/cache.py b/graphify/cache.py index c5edc083..4fd2074c 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -864,6 +864,16 @@ def save_semantic_cache( partial_paths = None if partial_source_files is not None: partial_paths = {resolved_source_path(path) for path in partial_source_files} + # A chunk that truncated to an EMPTY parse contributes no grouped items, + # so its file is absent from by_file and the write loop below would never + # stamp it partial — leaving a prior clean slice looking complete (#1950 + # empty-parse gap). Seed an empty group for each named partial file that + # isn't already present, so the loop merges its existing entry and stamps + # it partial. Keyed by the resolved path (deduped against present groups). + _present = {resolved_source_path(k) for k in by_file} + for _pp in partial_paths: + if _pp not in _present: + by_file[str(_pp)] # defaultdict: create an empty {nodes,edges,hyperedges} def group_skipped(fpath: str) -> bool: """Mirror the write-loop skip condition for one source_file group.""" @@ -948,20 +958,27 @@ def save_semantic_cache( prev = load_cached(p, root, kind=kind, prompt=prompt, prompt_file=prompt_file, allow_legacy=False, allow_partial=True) + _prev_partial = bool(prev.get("partial")) if prev else False if prev: result = { "nodes": (prev.get("nodes", []) or []) + result["nodes"], "edges": (prev.get("edges", []) or []) + result["edges"], "hyperedges": (prev.get("hyperedges", []) or []) + result["hyperedges"], } - # A file is partial if the caller named it OR any of its grouped - # items carries the intrinsic ``_partial`` marker. Stamp the flag - # load_cached keys off; copy so the caller's dict is never mutated. - # A later complete extraction overwrites this same content-hash key - # with a non-partial entry that then serves normally. + else: + _prev_partial = False + # A file is partial if the caller named it, any of its grouped items + # carries the intrinsic ``_partial`` marker, OR the entry it merged + # onto was already partial (an empty-parse truncation leaves a + # ``partial: True`` entry with no item markers, so a later clean slice + # merging over it must NOT silently promote the half-file to complete + # — #1950). Copy so the caller's dict is never mutated. A genuine + # complete re-extraction (merge_existing=False) overwrites the + # content-hash key with a non-partial entry that then serves normally. is_partial = ( (partial_paths is not None and p in partial_paths) or _group_has_partial_marker(result) + or _prev_partial ) if is_partial: result = {**result, "partial": True} diff --git a/graphify/cli.py b/graphify/cli.py index 2c18e2e2..5d13a47e 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -2675,6 +2675,16 @@ def dispatch_command(cmd: str) -> None: # graph without an explicit --allow-partial override. if _chunk_stats["total"] and _chunk_stats["succeeded"] < _chunk_stats["total"]: _extraction_incomplete = True + # Which files truncated this run (item markers + the empty-parse + # _partial_files set). Computed BEFORE the save so it can be passed + # as partial_source_files: without it, a file whose only truncated + # chunk parsed empty (so it has no item markers here) would be + # written as a complete cache entry, re-promoting it (#1950). + from graphify.llm import ( + _partial_source_files as _partial_sf, + _strip_partial_markers as _strip_partial, + ) + _partial_semantic_files = set(_partial_sf(fresh)) try: _save_semantic_cache( fresh.get("nodes", []), @@ -2684,19 +2694,12 @@ def dispatch_command(cmd: str) -> None: allowed_source_files=uncached_paths, mode=sem_cache_mode, prompt=sem_prompt, + partial_source_files=_partial_semantic_files or None, ) except Exception as exc: print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) - # Record which files truncated (before the markers are stripped) - # so they are left unstamped in the manifest and re-queued on the - # next incremental run. The save above consumed the marker to - # stamp affected cache entries partial: True; strip it before the - # corpus feeds the graph so it never leaks into graph.json. - from graphify.llm import ( - _partial_source_files as _partial_sf, - _strip_partial_markers as _strip_partial, - ) - _partial_semantic_files = set(_partial_sf(fresh)) + # Strip the markers before the corpus feeds the graph so the + # internal flag never leaks into graph.json. _strip_partial(fresh) sem_result["nodes"].extend(fresh.get("nodes", [])) sem_result["edges"].extend(fresh.get("edges", [])) diff --git a/graphify/llm.py b/graphify/llm.py index 3cbaf0ea..8d15105e 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -1655,9 +1655,28 @@ def _mark_partial(result: dict) -> None: item["_partial"] = True +def _chunk_partial_files(chunk) -> list[str]: + """Source paths covered by a chunk, for marking a chunk that truncated to an + EMPTY parse partial (#1950 gap): a mid-JSON cut yields zero items, so + ``_mark_partial`` has nothing to tag and the file it covered would be stamped + complete. Recording the chunk's own paths closes that. ``unit_path`` folds a + FileSlice back to its parent file so one truncated slice marks the whole doc.""" + return sorted({str(unit_path(u)) for u in chunk}) + + +def _merged_partial_files(*results: dict) -> list[str]: + """Union of the ``_partial_files`` carried by each result (survives merges).""" + out: set[str] = set() + for r in results: + out.update(r.get("_partial_files", []) or []) + return sorted(out) + + def _partial_source_files(result: dict) -> list[str]: - """Source files that carry at least one ``_partial`` item in ``result``.""" - seen: set[str] = set() + """Source files known partial: those carrying a ``_partial`` item marker, plus + any recorded in ``_partial_files`` (a chunk that truncated to an empty parse + and so has no items to mark).""" + seen: set[str] = set(result.get("_partial_files", []) or []) for bucket in ("nodes", "edges", "hyperedges"): for item in result.get(bucket, []): if isinstance(item, dict) and item.get("_partial"): @@ -1739,6 +1758,7 @@ def _extract_with_adaptive_retry( "output_tokens": left.get("output_tokens", 0) + right.get("output_tokens", 0), "model": model, "finish_reason": "stop", + "_partial_files": _merged_partial_files(left, right), } def _split_lone_slice() -> "tuple[FileSlice, FileSlice] | None": @@ -1797,6 +1817,7 @@ def _extract_with_adaptive_retry( "output_tokens": left.get("output_tokens", 0) + right.get("output_tokens", 0), "model": model, "finish_reason": "stop", + "_partial_files": _merged_partial_files(left, right), } if result.get("finish_reason") != "length": @@ -1817,8 +1838,13 @@ def _extract_with_adaptive_retry( file=sys.stderr, ) # The node set is incomplete; mark it so it is not promoted to the - # semantic cache as authoritative and is re-dispatched next run. + # semantic cache as authoritative and is re-dispatched next run. Also + # record the chunk's files so a truncation that parsed to nothing (an + # empty item set) still marks the file partial (#1950 empty-parse gap). _mark_partial(result) + result["_partial_files"] = sorted( + set(_chunk_partial_files(chunk)) | set(result.get("_partial_files", []) or []) + ) return result if _depth >= max_depth: @@ -1832,6 +1858,9 @@ def _extract_with_adaptive_retry( # re-extraction next run; under-marking would serve a truncated file as # complete, so err toward re-extraction. _mark_partial(result) + result["_partial_files"] = sorted( + set(_chunk_partial_files(chunk)) | set(result.get("_partial_files", []) or []) + ) return result print( @@ -1859,6 +1888,7 @@ def _extract_with_adaptive_retry( # truncation warning; the merged result is no longer truncated as a # logical unit. "finish_reason": "stop", + "_partial_files": _merged_partial_files(left, right), } @@ -2154,6 +2184,14 @@ def _merge_into(merged: dict, result: dict) -> None: merged["hyperedges"].extend(result.get("hyperedges", [])) merged["input_tokens"] += result.get("input_tokens", 0) merged["output_tokens"] += result.get("output_tokens", 0) + # Carry forward files a chunk truncated to an empty parse (#1950): these have + # no items to ride the merge, so they'd otherwise be lost from the run-level + # partial set the manifest stamp consults. + incoming = result.get("_partial_files") + if incoming: + merged["_partial_files"] = sorted( + set(merged.get("_partial_files", []) or []) | set(incoming) + ) def _call_llm( diff --git a/tests/test_partial_cache.py b/tests/test_partial_cache.py index 9dd5be91..66f4f765 100644 --- a/tests/test_partial_cache.py +++ b/tests/test_partial_cache.py @@ -83,6 +83,52 @@ def test_merge_existing_accumulates_slices_and_stays_partial(tmp_path): assert {n["id"] for n in peek["nodes"]} == {"n1", "n2"} +def test_save_stamps_partial_file_with_no_items(tmp_path): + """#1950 empty-parse gap: a chunk that truncates to an empty parse produces + NO items, so partial-ness can only come from partial_source_files. save must + still stamp such a file partial (seeding an empty group) — even when a prior + clean slice already cached it — so it is re-dispatched instead of served.""" + doc = _doc(tmp_path) + # A clean slice cached first. + save_semantic_cache([{"id": "n1", "source_file": "doc.md"}], [], root=tmp_path, prompt="P") + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is not None + # Now an empty-parse truncation covering the same file: no items, only the + # named partial file. The entry must flip to a miss. + save_semantic_cache([], [], root=tmp_path, prompt="P", + merge_existing=True, partial_source_files=["doc.md"]) + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None + # The earlier slice's node is not lost — it stays in the partial entry. + peek = load_cached(doc, root=tmp_path, kind="semantic", prompt="P", allow_partial=True) + assert peek is not None and {n["id"] for n in peek["nodes"]} == {"n1"} + + +def test_clean_slice_does_not_repromote_empty_parse_partial(tmp_path): + """Ordering guard: once a file is partial (from an empty-parse truncation, + so no item markers), a later clean slice merging over it must keep it partial + via the carried-forward prev flag — not silently promote it to complete.""" + doc = _doc(tmp_path) + # Empty-parse partial first (no markers, only the named file). + save_semantic_cache([], [], root=tmp_path, prompt="P", partial_source_files=["doc.md"]) + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None + # A later clean slice checkpoints with merge_existing and no partial arg. + save_semantic_cache([{"id": "n2", "source_file": "doc.md"}], [], root=tmp_path, + prompt="P", merge_existing=True) + # Must still be a miss — the prior truncation is unresolved. + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None + + +def test_partial_files_carries_empty_parse_truncation(): + """_partial_source_files must surface a file recorded in _partial_files even + when the result has zero items (the empty-parse case).""" + import graphify.llm as llm + result = {"nodes": [], "edges": [], "hyperedges": [], "_partial_files": ["big.md"]} + assert llm._partial_source_files(result) == ["big.md"] + # And it unions with intrinsic item markers. + result2 = {"nodes": [{"id": "a", "source_file": "x.md", "_partial": True}], + "edges": [], "hyperedges": [], "_partial_files": ["big.md"]} + assert llm._partial_source_files(result2) == ["big.md", "x.md"] + + def test_stamped_manifest_excludes_partial_files(): """A truncated file produced output this run but is left unstamped in the manifest (like a failed chunk) so detect_incremental re-queues it."""