fix(cache): mark files partial on empty-parse truncation (follow-up to #1950)

The _partial item-marker approach couldn't fire on the most common truncation
shape: a mid-JSON cut parses to zero items, so a sliced document whose second
slice truncated empty was still stamped complete (the PR's own headline case).

- The adaptive-retry give-up sites now record the chunk's own source files in a
  result-level _partial_files list, independent of parsed items; it propagates
  through _merge_two / the recursion merges / _merge_into so it reaches both the
  per-chunk checkpoint and the run-level manifest stamp.
- _partial_source_files unions _partial_files with the item markers.
- save_semantic_cache seeds an empty group for a named partial file with no
  items so its entry is stamped partial, and carries a partial prev entry's flag
  forward so a later clean slice merging over it can't re-promote it to complete.
- the CLI final save now passes partial_source_files (computed before the save)
  so an empty-parse file isn't written back as a complete entry.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

graphify/cache.py

@@ -864,6 +864,16 @@ def save_semantic_cache(
     partial_paths = None
     if partial_source_files is not None:
         partial_paths = {resolved_source_path(path) for path in partial_source_files}
+        # A chunk that truncated to an EMPTY parse contributes no grouped items,
+        # so its file is absent from by_file and the write loop below would never
+        # stamp it partial — leaving a prior clean slice looking complete (#1950
+        # empty-parse gap). Seed an empty group for each named partial file that
+        # isn't already present, so the loop merges its existing entry and stamps
+        # it partial. Keyed by the resolved path (deduped against present groups).
+        _present = {resolved_source_path(k) for k in by_file}
+        for _pp in partial_paths:
+            if _pp not in _present:
+                by_file[str(_pp)]  # defaultdict: create an empty {nodes,edges,hyperedges}
 
     def group_skipped(fpath: str) -> bool:
         """Mirror the write-loop skip condition for one source_file group."""
@@ -948,20 +958,27 @@ def save_semantic_cache(
                 prev = load_cached(p, root, kind=kind, prompt=prompt,
                                    prompt_file=prompt_file, allow_legacy=False,
                                    allow_partial=True)
+                _prev_partial = bool(prev.get("partial")) if prev else False
                 if prev:
                     result = {
                         "nodes": (prev.get("nodes", []) or []) + result["nodes"],
                         "edges": (prev.get("edges", []) or []) + result["edges"],
                         "hyperedges": (prev.get("hyperedges", []) or []) + result["hyperedges"],
                     }
-            # A file is partial if the caller named it OR any of its grouped
-            # items carries the intrinsic ``_partial`` marker. Stamp the flag
-            # load_cached keys off; copy so the caller's dict is never mutated.
-            # A later complete extraction overwrites this same content-hash key
-            # with a non-partial entry that then serves normally.
+            else:
+                _prev_partial = False
+            # A file is partial if the caller named it, any of its grouped items
+            # carries the intrinsic ``_partial`` marker, OR the entry it merged
+            # onto was already partial (an empty-parse truncation leaves a
+            # ``partial: True`` entry with no item markers, so a later clean slice
+            # merging over it must NOT silently promote the half-file to complete
+            # — #1950). Copy so the caller's dict is never mutated. A genuine
+            # complete re-extraction (merge_existing=False) overwrites the
+            # content-hash key with a non-partial entry that then serves normally.
             is_partial = (
                 (partial_paths is not None and p in partial_paths)
                 or _group_has_partial_marker(result)
+                or _prev_partial
             )
             if is_partial:
                 result = {**result, "partial": True}

graphify/cli.py

@@ -2675,6 +2675,16 @@ def dispatch_command(cmd: str) -> None:
                 # graph without an explicit --allow-partial override.
                 if _chunk_stats["total"] and _chunk_stats["succeeded"] < _chunk_stats["total"]:
                     _extraction_incomplete = True
+                # Which files truncated this run (item markers + the empty-parse
+                # _partial_files set). Computed BEFORE the save so it can be passed
+                # as partial_source_files: without it, a file whose only truncated
+                # chunk parsed empty (so it has no item markers here) would be
+                # written as a complete cache entry, re-promoting it (#1950).
+                from graphify.llm import (
+                    _partial_source_files as _partial_sf,
+                    _strip_partial_markers as _strip_partial,
+                )
+                _partial_semantic_files = set(_partial_sf(fresh))
                 try:
                     _save_semantic_cache(
                         fresh.get("nodes", []),
@@ -2684,19 +2694,12 @@ def dispatch_command(cmd: str) -> None:
                         allowed_source_files=uncached_paths,
                         mode=sem_cache_mode,
                         prompt=sem_prompt,
+                        partial_source_files=_partial_semantic_files or None,
                     )
                 except Exception as exc:
                     print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr)
-                # Record which files truncated (before the markers are stripped)
-                # so they are left unstamped in the manifest and re-queued on the
-                # next incremental run. The save above consumed the marker to
-                # stamp affected cache entries partial: True; strip it before the
-                # corpus feeds the graph so it never leaks into graph.json.
-                from graphify.llm import (
-                    _partial_source_files as _partial_sf,
-                    _strip_partial_markers as _strip_partial,
-                )
-                _partial_semantic_files = set(_partial_sf(fresh))
+                # Strip the markers before the corpus feeds the graph so the
+                # internal flag never leaks into graph.json.
                 _strip_partial(fresh)
                 sem_result["nodes"].extend(fresh.get("nodes", []))
                 sem_result["edges"].extend(fresh.get("edges", []))

graphify/llm.py

@@ -1655,9 +1655,28 @@ def _mark_partial(result: dict) -> None:
                 item["_partial"] = True
 
 
+def _chunk_partial_files(chunk) -> list[str]:
+    """Source paths covered by a chunk, for marking a chunk that truncated to an
+    EMPTY parse partial (#1950 gap): a mid-JSON cut yields zero items, so
+    ``_mark_partial`` has nothing to tag and the file it covered would be stamped
+    complete. Recording the chunk's own paths closes that. ``unit_path`` folds a
+    FileSlice back to its parent file so one truncated slice marks the whole doc."""
+    return sorted({str(unit_path(u)) for u in chunk})
+
+
+def _merged_partial_files(*results: dict) -> list[str]:
+    """Union of the ``_partial_files`` carried by each result (survives merges)."""
+    out: set[str] = set()
+    for r in results:
+        out.update(r.get("_partial_files", []) or [])
+    return sorted(out)
+
+
 def _partial_source_files(result: dict) -> list[str]:
-    """Source files that carry at least one ``_partial`` item in ``result``."""
-    seen: set[str] = set()
+    """Source files known partial: those carrying a ``_partial`` item marker, plus
+    any recorded in ``_partial_files`` (a chunk that truncated to an empty parse
+    and so has no items to mark)."""
+    seen: set[str] = set(result.get("_partial_files", []) or [])
     for bucket in ("nodes", "edges", "hyperedges"):
         for item in result.get(bucket, []):
             if isinstance(item, dict) and item.get("_partial"):
@@ -1739,6 +1758,7 @@ def _extract_with_adaptive_retry(
             "output_tokens": left.get("output_tokens", 0) + right.get("output_tokens", 0),
             "model": model,
             "finish_reason": "stop",
+            "_partial_files": _merged_partial_files(left, right),
         }
 
     def _split_lone_slice() -> "tuple[FileSlice, FileSlice] | None":
@@ -1797,6 +1817,7 @@ def _extract_with_adaptive_retry(
             "output_tokens": left.get("output_tokens", 0) + right.get("output_tokens", 0),
             "model": model,
             "finish_reason": "stop",
+            "_partial_files": _merged_partial_files(left, right),
         }
 
     if result.get("finish_reason") != "length":
@@ -1817,8 +1838,13 @@ def _extract_with_adaptive_retry(
             file=sys.stderr,
         )
         # The node set is incomplete; mark it so it is not promoted to the
-        # semantic cache as authoritative and is re-dispatched next run.
+        # semantic cache as authoritative and is re-dispatched next run. Also
+        # record the chunk's files so a truncation that parsed to nothing (an
+        # empty item set) still marks the file partial (#1950 empty-parse gap).
         _mark_partial(result)
+        result["_partial_files"] = sorted(
+            set(_chunk_partial_files(chunk)) | set(result.get("_partial_files", []) or [])
+        )
         return result
 
     if _depth >= max_depth:
@@ -1832,6 +1858,9 @@ def _extract_with_adaptive_retry(
         # re-extraction next run; under-marking would serve a truncated file as
         # complete, so err toward re-extraction.
         _mark_partial(result)
+        result["_partial_files"] = sorted(
+            set(_chunk_partial_files(chunk)) | set(result.get("_partial_files", []) or [])
+        )
         return result
 
     print(
@@ -1859,6 +1888,7 @@ def _extract_with_adaptive_retry(
         # truncation warning; the merged result is no longer truncated as a
         # logical unit.
         "finish_reason": "stop",
+        "_partial_files": _merged_partial_files(left, right),
     }
 
 
@@ -2154,6 +2184,14 @@ def _merge_into(merged: dict, result: dict) -> None:
     merged["hyperedges"].extend(result.get("hyperedges", []))
     merged["input_tokens"] += result.get("input_tokens", 0)
     merged["output_tokens"] += result.get("output_tokens", 0)
+    # Carry forward files a chunk truncated to an empty parse (#1950): these have
+    # no items to ride the merge, so they'd otherwise be lost from the run-level
+    # partial set the manifest stamp consults.
+    incoming = result.get("_partial_files")
+    if incoming:
+        merged["_partial_files"] = sorted(
+            set(merged.get("_partial_files", []) or []) | set(incoming)
+        )
 
 
 def _call_llm(

tests/test_partial_cache.py

@@ -83,6 +83,52 @@ def test_merge_existing_accumulates_slices_and_stays_partial(tmp_path):
     assert {n["id"] for n in peek["nodes"]} == {"n1", "n2"}
 
 
+def test_save_stamps_partial_file_with_no_items(tmp_path):
+    """#1950 empty-parse gap: a chunk that truncates to an empty parse produces
+    NO items, so partial-ness can only come from partial_source_files. save must
+    still stamp such a file partial (seeding an empty group) — even when a prior
+    clean slice already cached it — so it is re-dispatched instead of served."""
+    doc = _doc(tmp_path)
+    # A clean slice cached first.
+    save_semantic_cache([{"id": "n1", "source_file": "doc.md"}], [], root=tmp_path, prompt="P")
+    assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is not None
+    # Now an empty-parse truncation covering the same file: no items, only the
+    # named partial file. The entry must flip to a miss.
+    save_semantic_cache([], [], root=tmp_path, prompt="P",
+                        merge_existing=True, partial_source_files=["doc.md"])
+    assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None
+    # The earlier slice's node is not lost — it stays in the partial entry.
+    peek = load_cached(doc, root=tmp_path, kind="semantic", prompt="P", allow_partial=True)
+    assert peek is not None and {n["id"] for n in peek["nodes"]} == {"n1"}
+
+
+def test_clean_slice_does_not_repromote_empty_parse_partial(tmp_path):
+    """Ordering guard: once a file is partial (from an empty-parse truncation,
+    so no item markers), a later clean slice merging over it must keep it partial
+    via the carried-forward prev flag — not silently promote it to complete."""
+    doc = _doc(tmp_path)
+    # Empty-parse partial first (no markers, only the named file).
+    save_semantic_cache([], [], root=tmp_path, prompt="P", partial_source_files=["doc.md"])
+    assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None
+    # A later clean slice checkpoints with merge_existing and no partial arg.
+    save_semantic_cache([{"id": "n2", "source_file": "doc.md"}], [], root=tmp_path,
+                        prompt="P", merge_existing=True)
+    # Must still be a miss — the prior truncation is unresolved.
+    assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None
+
+
+def test_partial_files_carries_empty_parse_truncation():
+    """_partial_source_files must surface a file recorded in _partial_files even
+    when the result has zero items (the empty-parse case)."""
+    import graphify.llm as llm
+    result = {"nodes": [], "edges": [], "hyperedges": [], "_partial_files": ["big.md"]}
+    assert llm._partial_source_files(result) == ["big.md"]
+    # And it unions with intrinsic item markers.
+    result2 = {"nodes": [{"id": "a", "source_file": "x.md", "_partial": True}],
+               "edges": [], "hyperedges": [], "_partial_files": ["big.md"]}
+    assert llm._partial_source_files(result2) == ["big.md", "x.md"]
+
+
 def test_stamped_manifest_excludes_partial_files():
     """A truncated file produced output this run but is left unstamped in the
     manifest (like a failed chunk) so detect_incremental re-queues it."""