Recover from context-window-exceeded API errors in adaptive retry (#789)

Adaptive retry only recovered from `finish_reason="length"` (output
truncation). It did not handle the other shape of overflow: the API
rejecting the prompt outright with a 400 because the input plus
`max_completion_tokens` doesn't fit in the model's context window.

This shows up immediately on local OpenAI-compatible servers (LM
Studio, llama.cpp, vLLM) where the default context is small (4K-32K)
and a 60K-token chunk packed for cloud Kimi/Claude blows past it.
Without retry the whole chunk fails with no output, even though the
two halves would each fit cleanly.

Catch a heuristic set of context-overflow exception messages,
classify them as the same kind of recoverable failure as
`finish_reason="length"`, and split-recurse on the same path. Single-
file overflow returns an empty fragment so the rest of the corpus
keeps running. Unrelated errors (rate limit, auth, etc.) still
propagate.

Tested with qwen3.5-9b on LM Studio (32K ctx) against a 215-file
corpus where chunks 4-12 of 12 previously failed; with this change
the overflowing chunks self-heal by splitting in half.

graphify/llm.py

@@ -449,6 +449,35 @@ def _pack_chunks_by_tokens(
     return chunks
 
 
+_CONTEXT_EXCEEDED_MARKERS = (
+    "context size",
+    "context length",
+    "context_length",
+    "context window",
+    "n_keep",
+    "exceeds the available",
+    "n_ctx",
+    "maximum context",
+    "too many tokens",
+    "prompt is too long",
+    "context_length_exceeded",
+)
+
+
+def _looks_like_context_exceeded(exc: BaseException) -> bool:
+    """Heuristically classify an exception as a context-window overflow.
+
+    Different backends raise different exception types and messages for the
+    same underlying problem ("the prompt + max_completion_tokens did not fit
+    in the model's context window"). We match on substrings of the stringified
+    exception so the retry layer can recover without depending on a specific
+    SDK class. False positives are cheap (we'll re-extract on halves and
+    likely recover); false negatives are expensive (chunk fails entirely).
+    """
+    msg = str(exc).lower()
+    return any(marker in msg for marker in _CONTEXT_EXCEEDED_MARKERS)
+
+
 def _extract_with_adaptive_retry(
     chunk: list[Path],
     backend: str,
@@ -458,26 +487,73 @@ def _extract_with_adaptive_retry(
     max_depth: int,
     _depth: int = 0,
 ) -> dict:
-    """Extract a chunk; if the response is truncated (`finish_reason="length"`),
+    """Extract a chunk; if the response is truncated (`finish_reason="length"`)
+    or the API rejects the prompt as too large for the model's context window,
     split the chunk in half and recurse.
 
-    The signal driving the retry is the API's own `finish_reason` — `"length"`
-    means the model hit `max_completion_tokens` mid-output. The truncated JSON
-    has nothing useful in it (parse fails partway through a string or array),
-    so we discard it and re-extract on smaller inputs that produce shorter
-    outputs.
+    Two signals drive the retry:
+
+    - `finish_reason == "length"` — the model accepted the input but ran out of
+      `max_completion_tokens` mid-output. The truncated JSON is unparseable, so
+      we discard it and re-extract on smaller inputs that produce shorter
+      outputs.
+
+    - context-window-exceeded API errors — the model rejected the input
+      outright (HTTP 400 from LM Studio, llama.cpp, vLLM, OpenAI, etc.).
+      Without a retry the whole chunk would fail with no output. Splitting in
+      half is the same recovery as for the `length` case and works for the
+      same reason.
 
     Recursion is capped at `max_depth` to bound worst-case cost. A chunk of N
     files can split into up to 2**max_depth pieces — at depth=3 that's 8x. If
-    still truncated at the cap, we surface the (likely empty) result with a
+    still failing at the cap, we surface the (likely empty) result with a
     warning rather than infinite-loop.
 
-    A single-file chunk that truncates is unrecoverable here — we can't make
+    A single-file chunk that overflows is unrecoverable here — we can't make
     one file smaller than itself, so we return what we got and warn.
     """
-    result = extract_files_direct(
-        chunk, backend=backend, api_key=api_key, model=model, root=root
-    )
+    try:
+        result = extract_files_direct(
+            chunk, backend=backend, api_key=api_key, model=model, root=root
+        )
+    except Exception as exc:  # noqa: BLE001 — re-raise unless it's a known context overflow
+        if not _looks_like_context_exceeded(exc):
+            raise
+        if len(chunk) <= 1:
+            print(
+                f"[graphify] single-file chunk {chunk[0]} exceeds model context "
+                f"and cannot be split further: {exc}",
+                file=sys.stderr,
+            )
+            return {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0, "model": model, "finish_reason": "stop"}
+        if _depth >= max_depth:
+            print(
+                f"[graphify] chunk of {len(chunk)} still overflows context at "
+                f"recursion depth {_depth} (max {max_depth}) — dropping",
+                file=sys.stderr,
+            )
+            return {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0, "model": model, "finish_reason": "stop"}
+        print(
+            f"[graphify] chunk of {len(chunk)} exceeded context at depth "
+            f"{_depth} ({type(exc).__name__}); splitting in half and retrying",
+            file=sys.stderr,
+        )
+        mid = len(chunk) // 2
+        left = _extract_with_adaptive_retry(
+            chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1
+        )
+        right = _extract_with_adaptive_retry(
+            chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1
+        )
+        return {
+            "nodes": left.get("nodes", []) + right.get("nodes", []),
+            "edges": left.get("edges", []) + right.get("edges", []),
+            "hyperedges": left.get("hyperedges", []) + right.get("hyperedges", []),
+            "input_tokens": left.get("input_tokens", 0) + right.get("input_tokens", 0),
+            "output_tokens": left.get("output_tokens", 0) + right.get("output_tokens", 0),
+            "model": model,
+            "finish_reason": "stop",
+        }
 
     if result.get("finish_reason") != "length":
         return result

tests/test_llm_backends.py

@@ -95,3 +95,95 @@ def test_missing_gemini_key_names_both_supported_env_vars(monkeypatch):
         llm.extract_files_direct([Path("missing.md")], backend="gemini")
 
     assert "GEMINI_API_KEY or GOOGLE_API_KEY" in str(exc.value)
+
+
+# ---------------------------------------------------------------------------
+# Adaptive retry: context-window overflow recovery
+# ---------------------------------------------------------------------------
+
+
+def _ok(nodes=None, edges=None, model="m"):
+    return {
+        "nodes": nodes or [],
+        "edges": edges or [],
+        "hyperedges": [],
+        "input_tokens": 1,
+        "output_tokens": 1,
+        "model": model,
+        "finish_reason": "stop",
+    }
+
+
+def test_looks_like_context_exceeded_matches_common_messages():
+    msgs = [
+        "Error code: 400 - {'error': 'Context size has been exceeded.'}",
+        "n_keep: 22374 >= n_ctx: 4096",
+        "context_length_exceeded: This model's maximum context length is 8192 tokens",
+        "exceeds the available context size",
+        "The prompt is too long for this model.",
+    ]
+    for m in msgs:
+        assert llm._looks_like_context_exceeded(RuntimeError(m)), m
+
+
+def test_looks_like_context_exceeded_ignores_unrelated_errors():
+    for m in ["timeout", "rate limit", "401 unauthorized", "connection refused"]:
+        assert not llm._looks_like_context_exceeded(RuntimeError(m)), m
+
+
+def test_adaptive_retry_splits_on_context_exceeded(tmp_path):
+    files = [tmp_path / f"f{i}.md" for i in range(4)]
+    for f in files:
+        f.write_text("hello")
+
+    calls = {"n": 0}
+
+    def fake_extract(chunk, *_, **__):
+        calls["n"] += 1
+        # First call (whole chunk) fails with context overflow; recursive
+        # halves succeed. This is the same shape LM Studio / vLLM / OpenAI
+        # produce when a chunk overflows the model's context window.
+        if len(chunk) == 4:
+            raise RuntimeError("Error 400: Context size has been exceeded.")
+        return _ok(nodes=[{"id": f.stem} for f in chunk])
+
+    with patch("graphify.llm.extract_files_direct", side_effect=fake_extract):
+        result = llm._extract_with_adaptive_retry(
+            files, backend="kimi", api_key="k", model="m", root=tmp_path, max_depth=3
+        )
+
+    assert len(result["nodes"]) == 4
+    assert calls["n"] == 3  # 1 failure + 2 halves
+
+
+def test_adaptive_retry_gives_up_on_single_file_overflow(tmp_path):
+    f = tmp_path / "huge.md"
+    f.write_text("x")
+
+    def fake_extract(*_, **__):
+        raise RuntimeError("context_length_exceeded")
+
+    with patch("graphify.llm.extract_files_direct", side_effect=fake_extract):
+        result = llm._extract_with_adaptive_retry(
+            [f], backend="kimi", api_key="k", model="m", root=tmp_path, max_depth=3
+        )
+
+    # Single-file overflow returns an empty fragment instead of raising — the
+    # caller can keep going on the rest of the corpus.
+    assert result["nodes"] == []
+    assert result["edges"] == []
+    assert result["finish_reason"] == "stop"
+
+
+def test_adaptive_retry_re_raises_unrelated_errors(tmp_path):
+    f = tmp_path / "f.md"
+    f.write_text("x")
+
+    def fake_extract(*_, **__):
+        raise RuntimeError("rate limit hit")
+
+    with patch("graphify.llm.extract_files_direct", side_effect=fake_extract):
+        with pytest.raises(RuntimeError, match="rate limit"):
+            llm._extract_with_adaptive_retry(
+                [f], backend="kimi", api_key="k", model="m", root=tmp_path, max_depth=3
+            )