mirror of
https://github.com/safishamsi/graphify.git
synced 2026-09-24 14:35:46 +00:00
The 0.8.43 str-path coercion (#1386) ran Path(f) over every item in extract_files_direct, but extract_corpus_parallel feeds it FileSlice units from the oversized-doc slicing (#1369), and Path(FileSlice) raises TypeError -- so semantic extraction of any Markdown file larger than _FILE_CHAR_CAP crashed. Coerce only non-Path/non-FileSlice entries. The #1386 tests used small files, so slicing never ran; added a regression test with a file that actually slices. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+5
-3
@@ -1306,9 +1306,11 @@ def extract_files_direct(
|
||||
|
||||
Accepts ``str`` paths as well as ``Path``; string entries are coerced up
|
||||
front so downstream helpers (``_partition_semantic_files``, ``_read_files``,
|
||||
``_build_image_refs``) can rely on ``Path`` semantics (#1386).
|
||||
``_build_image_refs``) can rely on ``Path`` semantics (#1386). FileSlice units
|
||||
(from extract_corpus_parallel's oversized-doc slicing, #1369) pass through
|
||||
untouched — Path(FileSlice) would raise (#1397/#1399).
|
||||
"""
|
||||
files = [Path(f) for f in files]
|
||||
files = [f if isinstance(f, (Path, FileSlice)) else Path(f) for f in files]
|
||||
if backend is None:
|
||||
backend = detect_backend()
|
||||
if backend is None:
|
||||
@@ -1737,7 +1739,7 @@ def extract_corpus_parallel(
|
||||
Accepts ``str`` paths as well as ``Path``; string entries are coerced up
|
||||
front so packing/slicing helpers can rely on ``Path`` semantics (#1386).
|
||||
"""
|
||||
files = [Path(f) for f in files]
|
||||
files = [f if isinstance(f, (Path, FileSlice)) else Path(f) for f in files]
|
||||
# Split oversized splittable documents into slices that cover the whole file
|
||||
# before packing, so content past _FILE_CHAR_CAP is extracted instead of
|
||||
# silently dropped (#1369). Files at/under the cap pass through unchanged.
|
||||
|
||||
@@ -167,6 +167,26 @@ def test_extract_corpus_parallel_accepts_str_and_mixed_paths(tmp_path, monkeypat
|
||||
assert merged["failed_chunks"] == 0
|
||||
|
||||
|
||||
def test_corpus_parallel_oversized_markdown_does_not_crash_on_fileslice(tmp_path, monkeypatch):
|
||||
# #1397/#1399 regression: a Markdown file large enough to be sliced into
|
||||
# FileSlice units must not crash extract_files_direct's Path() coercion
|
||||
# (#1386). The earlier str-path tests used tiny files, so slicing never ran.
|
||||
from graphify.llm import _FILE_CHAR_CAP
|
||||
_clear_backend_env(monkeypatch)
|
||||
monkeypatch.setenv("GOOGLE_API_KEY", "google-key")
|
||||
big = tmp_path / "big.md"
|
||||
big.write_text(("# Section\n\n" + "lorem ipsum dolor sit amet " * 60 + "\n\n") * 30)
|
||||
assert len(big.read_text()) > _FILE_CHAR_CAP # guarantees slicing kicks in
|
||||
result = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1}
|
||||
|
||||
with patch("graphify.llm._call_openai_compat", return_value=result):
|
||||
# both a str path and a FileSlice unit must flow through without TypeError
|
||||
merged = llm.extract_corpus_parallel(
|
||||
[str(big)], backend="gemini", root=tmp_path, max_concurrency=1
|
||||
)
|
||||
assert merged["failed_chunks"] == 0 # no chunk raised Path(FileSlice) TypeError
|
||||
|
||||
|
||||
def test_str_path_entry_points_handle_edge_cases(tmp_path, monkeypatch):
|
||||
_clear_backend_env(monkeypatch)
|
||||
monkeypatch.setenv("GOOGLE_API_KEY", "google-key")
|
||||
|
||||
Reference in New Issue
Block a user