diff --git a/graphify/cli.py b/graphify/cli.py index 92f05875..3c1ae0e5 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -52,6 +52,50 @@ _GEMINI_NUDGE_TEXT = ( def _default_graph_path() -> str: return str(Path(_GRAPHIFY_OUT) / "graph.json") + + +def _stamped_manifest_files( + files_by_type: dict[str, list[str]], + sem_result: dict, + root: Path, +) -> dict[str, list[str]]: + """Manifest-safe files dict: only stamp semantic files that actually + produced output (cache hit or fresh extraction). Files whose chunk failed + have no source_file entry in sem_result — leaving their semantic_hash + empty so detect_incremental re-queues them (#933). + + Both sides of the membership test are resolved against the scan ``root`` + before comparing (#1897): node/edge ``source_file`` values are + root-relative on a fresh extraction while ``files_by_type`` entries are + absolute (from detect()), so a raw string comparison never matched and + every freshly-extracted semantic doc was dropped from the manifest. + Mirrors the #1890 path normalization in graphify.llm. + """ + root = Path(root) + + def _resolve(value: str) -> Path: + p = Path(value) + if not p.is_absolute(): + p = root / p + try: + return p.resolve() + except (OSError, RuntimeError): + return p + + sem_extracted: set[Path] = set() + for coll in ("nodes", "edges"): + for item in sem_result.get(coll, []): + sf = item.get("source_file", "") + if sf: + sem_extracted.add(_resolve(sf)) + sem_types = {"document", "paper", "image"} + return { + ftype: [ + f for f in flist + if ftype not in sem_types or _resolve(f) in sem_extracted + ] + for ftype, flist in files_by_type.items() + } class _StageTimer: """Print per-stage wall-clock timings to stderr when --timing is set (#1490). @@ -2438,17 +2482,10 @@ def dispatch_command(cmd: str) -> None: # that actually produced output (cache hit or fresh extraction). Files # whose chunk failed have no source_file entry in sem_result — leaving # their semantic_hash empty so detect_incremental re-queues them (#933). - _sem_extracted: set[str] = { - n.get("source_file", "") for n in sem_result.get("nodes", []) - } | { - e.get("source_file", "") for e in sem_result.get("edges", []) - } - _sem_extracted.discard("") - _sem_types = {"document", "paper", "image"} - _manifest_files = { - ftype: [f for f in flist if ftype not in _sem_types or f in _sem_extracted] - for ftype, flist in files_by_type.items() - } + # Path normalization against the scan root happens inside the helper + # (#1897) so fresh root-relative source_files match detect()'s + # absolute file lists. + _manifest_files = _stamped_manifest_files(files_by_type, sem_result, target) if no_cluster: # --no-cluster: dump the raw merged extraction as graph.json. diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index cf5762a7..f56de047 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -135,6 +135,93 @@ def test_extract_succeeds_when_at_least_one_chunk_completes( } == {str(corpus / "README.md")} +def test_manifest_stamps_freshly_extracted_semantic_docs(monkeypatch, tmp_path): + """#1897: fresh extraction returns nodes with ROOT-RELATIVE source_file, + while the #933 manifest filter compared them against detect()'s ABSOLUTE + paths — so `f in _sem_extracted` was always False and every freshly + extracted doc was dropped from the manifest (only code/zero-node files + survived). Both sides must be resolved against the scan root; a genuinely + omitted doc (zero nodes) must still stay unstamped (#933 is intentional).""" + import json + + corpus = _make_corpus(tmp_path) # main.go + README.md + (corpus / "OMITTED.md").write_text("# never extracted\n") + out_dir = tmp_path / "out" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + + def _fresh_relative(paths, **kwargs): + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) + # Root-relative source_file, exactly what a fresh extraction produces. + # OMITTED.md gets no nodes/edges — the model skipped it. + return { + "nodes": [{"id": "readme", "source_file": "README.md", + "file_type": "document"}], + "edges": [], + "hyperedges": [], + "input_tokens": 10, + "output_tokens": 5, + } + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _fresh_relative) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster", "--out", str(out_dir)], + ) + + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + manifest_path = out_dir / "graphify-out" / "manifest.json" + assert manifest_path.exists() + manifest = json.loads(manifest_path.read_text()) + + assert "README.md" in manifest, ( + f"freshly-extracted doc missing from manifest (#1897): {sorted(manifest)}" + ) + assert manifest["README.md"].get("semantic_hash"), ( + "freshly-extracted doc must carry a non-empty semantic_hash" + ) + # Code files are always stamped. + assert manifest.get("main.go", {}).get("semantic_hash") + # The zero-node doc stays unstamped so detect_incremental re-queues it (#933). + assert "OMITTED.md" not in manifest, ( + "zero-node doc must not be stamped in the manifest" + ) + + +def test_stamped_manifest_files_normalizes_both_sides(tmp_path): + """Unit test for the #1897 helper: relative (fresh) and absolute (cache-hit) + source_file values must both match detect()'s absolute file lists; docs with + no output are filtered; code files pass through untouched.""" + from graphify.cli import _stamped_manifest_files + + fresh_doc = tmp_path / "fresh.md"; fresh_doc.write_text("# fresh") + cached_doc = tmp_path / "cached.md"; cached_doc.write_text("# cached") + omitted_doc = tmp_path / "omitted.md"; omitted_doc.write_text("# omitted") + code = tmp_path / "app.py"; code.write_text("x = 1") + + files_by_type = { + "code": [str(code)], + "document": [str(fresh_doc), str(cached_doc), str(omitted_doc)], + } + sem_result = { + # fresh extraction: root-relative source_file + "nodes": [{"id": "n1", "source_file": "fresh.md"}], + # cache replay: absolute source_file (edge-only coverage counts too) + "edges": [{"source": "a", "target": "b", "source_file": str(cached_doc)}], + } + + out = _stamped_manifest_files(files_by_type, sem_result, tmp_path) + assert out["code"] == [str(code)] + assert out["document"] == [str(fresh_doc), str(cached_doc)] + + def _code_only_corpus(tmp_path): """A corpus with only code — no docs/papers/images.""" (tmp_path / "auth.py").write_text(