diff --git a/graphify/__main__.py b/graphify/__main__.py index fa112ee5b..49c24745d 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -2750,6 +2750,22 @@ def main() -> None: graph_json_path = graphify_out / "graph.json" analysis_path = graphify_out / ".graphify_analysis.json" + # Build a manifest-safe files dict: only stamp semantic_hash for files + # that actually produced output (cache hit or fresh extraction). Files + # whose chunk failed have no source_file entry in sem_result — leaving + # their semantic_hash empty so detect_incremental re-queues them (#933). + _sem_extracted: set[str] = { + n.get("source_file", "") for n in sem_result.get("nodes", []) + } | { + e.get("source_file", "") for e in sem_result.get("edges", []) + } + _sem_extracted.discard("") + _sem_types = {"document", "paper", "image"} + _manifest_files = { + ftype: [f for f in flist if ftype not in _sem_types or f in _sem_extracted] + for ftype, flist in files_by_type.items() + } + if no_cluster: # --no-cluster: dump the raw merged extraction as graph.json. # No NetworkX, no community detection, no analysis sidecar. @@ -2772,7 +2788,7 @@ def main() -> None: f"est. cost: ${cost:.4f}" ) try: - _save_manifest(files_by_type, manifest_path=str(manifest_path), kind="both") + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both") except Exception as exc: print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) if global_merge: @@ -2806,9 +2822,10 @@ def main() -> None: prune_sources=deleted_files or None, dedup=True, dedup_llm_backend=dedup_backend, + root=target, ) else: - G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend) + G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) if G.number_of_nodes() == 0: print( "[graphify extract] graph is empty — extraction produced no nodes. " @@ -2854,7 +2871,7 @@ def main() -> None: } analysis_path.write_text(json.dumps(analysis, indent=2), encoding="utf-8") try: - _save_manifest(files_by_type, manifest_path=str(manifest_path), kind="both") + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both") except Exception as exc: print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) @@ -2882,6 +2899,128 @@ def main() -> None: f"est. cost (~{backend}): ${cost:.4f}" ) + elif cmd == "cache-check": + # graphify cache-check [--root ] + # Reads file paths (one per line) from , checks semantic cache. + # Writes: + # graphify-out/.graphify_cached.json — already-cached nodes/edges/hyperedges + # graphify-out/.graphify_uncached.txt — paths that need extraction + # Stdout: "Cache: N hit, M miss" + from graphify.cache import check_semantic_cache + if len(sys.argv) < 3: + print("Usage: graphify cache-check [--root ]", file=sys.stderr) + sys.exit(1) + files_from = Path(sys.argv[2]) + root = Path(".") + i = 3 + while i < len(sys.argv): + if sys.argv[i] == "--root" and i + 1 < len(sys.argv): + root = Path(sys.argv[i + 1]) + i += 2 + else: + i += 1 + files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] + cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(files, root) + out = root / "graphify-out" + out.mkdir(parents=True, exist_ok=True) + if cached_nodes or cached_edges or cached_hyperedges: + (out / ".graphify_cached.json").write_text( + json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges}, + ensure_ascii=False), + encoding="utf-8", + ) + (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") + print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") + + elif cmd == "merge-chunks": + # graphify merge-chunks --out + # Concatenates .graphify_chunk_*.json files written by semantic subagents. + # Deduplicates nodes by id (first writer wins). Sums token counts. + import glob as _glob + if len(sys.argv) < 3: + print("Usage: graphify merge-chunks --out ", file=sys.stderr) + sys.exit(1) + out_path: Path | None = None + chunk_args: list[str] = [] + i = 2 + while i < len(sys.argv): + if sys.argv[i] == "--out" and i + 1 < len(sys.argv): + out_path = Path(sys.argv[i + 1]) + i += 2 + else: + chunk_args.append(sys.argv[i]) + i += 1 + if not out_path: + print("error: --out required", file=sys.stderr) + sys.exit(1) + chunk_files: list[str] = [] + for arg in chunk_args: + expanded = _glob.glob(arg) + chunk_files.extend(sorted(expanded) if expanded else [arg]) + merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + seen_ids: set[str] = set() + for cf in chunk_files: + try: + chunk = json.loads(Path(cf).read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + print(f"[graphify merge-chunks] warning: skipping {cf}: {exc}", file=sys.stderr) + continue + for n in chunk.get("nodes", []): + if n.get("id") not in seen_ids: + seen_ids.add(n["id"]) + merged["nodes"].append(n) + merged["edges"].extend(chunk.get("edges", [])) + merged["hyperedges"].extend(chunk.get("hyperedges", [])) + merged["input_tokens"] += chunk.get("input_tokens", 0) + merged["output_tokens"] += chunk.get("output_tokens", 0) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(merged, ensure_ascii=False), encoding="utf-8") + print( + f"Merged {len(chunk_files)} chunks: {merged['nodes']} nodes, {len(merged['edges'])} edges, " + f"{merged['input_tokens']:,} in / {merged['output_tokens']:,} out tokens" + ) + + elif cmd == "merge-semantic": + # graphify merge-semantic --cached --new --out + # Merges cached semantic results with freshly-extracted chunk results. + # Deduplicates nodes by id (cached entries take priority over new ones). + if len(sys.argv) < 3: + print("Usage: graphify merge-semantic --cached --new --out ", file=sys.stderr) + sys.exit(1) + cached_path: Path | None = None + new_path: Path | None = None + out_path2: Path | None = None + i = 2 + while i < len(sys.argv): + if sys.argv[i] == "--cached" and i + 1 < len(sys.argv): + cached_path = Path(sys.argv[i + 1]); i += 2 + elif sys.argv[i] == "--new" and i + 1 < len(sys.argv): + new_path = Path(sys.argv[i + 1]); i += 2 + elif sys.argv[i] == "--out" and i + 1 < len(sys.argv): + out_path2 = Path(sys.argv[i + 1]); i += 2 + else: + i += 1 + if not out_path2: + print("error: --out required", file=sys.stderr) + sys.exit(1) + empty: dict = {"nodes": [], "edges": [], "hyperedges": []} + cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty + new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty + seen_ids2: set[str] = set() + all_nodes: list[dict] = [] + for n in cached_data.get("nodes", []) + new_data.get("nodes", []): + if n.get("id") not in seen_ids2: + seen_ids2.add(n["id"]) + all_nodes.append(n) + merged2 = { + "nodes": all_nodes, + "edges": cached_data.get("edges", []) + new_data.get("edges", []), + "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []), + } + out_path2.parent.mkdir(parents=True, exist_ok=True) + out_path2.write_text(json.dumps(merged2, ensure_ascii=False), encoding="utf-8") + print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges") + elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")): # User ran `graphify ` directly — treat as `graphify extract `. # Common when following the PowerShell note in README (`graphify .`) or diff --git a/graphify/build.py b/graphify/build.py index 12accd7d0..cc229fdaa 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -22,6 +22,7 @@ # from __future__ import annotations import json +import os import re import sys import unicodedata @@ -64,10 +65,22 @@ def _normalize_id(s: str) -> str: return cleaned.strip("_").casefold() -def _norm_source_file(p: str | None) -> str | None: - """Normalize path separators to forward slashes so Windows backslash paths - and POSIX paths from semantic subagents resolve to the same node identity.""" - return p.replace("\\", "/") if p else p +def _norm_source_file(p: str | None, root: str | None = None) -> str | None: + """Normalize path separators and relativize absolute paths. + + Converts backslashes to forward slashes (Windows compatibility) and, when + root is provided, strips the absolute prefix from paths produced by semantic + subagents so source_file is always repo-relative (fixes #932). + """ + if not p: + return p + p = p.replace("\\", "/") + if root and os.path.isabs(p): + try: + p = Path(p).relative_to(root).as_posix() + except ValueError: + pass + return p def edge_data(G: nx.Graph, u: str, v: str) -> dict: @@ -91,12 +104,15 @@ def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]: return [raw] -def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: +def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph: """Build a NetworkX graph from an extraction dict. directed=True produces a DiGraph that preserves edge direction (source→target). directed=False (default) produces an undirected Graph for backward compatibility. + root: if given, absolute source_file paths from semantic subagents are made + relative to root so all nodes share a consistent path key (#932). """ + _root = str(Path(root).resolve()) if root else None # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. if "edges" not in extraction and "links" in extraction: extraction = dict(extraction, edges=extraction["links"]) @@ -137,7 +153,7 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: G: nx.Graph = nx.DiGraph() if directed else nx.Graph() for node in extraction.get("nodes", []): if "source_file" in node: - node["source_file"] = _norm_source_file(node["source_file"]) + node["source_file"] = _norm_source_file(node["source_file"], _root) G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) node_set = set(G.nodes()) # Normalized ID map: lets edges survive when the LLM generates IDs with @@ -161,7 +177,7 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: continue # skip edges to external/stdlib nodes - expected, not an error attrs = {k: v for k, v in edge.items() if k not in ("source", "target")} if "source_file" in attrs: - attrs["source_file"] = _norm_source_file(attrs["source_file"]) + attrs["source_file"] = _norm_source_file(attrs["source_file"], _root) # Preserve original edge direction - undirected graphs lose it otherwise, # causing display functions to show edges backwards. attrs["_src"] = src @@ -179,6 +195,7 @@ def build( directed: bool = False, dedup: bool = True, dedup_llm_backend: str | None = None, + root: str | Path | None = None, ) -> nx.Graph: """Merge multiple extraction results into one graph. @@ -187,6 +204,7 @@ def build( dedup=True (default) runs entity deduplication before building the graph. dedup_llm_backend: if set (e.g. "gemini", "claude", or "kimi"), uses LLM to resolve ambiguous pairs in the 75–92 Jaro-Winkler score zone. + root: if given, absolute source_file paths are made relative to root (#932). Extractions are merged in order. For nodes with the same ID, the last extraction's attributes win (NetworkX add_node overwrites). Pass AST @@ -206,7 +224,7 @@ def build( combined["nodes"], combined["edges"], communities={}, dedup_llm_backend=dedup_llm_backend, ) - return build_from_json(combined, directed=directed) + return build_from_json(combined, directed=directed, root=root) def _norm_label(label: str) -> str: @@ -268,11 +286,13 @@ def build_merge( directed: bool = False, dedup: bool = True, dedup_llm_backend: str | None = None, + root: str | Path | None = None, ) -> nx.Graph: """Load existing graph.json, merge new chunks into it, and save back. Never replaces - only grows (or prunes deleted-file nodes via prune_sources). Safe to call repeatedly: existing nodes and edges are preserved. + root: if given, absolute source_file paths in new_chunks are made relative (#932). """ graph_path = Path(graph_path) if graph_path.exists(): @@ -293,7 +313,7 @@ def build_merge( base = [] all_chunks = base + list(new_chunks) - G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend) + G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, root=root) # Prune nodes and edges from deleted source files if prune_sources: diff --git a/graphify/cache.py b/graphify/cache.py index bdaf1e773..2052cf7aa 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -1,6 +1,7 @@ # per-file extraction cache - skip unchanged files on re-run from __future__ import annotations +import atexit import hashlib import json import os @@ -23,6 +24,65 @@ def _body_content(content: bytes) -> bytes: return content +# Stat-based index: maps absolute path → {size, mtime_ns, hash}. +# Loaded once per process, flushed via atexit. Skips full file reads when +# size+mtime_ns are unchanged — same trade-off as make(1). +# Correctness risks: `touch` causes a harmless extra re-hash; same-size edits +# within NFS second-resolution mtime have a 1-second window (same as make). +# Use `graphify extract --force` to bypass when needed. +_stat_index: dict[str, dict] = {} +_stat_index_root: Path | None = None +_stat_index_dirty: bool = False + + +def _stat_index_file(root: Path) -> Path: + _out = Path(_GRAPHIFY_OUT) + base = _out if _out.is_absolute() else Path(root).resolve() / _out + return base / "cache" / "stat-index.json" + + +def _ensure_stat_index(root: Path) -> None: + global _stat_index, _stat_index_root, _stat_index_dirty + if _stat_index_root is not None: + return + _stat_index_root = Path(root).resolve() + p = _stat_index_file(_stat_index_root) + if p.exists(): + try: + _stat_index = json.loads(p.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + _stat_index = {} + else: + _stat_index = {} + atexit.register(_flush_stat_index) + + +def _flush_stat_index() -> None: + global _stat_index_dirty, _stat_index_root + if not _stat_index_dirty or _stat_index_root is None: + return + p = _stat_index_file(_stat_index_root) + try: + p.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=p.parent, prefix="stat-index.", suffix=".tmp") + try: + os.write(fd, json.dumps(_stat_index, separators=(",", ":")).encode()) + os.close(fd) + os.replace(tmp, p) + except Exception: + try: + os.close(fd) + except OSError: + pass + try: + os.unlink(tmp) + except OSError: + pass + except OSError: + pass + _stat_index_dirty = False + + def _normalize_path(path: Path) -> Path: """Normalize path for consistent cache keys across Windows path spellings.""" import sys @@ -37,6 +97,10 @@ def _normalize_path(path: Path) -> Path: def file_hash(path: Path, root: Path = Path(".")) -> str: """SHA256 of file contents + path relative to root. + Uses a stat-based fastpath (size + mtime_ns) to skip full reads when the + file hasn't changed. Falls through to full SHA256 on first encounter or + when stat changes. Index is flushed atomically at process exit. + Using a relative path (not absolute) makes cache entries portable across machines and checkout directories, so shared caches and CI work correctly. Falls back to the resolved absolute path if the file is outside root. @@ -44,10 +108,25 @@ def file_hash(path: Path, root: Path = Path(".")) -> str: For Markdown files (.md), only the body below the YAML frontmatter is hashed, so metadata-only changes (e.g. reviewed, status, tags) do not invalidate the cache. """ + global _stat_index_dirty p = _normalize_path(Path(path)) root = _normalize_path(Path(root)) if not p.is_file(): raise IsADirectoryError(f"file_hash requires a file, got: {p}") + + _ensure_stat_index(root) + abs_key = str(p.resolve()) + st: "os.stat_result | None" = None + try: + st = p.stat() + entry = _stat_index.get(abs_key) + if (entry + and entry.get("size") == st.st_size + and entry.get("mtime_ns") == st.st_mtime_ns): + return entry["hash"] + except OSError: + pass + raw = p.read_bytes() content = _body_content(raw) if p.suffix.lower() == ".md" else raw h = hashlib.sha256() @@ -58,7 +137,13 @@ def file_hash(path: Path, root: Path = Path(".")) -> str: h.update(rel.as_posix().lower().encode()) except ValueError: h.update(p.resolve().as_posix().lower().encode()) - return h.hexdigest() + digest = h.hexdigest() + + if st is not None: + _stat_index[abs_key] = {"size": st.st_size, "mtime_ns": st.st_mtime_ns, "hash": digest} + _stat_index_dirty = True + + return digest def cache_dir(root: Path = Path("."), kind: str = "ast") -> Path: diff --git a/graphify/extract.py b/graphify/extract.py index e747df04b..dbe8f0e4f 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -2755,7 +2755,7 @@ def extract_sql(path: Path) -> dict: except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} - stem = re.sub(r"[^a-z0-9]", "_", path.stem.lower()) + stem = _file_stem(path) str_path = str(path) file_nid = _make_id(str_path) nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code", @@ -4222,15 +4222,20 @@ def _resolve_cross_file_imports( language = Language(tspython.language()) parser = Parser(language) - # Pass 1: name → node_id across all files - # Map: stem → {ClassName: node_id} + # Pass 1: _file_stem(path) → {ClassName: node_id} + # Keyed by directory-qualified stem (e.g. "auth_models") to avoid collisions + # when multiple files share the same filename in different directories. + # A secondary bare-stem index handles absolute imports where only the module + # name is known — first writer wins when names collide (inherently ambiguous). stem_to_entities: dict[str, dict[str, str]] = {} + bare_to_qualified: dict[str, str] = {} for file_result in per_file: for node in file_result.get("nodes", []): src = node.get("source_file", "") if not src: continue - stem = Path(src).stem + src_path = Path(src) + fq_stem = _file_stem(src_path) label = node.get("label", "") nid = node.get("id", "") # Index class-level entities only. Function/method labels end in "()" @@ -4244,11 +4249,13 @@ def _resolve_cross_file_imports( and "_" not in label[:1] and node.get("file_type") != "rationale" ): - stem_to_entities.setdefault(stem, {})[label] = nid + stem_to_entities.setdefault(fq_stem, {})[label] = nid + if src_path.stem not in bare_to_qualified: + bare_to_qualified[src_path.stem] = fq_stem # Pass 2: for each file, find `from .X import A, B, C` and resolve new_edges: list[dict] = [] - stem_to_path: dict[str, Path] = {p.stem: p for p in paths} + stem_to_path: dict[str, Path] = {_file_stem(p): p for p in paths} for file_result, path in zip(per_file, paths): stem = _file_stem(path) @@ -4279,21 +4286,28 @@ def _resolve_cross_file_imports( # Find the module name - handles both absolute and relative imports. # Relative: `from .models import X` → relative_import → dotted_name # Absolute: `from models import X` → module_name field - target_stem: str | None = None + # target_fq is the directory-qualified stem used as the key in + # stem_to_entities. Relative imports are resolved exactly via the + # importing file's directory; absolute imports fall back to the + # bare-stem secondary index (first-writer-wins when names collide). + target_fq: str | None = None for child in node.children: if child.type == "relative_import": - # Dig into relative_import → dotted_name → identifier for sub in child.children: if sub.type == "dotted_name": raw = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace") - target_stem = raw.split(".")[-1] + bare = raw.split(".")[-1] + # Resolve relative import to exact qualified stem. + candidate = path.parent / f"{bare}.py" + target_fq = _file_stem(candidate) break break - if child.type == "dotted_name" and target_stem is None: + if child.type == "dotted_name" and target_fq is None: raw = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace") - target_stem = raw.split(".")[-1] + bare = raw.split(".")[-1] + target_fq = bare_to_qualified.get(bare) - if not target_stem or target_stem not in stem_to_entities: + if not target_fq or target_fq not in stem_to_entities: return # Collect imported names: dotted_name children of import_from_statement @@ -4320,7 +4334,7 @@ def _resolve_cross_file_imports( line = node.start_point[0] + 1 for name in imported_names: - tgt_nid = stem_to_entities[target_stem].get(name) + tgt_nid = stem_to_entities[target_fq].get(name) if tgt_nid: for src_class_nid in local_classes: new_edges.append({ diff --git a/tests/test_build.py b/tests/test_build.py index 54497b461..85d59fd5e 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -298,3 +298,49 @@ def test_edge_data_node_link_multigraph_roundtrip(): assert d.get("relation") in ("calls", "references") ds = edge_datas(G, "a", "b") assert len(ds) == 2 + + +def test_build_from_json_relativizes_absolute_source_file(tmp_path): + """Semantic subagents emit absolute source_file paths; build_from_json must + relativize them to root so MCP traversal works correctly (#932).""" + root = tmp_path / "myproject" + root.mkdir() + abs_path = str(root / "docs" / "overview.md") + extraction = { + "nodes": [ + {"id": "overview_intro", "label": "Intro", "source_file": abs_path, "file_type": "document"}, + ], + "edges": [ + {"source": "overview_intro", "target": "overview_intro", + "relation": "self", "confidence": "EXTRACTED", "confidence_score": 1.0, + "source_file": abs_path}, + ], + } + G = build_from_json(extraction, root=root) + sf = G.nodes["overview_intro"]["source_file"] + assert not sf.startswith("/"), f"source_file still absolute: {sf}" + assert sf == "docs/overview.md" + + +def test_build_relativizes_absolute_source_file(tmp_path): + """build() passes root through to build_from_json (#932).""" + root = tmp_path / "proj" + root.mkdir() + abs_path = str(root / "src" / "main.py") + extraction = { + "nodes": [{"id": "main_fn", "label": "main", "source_file": abs_path, "file_type": "code"}], + "edges": [], + } + G = build([extraction], root=root) + sf = G.nodes["main_fn"]["source_file"] + assert sf == "src/main.py" + + +def test_build_from_json_relative_source_file_unchanged(tmp_path): + """Already-relative source_file paths must not be modified.""" + extraction = { + "nodes": [{"id": "foo_bar", "label": "bar", "source_file": "src/foo.py", "file_type": "code"}], + "edges": [], + } + G = build_from_json(extraction, root=tmp_path) + assert G.nodes["foo_bar"]["source_file"] == "src/foo.py" diff --git a/tests/test_detect.py b/tests/test_detect.py index 869c6a0dd..0337a77cd 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -555,3 +555,56 @@ def test_sensitive_secret_handler_txt(): def test_sensitive_token_config_yaml(): # "token_config.yaml": "token" followed by "_" (not alpha) → flagged. assert _is_sensitive(Path("token_config.yaml")) + + +# ── Issue #933: failed-chunk files must not be frozen in manifest ───────────── + +def test_save_manifest_skips_semantic_hash_for_files_without_cache(tmp_path): + """Files in failed chunks have no semantic cache entry; save_manifest must + leave their semantic_hash empty so detect_incremental re-queues them (#933).""" + import json + from graphify.cache import save_cached + + doc1 = tmp_path / "docs" / "a.md" + doc2 = tmp_path / "docs" / "b.md" + doc1.parent.mkdir() + doc1.write_text("# A\n\ncontent a") + doc2.write_text("# B\n\ncontent b") + + # Simulate: doc1's chunk succeeded (has a cache entry), doc2's chunk failed (no entry). + save_cached(doc1, {"nodes": [{"id": "a", "source_file": str(doc1)}], "edges": [], "hyperedges": []}, root=tmp_path, kind="semantic") + # doc2: no cache entry written + + files = {"document": [str(doc1), str(doc2)]} + manifest_path = str(tmp_path / "manifest.json") + + # Simulate what __main__.py now does: only include files with semantic output. + sem_extracted = {str(doc1)} # doc2 not present — failed chunk + sem_types = {"document", "paper", "image"} + safe_files = { + ftype: [f for f in flist if ftype not in sem_types or f in sem_extracted] + for ftype, flist in files.items() + } + save_manifest(safe_files, manifest_path) + + manifest = json.loads(Path(manifest_path).read_text()) + assert str(doc1) in manifest, "successful file must be in manifest" + assert manifest[str(doc1)]["semantic_hash"] != "", "successful file must have semantic_hash" + assert str(doc2) not in manifest, "failed-chunk file must be absent from manifest" + + + +def test_save_manifest_without_filter_unchanged_for_code(tmp_path): + """Code files must be stamped in the manifest regardless of semantic cache.""" + import json + + py = tmp_path / "main.py" + py.write_text("print('hello')") + + files = {"code": [str(py)]} + manifest_path = str(tmp_path / "manifest.json") + save_manifest(files, manifest_path) + + manifest = json.loads(Path(manifest_path).read_text()) + assert str(py) in manifest + assert manifest[str(py)]["ast_hash"] != ""