diff --git a/graphify/__main__.py b/graphify/__main__.py index b838201b..312f3eef 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -4551,6 +4551,7 @@ def main() -> None: # Semantic extraction on docs/papers/images. Check cache first. from graphify.cache import ( check_semantic_cache as _check_semantic_cache, + prune_semantic_cache as _prune_semantic_cache, save_semantic_cache as _save_semantic_cache, ) sem_result: dict = { @@ -4641,6 +4642,32 @@ def main() -> None: sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) sem_result["input_tokens"] += fresh.get("input_tokens", 0) sem_result["output_tokens"] += fresh.get("output_tokens", 0) + + # Prune orphaned semantic cache entries. The semantic cache is + # content-hash-keyed and unversioned, so it is never swept by the AST + # version-cleanup: every content change or file deletion leaves a + # permanent orphan that accumulates unbounded (#1527). Sweep it against + # the FULL live document set (``files_by_type`` — present in both the + # incremental and full branches), NOT the incremental ``semantic_files`` + # changed-subset, which would delete every unchanged doc's valid entry. + # Best-effort: a prune failure must never break extraction. + try: + from graphify.cache import file_hash as _file_hash + _live_hashes: set[str] = set() + for _kind in ("document", "paper", "image"): + for _fp in files_by_type.get(_kind, []): + _abs = Path(_fp) + if not _abs.is_absolute(): + _abs = Path(out_root) / _abs + if not _abs.is_file(): + continue # deleted/missing — leave out so its entry is pruned + try: + _live_hashes.add(_file_hash(_abs, out_root)) + except OSError: + pass + _prune_semantic_cache(out_root, _live_hashes) + except Exception as exc: + print(f"[graphify extract] warning: could not prune semantic cache: {exc}", file=sys.stderr) stages.mark("semantic extract") pg_result: dict = {"nodes": [], "edges": []} diff --git a/graphify/cache.py b/graphify/cache.py index c00eaa55..f377889f 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -407,6 +407,44 @@ def clear_cache(root: Path = Path(".")) -> None: f.unlink() +def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: + """Remove orphaned semantic cache entries, returning the count pruned. + + The semantic cache is content-hash-keyed (``{file_hash}.json`` under + ``cache/semantic/``) and deliberately UNVERSIONED — entries are produced by + the LLM from file contents, so invalidating them on every release would + re-bill extraction. Because it is unversioned it is also never swept by the + AST version-cleanup, so every content change or file deletion leaves a + permanent orphan entry that accumulates unbounded. + + This sweeps ``cache/semantic/*.json`` and deletes any entry whose stem (the + content hash) is not in ``live_hashes`` — the hashes of the current live + document set. ``*.tmp`` atomic-write temporaries are skipped, and only this + directory is touched (never ``cache/ast/**`` or anything else). The + unversioned design is preserved: we prune by liveness, not by version. + + Best-effort, mirroring :func:`_cleanup_stale_ast_entries`: each unlink is + wrapped in ``try/except OSError`` and a failure is ignored. The worst-case + failure mode is benign — a surviving orphan costs only one re-extraction of + one doc on a future run, never incorrect output. + """ + _out = Path(_GRAPHIFY_OUT) + base = _out if _out.is_absolute() else Path(root).resolve() / _out + semantic_dir = base / "cache" / "semantic" + if not semantic_dir.is_dir(): + return 0 + pruned = 0 + for entry in semantic_dir.glob("*.json"): + if entry.stem in live_hashes: + continue + try: + entry.unlink() + pruned += 1 + except OSError: + pass + return pruned + + def check_semantic_cache( files: list[str], root: Path = Path("."), diff --git a/tests/test_cache.py b/tests/test_cache.py index b13f0850..730265be 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -420,3 +420,93 @@ def test_save_cached_in_root_symlink_keeps_symlink_name(tmp_path): f"cache must store symlink name, not resolved target; got " f"{on_disk['nodes'][0]['source_file']!r}" ) + + +def test_semantic_prune_removes_orphan_entries(tmp_path): + """Changing a file's content leaves the old content-hash entry orphaned; + pruning against the new live hash removes the stale entry and keeps the + current one.""" + from graphify.cache import prune_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# A\n\nContent A.\n") + h_a = file_hash(f, tmp_path) + save_cached(f, {"nodes": [{"id": "a"}], "edges": []}, root=tmp_path, kind="semantic") + + f.write_text("# B\n\nContent B.\n") + h_b = file_hash(f, tmp_path) + save_cached(f, {"nodes": [{"id": "b"}], "edges": []}, root=tmp_path, kind="semantic") + + semantic_dir = cache_dir(tmp_path, "semantic") + assert (semantic_dir / f"{h_a}.json").exists() + assert (semantic_dir / f"{h_b}.json").exists() + + pruned = prune_semantic_cache(tmp_path, {h_b}) + assert pruned == 1 + assert not (semantic_dir / f"{h_a}.json").exists() + assert (semantic_dir / f"{h_b}.json").exists() + + +def test_semantic_prune_keeps_live_unchanged_entries(tmp_path): + """Pruning against the FULL live set must keep every live entry — guards + the trap of pruning against an incremental changed-subset, which would + delete all unchanged docs' valid entries.""" + from graphify.cache import prune_semantic_cache + + live_hashes = set() + for i in range(5): + f = tmp_path / f"doc{i}.md" + f.write_text(f"# Doc {i}\n\nBody {i}.\n") + save_cached(f, {"nodes": [{"id": str(i)}], "edges": []}, root=tmp_path, kind="semantic") + live_hashes.add(file_hash(f, tmp_path)) + + semantic_dir = cache_dir(tmp_path, "semantic") + assert len(list(semantic_dir.glob("*.json"))) == 5 + + pruned = prune_semantic_cache(tmp_path, live_hashes) + assert pruned == 0 + assert len(list(semantic_dir.glob("*.json"))) == 5 + + +def test_semantic_prune_handles_deleted_file(tmp_path): + """An entry for a file that no longer exists (dropped from the live set) is + pruned.""" + from graphify.cache import prune_semantic_cache + + f = tmp_path / "gone.md" + f.write_text("# Gone\n\nWill be deleted.\n") + h = file_hash(f, tmp_path) + save_cached(f, {"nodes": [{"id": "g"}], "edges": []}, root=tmp_path, kind="semantic") + semantic_dir = cache_dir(tmp_path, "semantic") + assert (semantic_dir / f"{h}.json").exists() + + f.unlink() + # Live set is empty: the file is gone, so its entry must be pruned. + pruned = prune_semantic_cache(tmp_path, set()) + assert pruned == 1 + assert not (semantic_dir / f"{h}.json").exists() + + +def test_semantic_prune_ignores_ast_and_tmp(tmp_path): + """Prune touches only cache/semantic/*.json: AST entries and atomic-write + *.tmp temporaries are left untouched.""" + from graphify.cache import prune_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n\nBody.\n") + # AST entry (different subtree) must survive. + save_cached(f, {"nodes": [{"id": "ast"}], "edges": []}, root=tmp_path, kind="ast") + ast_dir = cache_dir(tmp_path, "ast") + assert len(list(ast_dir.glob("*.json"))) == 1 + + # A semantic orphan .json (to be pruned) plus a .tmp temporary (to survive). + semantic_dir = cache_dir(tmp_path, "semantic") + (semantic_dir / "deadbeef.json").write_text('{"nodes": [], "edges": []}') + tmp_entry = semantic_dir / "deadbeef.tmp" + tmp_entry.write_text("partial") + + pruned = prune_semantic_cache(tmp_path, set()) + assert pruned == 1 + assert not (semantic_dir / "deadbeef.json").exists() + assert tmp_entry.exists(), "*.tmp temporaries must not be swept" + assert len(list(ast_dir.glob("*.json"))) == 1, "AST entries must not be touched"