diff --git a/graphify/cache.py b/graphify/cache.py index 407ae467..ce47fec7 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -13,6 +13,51 @@ from pathlib import Path # absolute path ("/shared/graphify-out"). _GRAPHIFY_OUT = os.environ.get("GRAPHIFY_OUT", "graphify-out") +# AST cache entries are the output of graphify's own extractor code, so they +# are only valid for the version that wrote them: keying purely on file +# content means extractor fixes shipped in a new release keep serving stale +# pre-fix results. The AST cache is therefore namespaced by package version +# (cache/ast/v{version}/), with entries from other versions removed on first +# use. The semantic cache is deliberately NOT versioned — its entries are +# produced by the LLM from file contents, and invalidating them on every +# release would re-bill extraction for unchanged files. +try: + from importlib.metadata import version as _pkg_version + + _EXTRACTOR_VERSION = _pkg_version("graphifyy") +except Exception: + _EXTRACTOR_VERSION = "unknown" + +# Version dirs already swept this process — cleanup runs once per (base, version). +_cleaned_ast_dirs: set[str] = set() + + +def _cleanup_stale_ast_entries(ast_base: Path, current_dir: Path) -> None: + """Remove AST cache entries left behind by other graphify versions. + + Sweeps sibling ``v*/`` directories and unversioned ``*.json`` entries + (the pre-versioning layout) under ``cache/ast/``. Best-effort: failures + are ignored, stragglers are retried on the next run. + """ + key = str(current_dir) + if key in _cleaned_ast_dirs: + return + _cleaned_ast_dirs.add(key) + if not ast_base.is_dir(): + return + import shutil + + for child in ast_base.iterdir(): + if child == current_dir: + continue + try: + if child.is_dir() and child.name.startswith("v"): + shutil.rmtree(child, ignore_errors=True) + elif child.suffix == ".json": + child.unlink() + except OSError: + pass + def _body_content(content: bytes) -> bytes: """Strip YAML frontmatter from Markdown content, returning only the body.""" @@ -211,14 +256,22 @@ def _absolutize_source_files_in(payload: dict, root: Path) -> None: def cache_dir(root: Path = Path("."), kind: str = "ast") -> Path: - """Returns graphify-out/cache/{kind}/ - creates it if needed. + """Returns the cache directory for ``kind`` - creates it if needed. kind is "ast" or "semantic". Separate subdirectories prevent semantic cache entries from overwriting AST cache entries for the same source_file (#582). + + AST entries live in graphify-out/cache/ast/v{version}/ — namespaced by + graphify version because they depend on extractor code, not just file + contents. Semantic entries live unversioned in graphify-out/cache/semantic/ + (re-extraction costs LLM calls). """ _out = Path(_GRAPHIFY_OUT) base = _out if _out.is_absolute() else Path(root).resolve() / _out d = base / "cache" / kind + if kind == "ast": + d = d / f"v{_EXTRACTOR_VERSION}" + _cleanup_stale_ast_entries(d.parent, d) d.mkdir(parents=True, exist_ok=True) return d @@ -227,10 +280,13 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast") -> dict | """Return cached extraction for this file if hash matches, else None. Cache key: SHA256 of file contents. - Cache value: stored as graphify-out/cache/{kind}/{hash}.json + Cache value: stored as graphify-out/cache/{kind}/{hash}.json (AST entries + under the per-version subdirectory, see :func:`cache_dir`). - For kind="ast", also checks the legacy flat cache/ directory so users - upgrading from pre-0.5.3 don't lose their existing AST cache entries. + AST entries written by other graphify versions — including the legacy + flat cache/ layout (pre-0.5.3) and the unversioned cache/ast/ layout — + are deliberately not consulted: they were produced by a different + extractor and may be stale. Returns None if no cache entry or file has changed. """ try: @@ -249,17 +305,6 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast") -> dict | if isinstance(result, dict): _absolutize_source_files_in(result, root) return result - # Migration fallback: check legacy flat cache/ dir for AST entries - if kind == "ast": - legacy = Path(root).resolve() / _GRAPHIFY_OUT / "cache" / f"{h}.json" - if legacy.exists(): - try: - result = json.loads(legacy.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - return None - if isinstance(result, dict): - _absolutize_source_files_in(result, root) - return result return None @@ -325,11 +370,11 @@ def cached_files(root: Path = Path(".")) -> set[str]: # Legacy flat entries if base.is_dir(): hashes.update(p.stem for p in base.glob("*.json")) - # Namespaced entries - for kind in ("ast", "semantic"): + # Namespaced entries (ast/ recursively, covering per-version subdirs) + for kind, pattern in (("ast", "**/*.json"), ("semantic", "*.json")): d = base / kind if d.is_dir(): - hashes.update(p.stem for p in d.glob("*.json")) + hashes.update(p.stem for p in d.glob(pattern)) return hashes @@ -340,11 +385,11 @@ def clear_cache(root: Path = Path(".")) -> None: if base.is_dir(): for f in base.glob("*.json"): f.unlink() - # Namespaced entries - for kind in ("ast", "semantic"): + # Namespaced entries (ast/ recursively, covering per-version subdirs) + for kind, pattern in (("ast", "**/*.json"), ("semantic", "*.json")): d = base / kind if d.is_dir(): - for f in d.glob("*.json"): + for f in d.glob(pattern): f.unlink() diff --git a/tests/test_cache.py b/tests/test_cache.py index 1c4fbf21..840f5ef4 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -239,6 +239,96 @@ def test_cache_portable_across_roots(tmp_path): assert not str(repo_a) in loaded["nodes"][0]["source_file"] +# --- AST cache versioning ---------------------------------------------------- +# AST cache entries are the output of graphify's own extractor code, so they +# are only valid for the graphify version that wrote them. Keying purely on +# file content meant extractor fixes shipped in a new release kept serving +# stale pre-fix results. The AST cache is therefore namespaced by package +# version; the semantic cache is NOT (invalidating it would re-bill LLM +# extraction for unchanged files). + +def test_ast_cache_invalidated_on_version_bump(tmp_path, monkeypatch): + """An AST entry written by version X must not be served after upgrading + to version Y — the file is unchanged but the extractor is not.""" + import graphify.cache as cache_mod + + f = tmp_path / "mod.py" + f.write_text("def f(): pass\n") + + monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.0", raising=False) + save_cached(f, {"nodes": [{"id": "n1"}], "edges": []}, root=tmp_path, kind="ast") + assert load_cached(f, root=tmp_path, kind="ast") is not None + + monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.1", raising=False) + assert load_cached(f, root=tmp_path, kind="ast") is None, ( + "AST cache entry from a previous graphify version must not be served" + ) + + +def test_ast_cache_version_bump_cleans_stale_entries(tmp_path, monkeypatch): + """Upgrading removes AST entries left behind by previous versions so the + cache directory does not grow one full copy per release.""" + import graphify.cache as cache_mod + + f = tmp_path / "mod.py" + f.write_text("def f(): pass\n") + + monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.0", raising=False) + save_cached(f, {"nodes": [{"id": "n1"}], "edges": []}, root=tmp_path, kind="ast") + old_dir = cache_dir(tmp_path, "ast") + assert any(old_dir.glob("*.json")) + + monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.1", raising=False) + monkeypatch.setattr(cache_mod, "_cleaned_ast_dirs", set(), raising=False) + cache_dir(tmp_path, "ast") + assert not old_dir.exists(), ( + "stale AST version directory must be removed on upgrade" + ) + + +def test_legacy_unversioned_ast_entries_not_served(tmp_path): + """Entries written by pre-versioning graphify (flat cache/ or unversioned + cache/ast/) are by definition from an older extractor and must not be + served — that staleness is exactly what version namespacing fixes.""" + import json + from graphify.cache import file_hash, _GRAPHIFY_OUT + + f = tmp_path / "mod.py" + f.write_text("def f(): pass\n") + h = file_hash(f, tmp_path) + payload = json.dumps({"nodes": [{"id": "stale"}], "edges": []}) + + # Unversioned cache/ast/{hash}.json (pre-versioning layout) + unversioned = tmp_path / _GRAPHIFY_OUT / "cache" / "ast" + unversioned.mkdir(parents=True) + (unversioned / f"{h}.json").write_text(payload) + # Legacy flat cache/{hash}.json (pre-0.5.3 layout) + (unversioned.parent / f"{h}.json").write_text(payload) + + assert load_cached(f, root=tmp_path, kind="ast") is None + + +def test_semantic_cache_survives_version_bump(tmp_path, monkeypatch): + """The semantic cache is deliberately not versioned: entries are produced + by the LLM from file contents, and re-extraction costs real money.""" + import graphify.cache as cache_mod + + f = tmp_path / "doc.md" + f.write_text("# Title\n\nBody.\n") + + monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.0", raising=False) + save_cached(f, {"nodes": [{"id": "n1"}], "edges": []}, root=tmp_path, kind="semantic") + semantic_dir = cache_dir(tmp_path, "semantic") + + monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.1", raising=False) + monkeypatch.setattr(cache_mod, "_cleaned_ast_dirs", set(), raising=False) + cache_dir(tmp_path, "ast") # triggers stale-AST cleanup + assert load_cached(f, root=tmp_path, kind="semantic") is not None + assert any(semantic_dir.glob("*.json")), ( + "semantic entries must survive both the version bump and AST cleanup" + ) + + def test_save_cached_in_root_symlink_keeps_symlink_name(tmp_path): """``source_file`` for an in-root symlink must be stored under the symlink's own name, not the resolved target. Lower-impact than the