lee/graphify
1
0
Fork 0
mirror of https://github.com/safishamsi/graphify.git synced 2026-09-21 21:16:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(cache): namespace AST cache by graphify version (#1252)

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Timo Derstappen
2026-06-11 09:45:05 +02:00
co-authored by Cursor
parent 565026d3d1
commit 8401c50178
2 changed files with 156 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
graphify/cache.py
+66 -21
View File
@@ -13,6 +13,51 @@ from pathlib import Path
# absolute path ("/shared/graphify-out").
_GRAPHIFY_OUT = os.environ.get("GRAPHIFY_OUT", "graphify-out")
# AST cache entries are the output of graphify's own extractor code, so they
# are only valid for the version that wrote them: keying purely on file
# content means extractor fixes shipped in a new release keep serving stale
# pre-fix results. The AST cache is therefore namespaced by package version
# (cache/ast/v{version}/), with entries from other versions removed on first
# use. The semantic cache is deliberately NOT versioned — its entries are
# produced by the LLM from file contents, and invalidating them on every
# release would re-bill extraction for unchanged files.
try:
from importlib.metadata import version as _pkg_version
_EXTRACTOR_VERSION = _pkg_version("graphifyy")
except Exception:
_EXTRACTOR_VERSION = "unknown"
# Version dirs already swept this process — cleanup runs once per (base, version).
_cleaned_ast_dirs: set[str] = set()
def _cleanup_stale_ast_entries(ast_base: Path, current_dir: Path) -> None:
"""Remove AST cache entries left behind by other graphify versions.
Sweeps sibling ``v*/`` directories and unversioned ``*.json`` entries
(the pre-versioning layout) under ``cache/ast/``. Best-effort: failures
are ignored, stragglers are retried on the next run.
"""
key = str(current_dir)
if key in _cleaned_ast_dirs:
return
_cleaned_ast_dirs.add(key)
if not ast_base.is_dir():
return
import shutil
for child in ast_base.iterdir():
if child == current_dir:
continue
try:
if child.is_dir() and child.name.startswith("v"):
shutil.rmtree(child, ignore_errors=True)
elif child.suffix == ".json":
child.unlink()
except OSError:
pass
def _body_content(content: bytes) -> bytes:
"""Strip YAML frontmatter from Markdown content, returning only the body."""
@@ -211,14 +256,22 @@ def _absolutize_source_files_in(payload: dict, root: Path) -> None:
def cache_dir(root: Path = Path("."), kind: str = "ast") -> Path:
"""Returns graphify-out/cache/{kind}/ - creates it if needed.
"""Returns the cache directory for ``kind`` - creates it if needed.
kind is "ast" or "semantic". Separate subdirectories prevent semantic cache
entries from overwriting AST cache entries for the same source_file (#582).
AST entries live in graphify-out/cache/ast/v{version}/ — namespaced by
graphify version because they depend on extractor code, not just file
contents. Semantic entries live unversioned in graphify-out/cache/semantic/
(re-extraction costs LLM calls).
"""
_out = Path(_GRAPHIFY_OUT)
base = _out if _out.is_absolute() else Path(root).resolve() / _out
d = base / "cache" / kind
if kind == "ast":
d = d / f"v{_EXTRACTOR_VERSION}"
_cleanup_stale_ast_entries(d.parent, d)
d.mkdir(parents=True, exist_ok=True)
return d
@@ -227,10 +280,13 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast") -> dict |
"""Return cached extraction for this file if hash matches, else None.
Cache key: SHA256 of file contents.
Cache value: stored as graphify-out/cache/{kind}/{hash}.json
Cache value: stored as graphify-out/cache/{kind}/{hash}.json (AST entries
under the per-version subdirectory, see :func:`cache_dir`).
For kind="ast", also checks the legacy flat cache/ directory so users
upgrading from pre-0.5.3 don't lose their existing AST cache entries.
AST entries written by other graphify versions — including the legacy
flat cache/ layout (pre-0.5.3) and the unversioned cache/ast/ layout —
are deliberately not consulted: they were produced by a different
extractor and may be stale.
Returns None if no cache entry or file has changed.
"""
try:
@@ -249,17 +305,6 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast") -> dict |
if isinstance(result, dict):
_absolutize_source_files_in(result, root)
return result
# Migration fallback: check legacy flat cache/ dir for AST entries
if kind == "ast":
legacy = Path(root).resolve() / _GRAPHIFY_OUT / "cache" / f"{h}.json"
if legacy.exists():
try:
result = json.loads(legacy.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return None
if isinstance(result, dict):
_absolutize_source_files_in(result, root)
return result
return None
@@ -325,11 +370,11 @@ def cached_files(root: Path = Path(".")) -> set[str]:
# Legacy flat entries
if base.is_dir():
hashes.update(p.stem for p in base.glob("*.json"))
# Namespaced entries
for kind in ("ast", "semantic"):
# Namespaced entries (ast/ recursively, covering per-version subdirs)
for kind, pattern in (("ast", "**/*.json"), ("semantic", "*.json")):
d = base / kind
if d.is_dir():
hashes.update(p.stem for p in d.glob("*.json"))
hashes.update(p.stem for p in d.glob(pattern))
return hashes
@@ -340,11 +385,11 @@ def clear_cache(root: Path = Path(".")) -> None:
if base.is_dir():
for f in base.glob("*.json"):
f.unlink()
# Namespaced entries
for kind in ("ast", "semantic"):
# Namespaced entries (ast/ recursively, covering per-version subdirs)
for kind, pattern in (("ast", "**/*.json"), ("semantic", "*.json")):
d = base / kind
if d.is_dir():
for f in d.glob("*.json"):
for f in d.glob(pattern):
f.unlink()
tests/test_cache.py
+90
View File
@@ -239,6 +239,96 @@ def test_cache_portable_across_roots(tmp_path):
assert not str(repo_a) in loaded["nodes"][0]["source_file"]
# --- AST cache versioning ----------------------------------------------------
# AST cache entries are the output of graphify's own extractor code, so they
# are only valid for the graphify version that wrote them. Keying purely on
# file content meant extractor fixes shipped in a new release kept serving
# stale pre-fix results. The AST cache is therefore namespaced by package
# version; the semantic cache is NOT (invalidating it would re-bill LLM
# extraction for unchanged files).
def test_ast_cache_invalidated_on_version_bump(tmp_path, monkeypatch):
"""An AST entry written by version X must not be served after upgrading
to version Y — the file is unchanged but the extractor is not."""
import graphify.cache as cache_mod
f = tmp_path / "mod.py"
f.write_text("def f(): pass\n")
monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.0", raising=False)
save_cached(f, {"nodes": [{"id": "n1"}], "edges": []}, root=tmp_path, kind="ast")
assert load_cached(f, root=tmp_path, kind="ast") is not None
monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.1", raising=False)
assert load_cached(f, root=tmp_path, kind="ast") is None, (
"AST cache entry from a previous graphify version must not be served"
)
def test_ast_cache_version_bump_cleans_stale_entries(tmp_path, monkeypatch):
"""Upgrading removes AST entries left behind by previous versions so the
cache directory does not grow one full copy per release."""
import graphify.cache as cache_mod
f = tmp_path / "mod.py"
f.write_text("def f(): pass\n")
monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.0", raising=False)
save_cached(f, {"nodes": [{"id": "n1"}], "edges": []}, root=tmp_path, kind="ast")
old_dir = cache_dir(tmp_path, "ast")
assert any(old_dir.glob("*.json"))
monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.1", raising=False)
monkeypatch.setattr(cache_mod, "_cleaned_ast_dirs", set(), raising=False)
cache_dir(tmp_path, "ast")
assert not old_dir.exists(), (
"stale AST version directory must be removed on upgrade"
)
def test_legacy_unversioned_ast_entries_not_served(tmp_path):
"""Entries written by pre-versioning graphify (flat cache/ or unversioned
cache/ast/) are by definition from an older extractor and must not be
served — that staleness is exactly what version namespacing fixes."""
import json
from graphify.cache import file_hash, _GRAPHIFY_OUT
f = tmp_path / "mod.py"
f.write_text("def f(): pass\n")
h = file_hash(f, tmp_path)
payload = json.dumps({"nodes": [{"id": "stale"}], "edges": []})
# Unversioned cache/ast/{hash}.json (pre-versioning layout)
unversioned = tmp_path / _GRAPHIFY_OUT / "cache" / "ast"
unversioned.mkdir(parents=True)
(unversioned / f"{h}.json").write_text(payload)
# Legacy flat cache/{hash}.json (pre-0.5.3 layout)
(unversioned.parent / f"{h}.json").write_text(payload)
assert load_cached(f, root=tmp_path, kind="ast") is None
def test_semantic_cache_survives_version_bump(tmp_path, monkeypatch):
"""The semantic cache is deliberately not versioned: entries are produced
by the LLM from file contents, and re-extraction costs real money."""
import graphify.cache as cache_mod
f = tmp_path / "doc.md"
f.write_text("# Title\n\nBody.\n")
monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.0", raising=False)
save_cached(f, {"nodes": [{"id": "n1"}], "edges": []}, root=tmp_path, kind="semantic")
semantic_dir = cache_dir(tmp_path, "semantic")
monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.1", raising=False)
monkeypatch.setattr(cache_mod, "_cleaned_ast_dirs", set(), raising=False)
cache_dir(tmp_path, "ast") # triggers stale-AST cleanup
assert load_cached(f, root=tmp_path, kind="semantic") is not None
assert any(semantic_dir.glob("*.json")), (
"semantic entries must survive both the version bump and AST cleanup"
)
def test_save_cached_in_root_symlink_keeps_symlink_name(tmp_path):
"""``source_file`` for an in-root symlink must be stored under the
symlink's own name, not the resolved target. Lower-impact than the