mirror of
https://github.com/safishamsi/graphify.git
synced 2026-07-12 18:37:12 +00:00
safishamsi
f9174943a2
#1649: detect_incremental tracks the converted markdown sidecar, and convert_office_file early-returned whenever the sidecar existed — so a .docx/ .xlsx edited after its first conversion never updated its sidecar and was reported "unchanged" forever, freezing the graph. It now re-converts when the source is newer than the sidecar (bumping the sidecar so the hash check catches it); an unchanged source still skips the rewrite (#1226). #1655: _md5_file/save_manifest/count_words used plain open()/stat(), which the Windows file APIs reject for absolute paths over 260 chars unless prefixed with `\\?\`. Deeply-nested files never hashed, their manifest entry never stabilized, and detect_incremental re-flagged them as changed every run. A new _os_path adds the extended-length prefix on win32 for change-detection I/O (mirror of cache._normalize_path, which strips it for keys). No-op elsewhere. #1656: detect() re-parsed every PDF/docx/text file to size the corpus on each run. Word counts are now memoized in the existing content-hash stat index (keyed by size + mtime_ns), so an unchanged file is parsed once. file_hash's fastpath is guarded so a word-count-only entry (no hash) can't KeyError, and both writers augment a co-located entry in place instead of clobbering the other's field. Full suite: 2906 passed, 3 skipped. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
53 lines
1.8 KiB
Python
53 lines
1.8 KiB
Python
"""#1656 — word counts are cached against each file's stat signature so
|
|
detect() doesn't re-parse every unchanged PDF/docx on each run just to size
|
|
the corpus.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from graphify import cache
|
|
|
|
|
|
def test_word_count_cached_until_file_changes(tmp_path, monkeypatch):
|
|
# Isolate the stat index to this tmp root.
|
|
monkeypatch.setattr(cache, "_stat_index", {})
|
|
monkeypatch.setattr(cache, "_stat_index_root", None)
|
|
|
|
f = tmp_path / "doc.txt"
|
|
f.write_text("one two three four five")
|
|
|
|
calls = {"n": 0}
|
|
def compute(p: Path) -> int:
|
|
calls["n"] += 1
|
|
return len(p.read_text().split())
|
|
|
|
assert cache.cached_word_count(f, tmp_path, compute) == 5
|
|
assert calls["n"] == 1
|
|
# Second call, file unchanged → served from cache, compute NOT re-run.
|
|
assert cache.cached_word_count(f, tmp_path, compute) == 5
|
|
assert calls["n"] == 1
|
|
|
|
# Change the file → recompute.
|
|
f.write_text("only three words now") # 4 words
|
|
assert cache.cached_word_count(f, tmp_path, compute) == 4
|
|
assert calls["n"] == 2
|
|
|
|
|
|
def test_word_count_augments_existing_hash_entry(tmp_path, monkeypatch):
|
|
# cached_word_count must not clobber a hash already stored for the file.
|
|
monkeypatch.setattr(cache, "_stat_index", {})
|
|
monkeypatch.setattr(cache, "_stat_index_root", None)
|
|
|
|
f = tmp_path / "m.py"
|
|
f.write_text("x = 1\n") # -> ["x", "=", "1"] == 3 tokens
|
|
h = cache.file_hash(f, tmp_path)
|
|
assert h
|
|
wc = cache.cached_word_count(f, tmp_path, lambda p: len(p.read_text().split()))
|
|
assert wc == 3
|
|
# The hash entry survives alongside the word_count.
|
|
assert cache.file_hash(f, tmp_path) == h
|
|
key = str(cache._normalize_path(f).resolve())
|
|
entry = cache._stat_index[key]
|
|
assert entry.get("hash") == h and entry.get("word_count") == 3
|