mirror of
https://github.com/safishamsi/graphify.git
synced 2026-10-14 07:59:40 +00:00
perf(resolution): parse each Python file once across both resolution passes
The symbol-resolution facts pass and the cross-file import pass each parsed
the whole .py corpus, back to back, in the main process — every file read
and tree-sitter-parsed twice for nothing. _parse_python_tree now memoizes on
(path, mtime, size), so the second pass reuses the first pass's tree while a
file changed between watch runs still re-parses. On a 364-file self-corpus a
sequential extract drops from ~27s to ~24s, graph byte-identical.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JJbLfztSxm2tH5cJwBbe9q
(cherry picked from commit dcbab53879)
This commit is contained in:
1 parent
a572b88777
commit
f077217191
2 files changed
+96
-16
No files matched your search
@@ -0,0 +1,66 @@
|
||||
"""_parse_python_tree memoizes across the two Python resolution passes (#perf).
|
||||
|
||||
The symbol-resolution facts pass and the cross-file import pass each parse the
|
||||
whole .py corpus, back to back, in the main process — so every file was read
|
||||
and tree-sitter-parsed twice. The parse is now memoized on (path, mtime, size):
|
||||
the second pass reuses the first pass's tree, a changed file still re-parses.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("tree_sitter_python")
|
||||
|
||||
from graphify.extractors.resolution import _parse_python_tree
|
||||
|
||||
try:
|
||||
from graphify.extractors.resolution import _parse_python_tree_cached
|
||||
except ImportError: # pre-fix tree
|
||||
_parse_python_tree_cached = None
|
||||
|
||||
needs_cache = pytest.mark.skipif(
|
||||
_parse_python_tree_cached is None, reason="parse memo not present"
|
||||
)
|
||||
|
||||
|
||||
def test_parses_and_returns_source_and_root(tmp_path):
|
||||
f = tmp_path / "m.py"
|
||||
f.write_text("def foo():\n return 1\n", encoding="utf-8")
|
||||
parsed = _parse_python_tree(f)
|
||||
assert parsed is not None
|
||||
source, root = parsed
|
||||
assert b"def foo" in source
|
||||
assert root.type == "module"
|
||||
|
||||
|
||||
def test_missing_file_returns_none(tmp_path):
|
||||
assert _parse_python_tree(tmp_path / "nope.py") is None
|
||||
|
||||
|
||||
@needs_cache
|
||||
def test_reuses_parse_within_a_run(tmp_path):
|
||||
f = tmp_path / "m.py"
|
||||
f.write_text("x = 1\n", encoding="utf-8")
|
||||
_parse_python_tree_cached.cache_clear()
|
||||
r1 = _parse_python_tree(f)
|
||||
for _ in range(9):
|
||||
_parse_python_tree(f)
|
||||
info = _parse_python_tree_cached.cache_info()
|
||||
assert info.misses == 1 and info.hits == 9, info
|
||||
# Same cached (source, root) object handed back.
|
||||
assert _parse_python_tree(f)[1] is r1[1]
|
||||
|
||||
|
||||
@needs_cache
|
||||
def test_edit_reparses(tmp_path):
|
||||
f = tmp_path / "m.py"
|
||||
f.write_text("x = 1\n", encoding="utf-8")
|
||||
_parse_python_tree_cached.cache_clear()
|
||||
src1, _ = _parse_python_tree(f)
|
||||
assert b"x = 1" in src1
|
||||
# A rewrite that changes size AND mtime must re-parse, not replay.
|
||||
time.sleep(0.01)
|
||||
f.write_text("x = 22222\ny = 3\n", encoding="utf-8")
|
||||
src2, _ = _parse_python_tree(f)
|
||||
assert b"y = 3" in src2, "edited file was served from a stale parse"
|
||||
Reference in new issue
Block a user