From cd8d99b2f472b3f156f7a569810270989591cf0e Mon Sep 17 00:00:00 2001 From: abhay-codes07 Date: Fri, 11 Sep 2026 23:26:10 +0530 Subject: [PATCH] perf(resolution): memoize Path.resolve() across the symbol-resolution passes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The symbol-resolution and cross-file import passes call Path.resolve() once per import/export/use fact and per node — ~44k calls over a few hundred distinct corpus paths, each walking nt._getfinalpathname on Windows. A per-(path, cwd) memo collapses that to one syscall per distinct path. On a 364-file self-corpus a sequential extract drops from ~27s to ~14s, graph byte-identical. The cwd is part of the key so a relative path resolved after a chdir misses rather than replaying a stale answer. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01JJbLfztSxm2tH5cJwBbe9q (cherry picked from commit 5ca30dfd8adade5d2459355e25c9f4fa5de76973) --- graphify/extractors/resolution.py | 97 +++++++++++++++++++------------ tests/test_resolve_memoization.py | 70 ++++++++++++++++++++++ 2 files changed, 129 insertions(+), 38 deletions(-) create mode 100644 tests/test_resolve_memoization.py diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 22e250a8f..62bb4a6e1 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -138,7 +138,7 @@ def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[st # Skip scoped npm package configs (e.g. @tsconfig/svelte) — not on disk. if not ext or ext.startswith("@"): continue - extended_path = (base_dir / ext).resolve() + extended_path = _resolve_cached((base_dir / ext)) if not extended_path.suffix: extended_path = extended_path.with_suffix(".json") if extended_path.exists(): @@ -201,7 +201,7 @@ def _find_js_config(start_dir: Path) -> "tuple[Path, Path] | None": tsconfig.json wins when both sit in one directory, matching tsc and editors, which consult jsconfig.json only when there is no tsconfig.json. """ - current = start_dir.resolve() + current = _resolve_cached(start_dir) for candidate in [current, *current.parents]: for name in ("tsconfig.json", "jsconfig.json"): config = candidate / name @@ -331,7 +331,7 @@ def _resolve_tsconfig_alias(raw: str, aliases: dict[str, list[str]], return first def _find_workspace_root(start_dir: Path) -> Path | None: - current = start_dir.resolve() + current = _resolve_cached(start_dir) for candidate in [current, *current.parents]: if (candidate / "pnpm-workspace.yaml").exists(): return candidate @@ -440,7 +440,7 @@ def _contained_in_package(resolved: Path, package_dir: Path) -> bool: (e.g. "./evil": "../../../etc/passwd"). Only accept paths that stay within package_dir after resolution.""" try: - return resolved.resolve().is_relative_to(package_dir.resolve()) + return _resolve_cached(resolved).is_relative_to(_resolve_cached(package_dir)) except ValueError: return False @@ -521,7 +521,7 @@ def _find_js_project_anchor(start_dir: Path) -> Path: """ from graphify.detect import _find_vcs_root - current = start_dir.resolve() + current = _resolve_cached(start_dir) home = Path.home() for candidate in [current, *current.parents]: @@ -552,7 +552,7 @@ def _load_package_imports(start_dir: Path) -> "tuple[Path, dict] | None": Returns (package_dir, imports) or None. The nearest package.json wins even when it has no `imports` (Node never walks past the enclosing package). """ - current = start_dir.resolve() + current = _resolve_cached(start_dir) key = str(current) if key in _PACKAGE_IMPORTS_CACHE: return _PACKAGE_IMPORTS_CACHE[key] @@ -708,7 +708,7 @@ def _resolve_c_include_path(raw: str, str_path: str) -> "Path | None": """ if not raw: return None - candidate = (Path(str_path).parent / raw).resolve() + candidate = _resolve_cached((Path(str_path).parent / raw)) if candidate.is_file(): return candidate return None @@ -789,7 +789,7 @@ def _vue_mask_non_script(src: str) -> tuple[str, str | None]: def _cached_source_key(source_file: str, root_str: str, _cwd: str) -> str: """Resolve-and-relativize one source path, memoized (#perf). - ``Path.resolve()`` walks the path through ``nt._getfinalpathname`` / + ``_resolve_cached(Path)`` walks the path through ``nt._getfinalpathname`` / ``readlink`` syscalls, and ``_disambiguate_colliding_node_ids`` calls ``_source_key`` once per node, per edge endpoint and per raw_call — tens of thousands of calls for a few hundred distinct ``source_file`` strings. @@ -970,6 +970,27 @@ def _is_type_like_definition(node: dict) -> bool: return False return node.get("file_type") == "code" +@functools.lru_cache(maxsize=65536) +def _cached_realpath(path_str: str, _cwd: str) -> Path: + return Path(path_str).resolve() + + +def _resolve_cached(path: "Path | str") -> Path: + """``Path.resolve()`` with a per-(path, cwd) memo (#perf). + + The symbol-resolution passes resolve the same few hundred corpus paths + once per FACT — per import, per export, per use, per node — which on + Windows walks ``nt._getfinalpathname`` every time: 44k resolve calls and + ~15s of a sequential 364-file extract after the source-key memo alone. + Results are stable within a run (files do not move mid-extract); the cwd + is part of the key so a relative path resolved after a chdir misses the + cache instead of replaying a stale answer. Raises exactly like + ``Path.resolve()`` — callers keep their own try/except — and an + exception is never cached. + """ + return _cached_realpath(str(path), os.getcwd()) + + def _js_source_path(source_file: str, root: Path) -> Path | None: if not source_file: return None @@ -977,7 +998,7 @@ def _js_source_path(source_file: str, root: Path) -> Path | None: if not path.is_absolute(): path = root / path try: - return path.resolve() + return _resolve_cached(path) except Exception: return path @@ -1001,8 +1022,8 @@ def _apply_symbol_resolution_facts( ): return - path_by_resolved = {path.resolve(): path for path in paths} - source_file_id = {path.resolve(): _make_id(str(path)) for path in paths} + path_by_resolved = {_resolve_cached(path): path for path in paths} + source_file_id = {_resolve_cached(path): _make_id(str(path)) for path in paths} symbol_nodes: dict[tuple[Path, str], str] = {} # Member nodes (`.method()` labels) share their bare name with top-level # symbols once the leading dot is stripped. A module can only re-export @@ -1028,7 +1049,7 @@ def _apply_symbol_resolution_facts( symbol_nodes[key] = str(node["id"]) def ensure_symbol_node(path: Path, name: str, line: int) -> str: - resolved_path = path.resolve() + resolved_path = _resolve_cached(path) existing = symbol_nodes.get((resolved_path, name)) if existing is not None: return existing @@ -1088,15 +1109,15 @@ def _apply_symbol_resolution_facts( local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} for import_fact in facts.imports: - file_path = import_fact.file_path.resolve() + file_path = _resolve_cached(import_fact.file_path) local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = ( - import_fact.target_path.resolve(), + _resolve_cached(import_fact.target_path), import_fact.imported_name, ) pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {} for alias_fact in facts.aliases: - pending_aliases_by_file.setdefault(alias_fact.file_path.resolve(), []).append(alias_fact) + pending_aliases_by_file.setdefault(_resolve_cached(alias_fact.file_path), []).append(alias_fact) for file_path, aliases in pending_aliases_by_file.items(): local_aliases = local_aliases_by_file.setdefault(file_path, {}) @@ -1116,8 +1137,8 @@ def _apply_symbol_resolution_facts( star_exports_by_file: dict[Path, list[Path]] = {} for star_fact in facts.star_exports: - source_path = star_fact.file_path.resolve() - target_path = star_fact.target_path.resolve() + source_path = _resolve_cached(star_fact.file_path) + target_path = _resolve_cached(star_fact.target_path) star_exports_by_file.setdefault(source_path, []).append(target_path) source_id = source_file_id.get(source_path) if source_id is not None: @@ -1133,8 +1154,8 @@ def _apply_symbol_resolution_facts( ) for namespace_fact in facts.namespace_exports: - source_path = namespace_fact.file_path.resolve() - target_path = namespace_fact.target_path.resolve() + source_path = _resolve_cached(namespace_fact.file_path) + target_path = _resolve_cached(namespace_fact.target_path) namespace_id = ensure_symbol_node( namespace_fact.file_path, namespace_fact.exported_name, @@ -1165,10 +1186,10 @@ def _apply_symbol_resolution_facts( ) for export_fact in facts.exports: - file_path = export_fact.file_path.resolve() + file_path = _resolve_cached(export_fact.file_path) origin: tuple[Path, str] | None = None if export_fact.target_path is not None and export_fact.target_name is not None: - origin = (export_fact.target_path.resolve(), export_fact.target_name) + origin = (_resolve_cached(export_fact.target_path), export_fact.target_name) elif export_fact.local_name is not None: origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name) if origin is None and (file_path, export_fact.local_name) in symbol_nodes: @@ -1198,7 +1219,7 @@ def _apply_symbol_resolution_facts( ) def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: - target_path = target_path.resolve() + target_path = _resolve_cached(target_path) key = (target_path, imported_name) if seen is None: seen = set() @@ -1252,9 +1273,9 @@ def _apply_symbol_resolution_facts( for export_fact in facts.exports: if export_fact.target_path is not None and export_fact.target_name is not None: site = ( - export_fact.file_path.resolve(), + _resolve_cached(export_fact.file_path), f"L{export_fact.line}", - export_fact.target_path.resolve(), + _resolve_cached(export_fact.target_path), ) export_sites.setdefault(site, []).append(export_fact) owned_ids = {node.get("id") for node in nodes} @@ -1266,13 +1287,13 @@ def _apply_symbol_resolution_facts( if not target_file or source_path is None: continue target_path = Path(target_file) - site = (source_path, str(edge.get("source_location", "")), target_path.resolve()) + site = (source_path, str(edge.get("source_location", "")), _resolve_cached(target_path)) for export_fact in export_sites.get(site, []): expected = _make_id(_file_stem(target_path), export_fact.target_name) if edge.get("target") != expected: continue candidates, _ = exported_candidates( - (export_fact.target_path.resolve(), export_fact.target_name), frozenset() + (_resolve_cached(export_fact.target_path), export_fact.target_name), frozenset() ) if len(candidates) == 1: origin = next(iter(candidates)) @@ -1283,7 +1304,7 @@ def _apply_symbol_resolution_facts( break for import_fact in facts.imports: - source_id = source_file_id.get(import_fact.file_path.resolve()) + source_id = source_file_id.get(_resolve_cached(import_fact.file_path)) if source_id is None: continue origin_path, origin_symbol = resolve_exported_origin( @@ -1327,7 +1348,7 @@ def _apply_symbol_resolution_facts( # canonicalizes — or drop it when no file node id is available. owned = {str(n.get("id")) for n in nodes} for use_fact in facts.uses: - file_path = use_fact.file_path.resolve() + file_path = _resolve_cached(use_fact.file_path) target_id = None unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name) if unresolved_origin is not None: @@ -1764,7 +1785,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut trees: dict[Path, tuple[bytes, object]] = {} for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_cached(path) parsed = _parse_js_tree(path) if parsed is None: continue @@ -1786,7 +1807,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut target_path = _resolve_js_module_path(raw_module, path.parent) if target_path is None: continue - target_path = target_path.resolve() + target_path = _resolve_cached(target_path) for imported_name, local_name in _js_named_specifiers(node, source, "import_specifier"): facts.imports.append( _SymbolImportFact( @@ -1816,7 +1837,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_cached(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1839,7 +1860,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut target_path = _resolve_js_module_path(raw_module, path.parent) if target_path is None: continue - target_path = target_path.resolve() + target_path = _resolve_cached(target_path) namespace_name = _js_namespace_export_name(node, source) if namespace_name is not None: facts.namespace_exports.append( @@ -1912,7 +1933,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_cached(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1934,7 +1955,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_cached(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -2112,7 +2133,7 @@ def _resolve_python_namespace_dir(module_name: str, current_path: Path, root: Pa if not candidate.is_dir() or (candidate / "__init__.py").is_file(): return None try: - candidate.resolve().relative_to(root.resolve()) + _resolve_cached(candidate).relative_to(_resolve_cached(root)) except ValueError: return None return candidate @@ -2175,7 +2196,7 @@ def _collect_python_symbol_resolution_facts( if parsed is None: continue source, root_node = parsed - trees[path.resolve()] = parsed + trees[_resolve_cached(path)] = parsed for node in _walk_python_tree(root_node): if node.type != "import_from_statement": @@ -2226,7 +2247,7 @@ def _collect_python_symbol_resolution_facts( ) for path in py_paths: - parsed = trees.get(path.resolve()) + parsed = trees.get(_resolve_cached(path)) if parsed is None: continue source, root_node = parsed @@ -2827,7 +2848,7 @@ def _go_import_path_for_file( if not path.is_absolute(): path = root / path try: - directory = path.resolve().parent + directory = _resolve_cached(path).parent except OSError: directory = path.absolute().parent diff --git a/tests/test_resolve_memoization.py b/tests/test_resolve_memoization.py new file mode 100644 index 000000000..113b913ac --- /dev/null +++ b/tests/test_resolve_memoization.py @@ -0,0 +1,70 @@ +"""_resolve_cached: Path.resolve() with a per-(path, cwd) memo (#perf). + +The symbol-resolution passes call Path.resolve() once per import/export/use +fact and per node — tens of thousands of calls over a few hundred distinct +corpus paths, each walking nt._getfinalpathname / readlink. Memoizing on the +(path, cwd) pair collapses that to one syscall per distinct path; on a +364-file self-corpus a sequential extract dropped from ~27s to ~14s, with a +byte-identical graph (verified separately). +""" + +import os +from pathlib import Path + +import pytest + +try: + from graphify.extractors.resolution import _resolve_cached, _cached_realpath +except ImportError: # pre-fix tree + _resolve_cached = None + _cached_realpath = None + +needs_cache = pytest.mark.skipif( + _resolve_cached is None, reason="resolve memo not present" +) + + +@needs_cache +def test_matches_path_resolve(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "pkg").mkdir() + (tmp_path / "pkg" / "mod.py").write_text("x = 1\n", encoding="utf-8") + for arg in ("pkg/mod.py", str(tmp_path / "pkg" / "mod.py"), "missing.py"): + assert _resolve_cached(arg) == Path(arg).resolve() + assert _resolve_cached(Path(arg)) == Path(arg).resolve() + + +@needs_cache +def test_resolves_once_per_distinct_path(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "a.py").write_text("x = 1\n", encoding="utf-8") + _cached_realpath.cache_clear() + for _ in range(50): + _resolve_cached("a.py") + _resolve_cached(Path("a.py")) # str/Path collapse to one key + info = _cached_realpath.cache_info() + assert info.misses == 1 and info.hits == 99, info + + +@needs_cache +def test_cache_is_cwd_sensitive(tmp_path, monkeypatch): + d1, d2 = tmp_path / "one", tmp_path / "two" + for d in (d1, d2): + d.mkdir() + (d / "m.py").write_text("x = 1\n", encoding="utf-8") + monkeypatch.chdir(d1) + r1 = _resolve_cached("m.py") + monkeypatch.chdir(d2) + r2 = _resolve_cached("m.py") + assert r1 == d1.resolve() / "m.py" + assert r2 == d2.resolve() / "m.py" + + +@needs_cache +def test_exception_paths_match_resolve(tmp_path, monkeypatch): + """A path that resolve() can handle returns the same as a direct call; + the helper never swallows what resolve() would raise (callers keep their + own try/except, unchanged).""" + monkeypatch.chdir(tmp_path) + weird = "a/../b/./c.py" + assert _resolve_cached(weird) == Path(weird).resolve()