"""Deterministic structural extraction from source code using tree-sitter. Outputs nodes+edges dicts.""" from __future__ import annotations import hashlib import importlib import json import os import re import sys import textwrap from bisect import bisect_right from collections import Counter from dataclasses import dataclass, field from pathlib import Path, PurePath from typing import Any, Callable from .cache import load_cached, save_cached from .mcp_ingest import extract_mcp_config, is_mcp_config_path from .manifest_ingest import extract_package_manifest, is_package_manifest_path from .resolver_registry import ( LanguageResolver, register as register_language_resolver, run_language_resolvers, ) from .ruby_resolution import resolve_ruby_member_calls from .csharp_dispatch import resolve_csharp_interface_dispatch from .swift_dispatch import resolve_swift_protocol_dispatch from .pascal_resolution import resolve_pascal_inherited_calls from .markdown_resolution import MARKDOWN_MENTION_SUFFIXES, resolve_markdown_mentions # --- migrated to graphify/extractors/ (see graphify/extractors/MIGRATION.md) --- from graphify.extractors.base import ( # noqa: F401 _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id, _read_source_bytes, _read_source_text, _read_text, ) from graphify.extractors.apex import extract_apex # noqa: F401 from graphify.extractors.bash import extract_bash # noqa: F401 from graphify.extractors.blade import extract_blade # noqa: F401 from graphify.extractors.cobol import extract_cobol # noqa: F401 from graphify.extractors.csharp import ( CsharpNameResolver, _resolve_cross_file_csharp_imports, _resolve_csharp_type_references, ) from graphify.extractors.dart import extract_dart # noqa: F401 from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm # noqa: F401 from graphify.extractors.elixir import extract_elixir # noqa: F401 from graphify.extractors.erlang import extract_erlang, resolve_erlang_remote_calls # noqa: F401 from graphify.extractors.fortran import _cpp_preprocess, extract_fortran # noqa: F401 from graphify.extractors.go import _GO_PREDECLARED_FUNCS, extract_go # noqa: F401 from graphify.extractors.json_config import extract_json # noqa: F401 from graphify.extractors.commonlisp import extract_commonlisp # noqa: F401 from graphify.extractors.markdown import extract_markdown, _MD_LINK_INDEX_CACHE # noqa: F401 from graphify.extractors.ocaml import extract_ocaml # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 from graphify.extractors.r import extract_r, resolve_r_sourced_calls # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 from graphify.extractors.robot import extract_robot # noqa: F401 from graphify.extractors.rust import extract_rust # noqa: F401 from graphify.extractors.sln import extract_sln # noqa: F401 from graphify.extractors.solidity import ( # noqa: F401 extract_solidity, resolve_solidity_type_references, ) from graphify.extractors.sql import extract_sql # noqa: F401 from graphify.extractors.terraform import extract_terraform, prepare_terraform, resolve_terraform_modules # noqa: F401 from graphify.extractors.verilog import extract_verilog # noqa: F401 from graphify.extractors.vbnet import extract_vbnet, resolve_vbnet_partial_calls # noqa: F401 from graphify.extractors.zig import extract_zig # noqa: F401 from graphify.security import sanitize_metadata from graphify.paths import disambiguate_ambiguous_candidates from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401 from graphify.extractors.resolution import ( # noqa: E402,F401 _DECLDEF_HEADER_SUFFIXES, _DECLDEF_IMPL_SUFFIXES, _EXPORT_CONDITION_PRIORITY, _JS_INDEX_FILES, _JS_PRIMITIVE_TYPES, _JS_RESOLVE_EXTS, _PACKAGE_IMPORTS_CACHE, _SCAN_ROOT_NAMESPACE_CACHE, _TSCONFIG_ALIAS_CACHE, _TSCONFIG_BASEURL_CACHE, _VUE_SCRIPT_LANG_RE, _VUE_SCRIPT_RE, _WORKSPACE_MANIFEST_NAMES, _apply_symbol_resolution_facts, _augment_symbol_resolution_edges, _cached_realpath, _cached_source_key, _collect_js_symbol_resolution_facts, _collect_python_symbol_resolution_facts, _contained_in_package, _decldef_class_stem, _disambiguate_colliding_node_ids, _find_workspace_root, _go_import_path_for_file, _is_type_like_definition, _js_call_identifier, _js_default_export_name, _js_default_import_name, _js_export_clause, _js_export_statement_is_star, _js_exported_declaration_names, _js_lexical_aliases, _js_module_specifier, _js_named_specifiers, _js_namespace_export_name, _js_source_path, _js_top_level_function_bodies, _load_tsconfig_aliases, _load_tsconfig_base_url, _load_workspace_packages, _match_tsconfig_alias, _merge_decl_def_classes, _node_disambiguation_source_key, _package_entry_candidates, _parse_js_tree, _parse_python_tree, _pascal_class_stem_cache, _pascal_project_root, _pascal_resolve_class, _pascal_resolve_unit, _pascal_unit_cache, _pnpm_workspace_globs, _python_call_identifier, _python_import_from_module, _python_imported_names, _python_top_level_function_bodies, _read_tsconfig_aliases, _resolve_c_include_path, _resolve_cross_file_imports, _resolve_cross_file_java_imports, _resolve_export_target, _resolve_go_type_references, _resolve_java_type_references, _resolve_php_type_references, _resolve_js_import_path, _resolve_js_import_target, _resolve_js_module_path, _resolve_lua_import_target, _probe_python_module_candidate, _PYTHON_STDLIB_MODULE_NAMES, _resolve_python_module_path, _resolve_python_namespace_dir, _resolve_tsconfig_alias, _resolve_workspace_import, _source_key, _strip_jsonc, _ts_collect_type_refs, _ts_heritage_clause_entries, _ts_walk_class_members, _VUE_SCRIPT_LANG_RE as _SCRIPT_LANG_RE, _vue_mask_non_script, _walk_js_tree, _walk_python_tree, _workspace_globs, ) from graphify.symbol_resolution import resolve_bash_source_edges # noqa: E402 from graphify.extractors.engine import REFERENCE_CONTEXTS, _CSHARP_TYPE_PARAMETER_SCOPE_DECLARATIONS, _C_PRIMITIVE_TYPE_NODES, _JAVA_BUILTIN_TYPES, _JAVA_TYPE_PARAMETER_SCOPE_DECLARATIONS, _JS_FUNCTION_VALUE_TYPES, _JS_SCOPE_BOUNDARY, _PYTHON_ANNOTATION_NOISE, _PYTHON_TYPE_CONTAINERS, _RUBY_CLASS_FACTORIES, _c_collect_type_refs, _cpp_collect_type_refs, _cpp_declarator_name, _cpp_local_var_types, _csharp_attribute_names, _csharp_classify_base, _csharp_collect_type_refs, _csharp_extra_walk, _csharp_namespace_id, _csharp_namespace_name, _csharp_pre_scan_interfaces, _csharp_type_parameters_in_scope, _dynamic_import_js, _extract_generic, _find_body, _find_require_call, _get_cpp_func_name, _java_annotation_names, _java_collect_type_refs, _java_extra_walk, _java_type_parameters_in_scope, _js_collect_pattern_idents, _js_dispatch_value_idents, _js_extra_walk, _js_local_bound_names, _js_member_assignment_target, _js_module_bound_names, _kotlin_collect_type_refs, _kotlin_function_return_type_node, _kotlin_property_type_node, _kotlin_user_type_name, _php_collect_type_refs, _php_method_return_type_node, _php_name_text, _python_collect_assignment_targets, _python_collect_param_refs, _python_collect_type_refs, _python_local_bound_names, _python_module_bound_names, _python_param_names, _read_csharp_type_name, _require_imports_js, _ruby_const_last_name, _ruby_extra_walk, _ruby_local_class_bindings, _ruby_new_class_name, _scala_collect_type_refs, _semantic_reference_edge, _source_location, _swift_classify_base, _swift_collect_type_refs, _swift_constructor_type, _swift_declaration_keyword, _swift_extra_walk, _swift_local_var_types, _swift_pre_scan, _swift_property_name, _swift_property_type_node, _swift_receiver_name, _swift_user_type_name, _ts_decorator_name, _ts_descendant_decorators, _ts_emit_decorator_edges, _ts_extra_walk, _ts_method_name, _ts_receiver_type_table # noqa: E402,F401 from graphify.extractors.pascal import _PAS_BEGIN_END_TOKEN_RE, _PAS_CALL_RE, _PAS_END_SEMI_RE, _PAS_IMPL_HEADER_RE, _PAS_KEYWORDS, _PAS_METHOD_DECL_RE, _PAS_MODULE_RE, _PAS_TOKEN_RE, _PAS_TYPE_HEADER_RE, _PAS_USES_RE, _extract_pascal_regex, _pascal_find_body, _pascal_split_bases, _pascal_split_sections, _pascal_split_uses, _pascal_strip_comments, extract_pascal # noqa: E402,F401 from graphify.extractors.objc import _objc_local_var_types, extract_objc # noqa: E402,F401 from graphify.extractors.julia import extract_julia # noqa: E402,F401 _RECURSION_LIMIT = 10_000 # Language built-in globals that AST may classify as call targets when used as # constructors or coercion functions (e.g. String(x), Number(x), Boolean(x)). # Without this filter they become god-nodes accumulating spurious edges from # every call site. Filter applied at same-file and cross-file resolution. # See issue #726. def _raise_recursion_limit() -> None: if sys.getrecursionlimit() < _RECURSION_LIMIT: sys.setrecursionlimit(_RECURSION_LIMIT) def _safe_extract( extractor: Callable, path: Path, *, scan_root: Path | None = None ) -> dict: try: if extractor is extract_python: return extractor(path, root=scan_root) return extractor(path) except RecursionError: print(f" warning: skipped {path} (recursion limit exceeded)", file=sys.stderr, flush=True) return {"nodes": [], "edges": [], "error": "recursion_limit_exceeded"} except Exception as e: if os.environ.get("GRAPHIFY_DEBUG"): import traceback traceback.print_exc(file=sys.stderr) print(f" warning: skipped {path} ({type(e).__name__}: {e})", file=sys.stderr, flush=True) return {"nodes": [], "edges": [], "error": f"{type(e).__name__}: {e}"} def _file_node_id(rel_path: Path) -> str: """File-level node ID matching the skill.md spec: ``{parent_dir}_{stem}`` — one parent directory level, no extension. ``rel_path`` MUST be relative to the project root so top-level files collapse to a bare stem (``setup.py`` -> ``setup``) instead of picking up the root directory name. This must equal the ID semantic subagents generate, or AST and semantic extraction split a file into two disconnected ghost nodes (#1033).""" return _make_id(_file_stem(rel_path)) def _python_absolute_import_alias_files(paths, root) -> tuple[dict[str, set[str]], set[str]]: """Map importable absolute module ids to their scanned file paths. A file can be importable from more than one sys.path root inside the scan root (for example ``a/src/pkg/mod.py`` as either ``pkg.mod`` or ``src.pkg.mod``). Probe each in-root, non-package ancestor the same way the shared resolver does. Keeping every candidate lets the extraction passes fail closed when the same absolute module name identifies multiple files. """ try: root = Path(root).resolve() except OSError: root = Path(root) package_dirs: dict[Path, bool] = {} def _is_package(path: Path) -> bool: if path not in package_dirs: package_dirs[path] = (path / "__init__.py").is_file() return package_dirs[path] aliases: dict[str, set[str]] = {} scan_root_aliases: set[str] = set() for raw_path in paths: p = Path(raw_path) if p.suffix.lower() != ".py": # _resolve_python_module_path probes .py, not a lone .pyi file. continue try: p = p.resolve() rel = p.relative_to(root) except (ValueError, OSError, RuntimeError): continue module_path = p.parent if p.name == "__init__.py" else p.with_suffix("") candidate_roots = [root] for ancestor in p.parents: if ancestor == root: break try: ancestor.relative_to(root) except ValueError: break if not _is_package(ancestor): candidate_roots.append(ancestor) for candidate_root in candidate_roots: try: module_rel = module_path.relative_to(candidate_root) except ValueError: continue if not module_rel.parts or not all(part.isidentifier() for part in module_rel.parts): continue module_name = ".".join(module_rel.parts) candidate = candidate_root.joinpath(*module_rel.parts) try: hit = _probe_python_module_candidate(candidate) if hit is None or hit.resolve() != p: continue except (OSError, RuntimeError): continue module_id = _make_id(module_name) aliases.setdefault(module_id, set()).add(str(p)) if candidate_root == root: # The shared resolver probes the scan root before importer # ancestors. A unique root hit is therefore authoritative even # if a nested, independently importable tree has the same name. scan_root_aliases.add(module_id) return aliases, scan_root_aliases def _suppress_ambiguous_python_imports( edges: list[dict], nodes: list[dict], raw_calls: list[dict], ambiguous_modules: set[str], module_alias_files: dict[str, set[str]], root: Path, ) -> None: """Keep ambiguous Python imports dangling instead of choosing one file. The transient module name is also consumed by the symbol-level resolvers; removing it here prevents a direct file edge and any later inferred symbol or call edge from binding to an arbitrary same-named package. A plain ``import helper`` resolved to a scanned loose sibling in the importer's own non-package directory is an exception: that directory-local target is unambiguous even when another loose directory has its own ``helper.py``. """ node_ids = {n.get("id") for n in nodes if isinstance(n, dict)} ambiguous_bindings_by_file: dict[str, set[str]] = {} ambiguous_all_call_files: set[str] = set() def _is_resolved_loose_sibling(edge: dict, module_id: str) -> bool: """Trust only a concrete, scanned same-directory target for a bare import.""" target_file = edge.get("target_file") source_file = edge.get("source_file") if not target_file or not source_file: return False candidates = module_alias_files.get(module_id, set()) if not candidates: return False try: source_path = Path(source_file) if not source_path.is_absolute(): source_path = root / source_path source_path = source_path.resolve() target_path = Path(target_file).resolve() if target_path.parent != source_path.parent: return False if (source_path.parent / "__init__.py").is_file() or ( source_path.parent / "__init__.pyi" ).is_file(): return False return any(Path(candidate).resolve() == target_path for candidate in candidates) except (OSError, RuntimeError): return False kept_edges: list[dict] = [] for edge in edges: if not isinstance(edge, dict): kept_edges.append(edge) continue module_name = edge.pop("_python_import_module", None) bindings = edge.pop("_python_import_bindings", []) is_module_binding = edge.pop("_python_import_module_binding", False) if is_module_binding is True: edge["_python_plain_import"] = True marker_only = edge.pop("_python_import_marker_only", False) if not module_name: if not marker_only: kept_edges.append(edge) continue if not marker_only and edge.get("relation") not in ("imports", "imports_from"): kept_edges.append(edge) continue module_id = _make_id(module_name) module_is_ambiguous = module_id in ambiguous_modules if ( module_is_ambiguous and is_module_binding and "." not in module_name and _is_resolved_loose_sibling(edge, module_id) ): module_is_ambiguous = False if is_module_binding: if module_is_ambiguous: ambiguous_bindings_by_file.setdefault( str(edge.get("source_file", "")), set() ).update(local for _, local in bindings) else: for imported, local in bindings: if imported == "*": module_prefix = _make_id(module_name) + "_" if ( module_is_ambiguous or any(name.startswith(module_prefix) for name in ambiguous_modules) ): ambiguous_all_call_files.add( str(edge.get("source_file", "")) ) continue if ( module_is_ambiguous or _make_id(f"{module_name}.{imported}") in ambiguous_modules ): ambiguous_bindings_by_file.setdefault( str(edge.get("source_file", "")), set() ).add(local) if marker_only: continue if not module_is_ambiguous: kept_edges.append(edge) continue # A raw dotted alias could itself belong to an unrelated real node. Use # a stable, node-less id so the graph builder drops the unresolved edge # instead of fabricating a link to that node. identity = "\0".join(( str(module_name), str(edge.get("source_file", "")), str(edge.get("source_location", "")), str(edge.get("source", "")), )) target = _make_id(f"ambiguous_python_import_{hashlib.sha1(identity.encode()).hexdigest()[:12]}") while target in node_ids: target += "_" edge["target"] = target edge.pop("target_file", None) kept_edges.append(edge) edges[:] = kept_edges for raw_call in raw_calls: source_file = str(raw_call.get("source_file", "")) if source_file in ambiguous_all_call_files: raw_call["_ambiguous_python_import"] = True continue bindings = ambiguous_bindings_by_file.get(source_file) if not bindings: continue callee = str(raw_call.get("callee", "")) receiver = str(raw_call.get("receiver", "")) receiver_root = receiver.split(".", 1)[0] if callee in bindings or receiver_root in bindings: raw_call["_ambiguous_python_import"] = True def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: """Repoint Python absolute-import edges to the real file node under a nested (e.g. ``src/``) package root (#2072). Absolute imports target an id derived from the dotted module path (``_make_id('pkg.mod')`` -> ``pkg_mod``), but file-node ids are scan-root-relative (``src_pkg_mod`` when the code lives under ``src/``), so the edge dangles and is silently dropped — the graph loses most ``imports`` edges purely because of where the scan started. Build an alias map from the dotted-module id to the real file-node id by detecting each ``.py`` file's package root (the contiguous run of ancestor dirs carrying ``__init__.py``) and rewrite matching ``imports``/``imports_from`` edge targets. Guards: never shadow an existing node id, and drop an alias claimed by more than one file (ambiguous -> leave dangling, as before). Files whose package root IS the scan root are skipped (ids already coincide).""" try: root = Path(root).resolve() except OSError: root = Path(root) node_ids = {n.get("id") for n in all_nodes if isinstance(n, dict)} alias_to_files: dict[str, set[str]] = {} for p in paths: if p.suffix.lower() not in (".py", ".pyi"): continue try: rel = Path(p).resolve().relative_to(root) except (ValueError, OSError): continue parts = rel.parts if len(parts) < 2: continue # top-level file: scan-root-relative id already matches d = Path(p).resolve().parent levels = 0 # Bounded by the number of dirs between the file and the scan root, so a # pathological `/__init__.py` chain can't loop forever. while levels < len(parts) - 1 and (d / "__init__.py").is_file(): levels += 1 d = d.parent if levels == 0: continue # not inside a package (namespace pkg / loose module) mod_parts = parts[-(levels + 1):] # package dirs + the file itself if len(mod_parts) == len(parts): continue # package root == scan root: file-node id already coincides file_node = _file_node_id(rel) alias = _make_id(str(Path(*mod_parts).with_suffix(""))) alias_to_files.setdefault(alias, set()).add(file_node) if p.name in ("__init__.py", "__init__.pyi") and len(mod_parts) > 1: # `import pkg` / `from pkg import x` targets the package-dir id. pkg_alias = _make_id(str(Path(*mod_parts[:-1]))) alias_to_files.setdefault(pkg_alias, set()).add(file_node) alias_map = { a: next(iter(fs)) for a, fs in alias_to_files.items() if len(fs) == 1 and a not in node_ids } if not alias_map: return for e in all_edges: # Only repoint edges emitted from a Python file: a non-Python import edge # (e.g. C# `using Pkg.Mod;`, Java/Go dotted imports) can have a dangling # target string that coincides with a Python alias, and repointing it # would fabricate a cross-language import edge (#2072 review). if ( isinstance(e, dict) and e.get("relation") in ("imports", "imports_from") and str(e.get("source_file", "")).lower().endswith((".py", ".pyi")) ): tgt = e.get("target") if tgt in alias_map: e["target"] = alias_map[tgt] def _repoint_python_sibling_imports(paths, all_nodes, all_edges, root) -> None: """Repoint Python sibling-import edges to the real file node in directories without an __init__.py (#3430). When a Python file below the scan root lives in a non-package directory (no __init__.py in its immediate parent), plain imports of same-directory modules (e.g. `scripts/main.py: import greeter`) target a bare name (`greeter`), while the real file node is scan-root-relative (`scripts_greeter`). Because the directory is not a package, `_repoint_python_package_imports` skips it (levels == 0), leaving the edge dangling and causing downstream member calls (`greeter.greet()`) to be dropped. This pass is strictly importer-directory-local: - Only applies when the importing file's parent directory has no __init__.py. - Resolves only to unambiguous same-directory candidate modules in the scanned corpus. - Never builds a global alias map and never searches outside the importer's directory. - Preserves local aliases (e.g. `import greeter as g`). """ try: root = Path(root).resolve() except OSError: root = Path(root) node_ids = {n.get("id") for n in all_nodes if isinstance(n, dict)} # Index corpus modules by directory for non-package directories only. # dir_path -> {module_name_id: set_of_file_node_ids} dir_siblings: dict[Path, dict[str, set[str]]] = {} for p in paths: if p.suffix.lower() not in (".py", ".pyi"): continue try: p_res = Path(p).resolve() rel = p_res.relative_to(root) except (ValueError, OSError): continue file_node = _file_node_id(rel) if file_node not in node_ids: continue if p_res.name in ("__init__.py", "__init__.pyi"): # Sibling package directory inside parent_dir (parent_dir / subpkg / __init__.py) pkg_dir = p_res.parent parent_dir = pkg_dir.parent # A loose package named like a stdlib module (e.g. `logging/`) must not # capture a bare `import logging`, which means the stdlib module (#4261). if pkg_dir.name in _PYTHON_STDLIB_MODULE_NAMES: continue if not (parent_dir / "__init__.py").is_file() and not (parent_dir / "__init__.pyi").is_file(): mod_key = _make_id(pkg_dir.name) dir_siblings.setdefault(parent_dir, {}).setdefault(mod_key, set()).add(file_node) continue d = p_res.parent # PEP 328 guard: if the directory is a package, implicit relative imports # are forbidden in Python 3. Do not index packages as loose sibling directories. if (d / "__init__.py").is_file() or (d / "__init__.pyi").is_file(): continue # A loose module named like a stdlib module (e.g. `logging.py`) must not # capture a bare `import logging`, which means the stdlib module (#4261). if p_res.stem in _PYTHON_STDLIB_MODULE_NAMES: continue mod_key = _make_id(p_res.stem) dir_siblings.setdefault(d, {}).setdefault(mod_key, set()).add(file_node) # Require an unambiguous single candidate per module name in that directory. dir_alias_map: dict[Path, dict[str, str]] = { d: { mod: next(iter(fns)) for mod, fns in mod_map.items() if len(fns) == 1 } for d, mod_map in dir_siblings.items() } if not dir_alias_map: return for e in all_edges: if not ( isinstance(e, dict) and e.get("relation") in ("imports", "imports_from") and str(e.get("source_file", "")).lower().endswith((".py", ".pyi")) ): continue sf = e.get("source_file") if not sf: continue try: sf_path = Path(sf) if not sf_path.is_absolute(): sf_path = (root / sf_path).resolve() else: sf_path = sf_path.resolve() except OSError: continue imp_dir = sf_path.parent aliases = dir_alias_map.get(imp_dir) if not aliases: continue tgt = e.get("target") if tgt in aliases: repointed = aliases[tgt] if repointed != e.get("source"): e["target"] = repointed SEMANTIC_RELATIONS = frozenset({ "inherits", "implements", "mixes_in", "embeds", "references", "calls", "imports", "imports_from", "re_exports", "contains", "method", }) # Condition keys consulted when resolving an `exports` target, in priority # order. `default` is Node's catch-all and must be consulted LAST so a more # specific condition (source/import/module/etc.) wins when several match. # ── LanguageConfig dataclass ───────────────────────────────────────────────── # ── Generic helpers ─────────────────────────────────────────────────────────── # Scalar builtins and test-mock names that appear as type annotations but carry # no useful semantic meaning as graph nodes (#1147). Suppressed at the annotation # walker level so they are never created as nodes or emitted as edges. # java.lang (auto-imported) plus the ubiquitous java.util / java.io / java.time / # java.util.{stream,function,concurrent} / java.math / java.nio.file types that # appear as field, parameter, return, and generic-argument annotations. They never # resolve to a project node, so emitting `references` edges to them is pure noise # (mirrors _GO_PREDECLARED_TYPES / _PYTHON_ANNOTATION_NOISE). Suppressed at the # type-ref walker so they are never created as nodes or emitted as edges. The # boxed-scalar/`void` primitives are already dropped by grammar node type above; # these are the class/interface names the grammar reports as identifiers. # ── C / C++ type-ref helpers ───────────────────────────────────────────────── # ── Scala type-ref helpers ─────────────────────────────────────────────────── def _resolve_name(node, source: bytes, config: LanguageConfig) -> str | None: """Get the name from a node using config.name_field, falling back to child types.""" if config.resolve_function_name_fn is not None: # For C/C++ where the name is inside a declarator return None # caller handles this separately n = node.child_by_field_name(config.name_field) if n: return _read_text(n, source) for child in node.children: if child.type in config.name_fallback_child_types: return _read_text(child, source) return None # ── Import handlers ─────────────────────────────────────────────────────────── def _python_import_bindings(node, source: bytes) -> list[tuple[str, str]]: """Return (imported name, local binding) pairs from a Python import node.""" bindings: list[tuple[str, str]] = [] past_import = False for child in node.children: if child.type == "import": past_import = True continue if not past_import: continue if child.type == "dotted_name": imported = _read_text(child, source) bindings.append((imported, imported.split(".")[-1])) elif child.type == "aliased_import": name_node = child.child_by_field_name("name") alias_node = child.child_by_field_name("alias") if name_node is None: continue imported = _read_text(name_node, source) local = _read_text(alias_node, source) if alias_node is not None else imported.split(".")[-1] bindings.append((imported, local)) elif child.type == "wildcard_import": bindings.append(("*", "*")) return bindings def _missing_relative_module_name(attempted: Path, root: Path, raw: str) -> str: """Dotted module name, relative to ``root``, of an unresolved relative import. ``attempted`` is the file the import would have named (``pkg/missing.py`` or ``pkg/__init__.py``). Falls back to ``ref:`` + the raw specifier when the import climbs above the scan root, so the id never carries an absolute path. """ try: rel = attempted.relative_to(root) except ValueError: return f"ref:{raw}" parts = list(rel.with_suffix("").parts) if parts and parts[-1] == "__init__": parts.pop() return ".".join(parts) if parts else f"ref:{raw}" def _python_under_type_checking(node, source: bytes) -> bool: """True when ``node`` sits in the body of an ``if TYPE_CHECKING:`` block. Matches ``TYPE_CHECKING`` and ``.TYPE_CHECKING`` (``typing.``, ``t.``, ``typing_extensions.``). Only the guarded body counts: the ``else:`` branch of the same ``if`` runs at import time, and so does the body of ``if not TYPE_CHECKING:``. """ child, parent = node, node.parent while parent is not None: if ( parent.type in ("if_statement", "elif_clause") and parent.child_by_field_name("consequence") == child ): cond = parent.child_by_field_name("condition") if cond is not None and cond.type == "attribute": cond = cond.child_by_field_name("attribute") if ( cond is not None and cond.type == "identifier" and _read_text(cond, source) == "TYPE_CHECKING" ): return True child, parent = parent, parent.parent return False def _import_python( node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None, scan_root: Path | None = None, ) -> None: t = node.type current_path = Path(str_path) try: current_path = current_path.resolve() except OSError: pass root = Path(scan_root) if scan_root is not None else current_path.parent try: root = root.resolve() except OSError: pass # Imports under `if TYPE_CHECKING:` never run, so they are stamped `type_only` # like TS `import type`: find_import_cycles skips them, the edge stays (#3159). type_only = _python_under_type_checking(node, source) if t == "import_statement": for child in node.children: if child.type in ("dotted_name", "aliased_import"): raw = _read_text(child, source) raw_module, _, raw_alias = raw.partition(" as ") module_name = raw_module.strip().lstrip(".") local_binding = raw_alias.strip() if raw_alias else module_name.split(".")[0] target_path = _resolve_python_module_path( module_name, current_path, root, level=0 ) # Keep the target-file stamp even for a nested sys.path root so # incremental extraction can canonicalize imports whose target # is not in this batch. The corpus-wide ambiguity guard runs # before symbol resolution and clears this edge when another # scanned file claims the same absolute module name. if target_path is not None: try: if target_path.resolve() == current_path: target_path = None except OSError: pass tgt_nid = ( _make_id(str(target_path)) if target_path is not None else _make_id(module_name) ) edge = { "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, "_python_import_module": module_name, "_python_import_bindings": [(module_name, local_binding)], "_python_import_module_binding": True, } if target_path is not None: edge["target_file"] = str(target_path) if raw_alias: # `import pkg.mod as alias` binds the local name `alias`, not # `mod`'s own stem, to the module -- stash it so the cross-file # member-call resolver can match `alias.func()` against this # edge instead of dropping it (#2082). edge["local_alias"] = raw_alias.strip() if type_only: edge["type_only"] = True edges.append(edge) elif t == "import_from_statement": module_node = node.child_by_field_name("module_name") if module_node: raw = _read_text(module_node, source) target_path: "Path | None" = None if raw.startswith("."): # Relative import - resolve to full path so IDs match file node IDs dots = len(raw) - len(raw.lstrip(".")) module_name = raw.lstrip(".") target_path = _resolve_python_module_path( module_name, current_path, root, level=dots ) missing = False if target_path is None: base = current_path.parent for _ in range(dots - 1): base = base.parent rel = (module_name.replace(".", "/") + ".py") if module_name else "__init__.py" target_path = base / rel missing = not target_path.is_file() if missing: # No file behind the import: minting the id from the # attempted absolute path bakes the checkout location (and # OS username) into the graph, and the root-relative id # remap never rewrites it. Use the dotted module name # relative to the scan root, the id an unresolved absolute # import of the same module already gets. tgt_nid = _make_id(_missing_relative_module_name(target_path, root, raw)) else: tgt_nid = _make_id(str(target_path)) else: # Use the shared scan-root-aware resolver for absolute imports. # It stops at the corpus boundary and handles package roots the # same way as symbol resolution. A namespace package has no file # node of its own; the corpus pass will emit edges to any # imported submodules that exist on disk. target_path = _resolve_python_module_path(raw, current_path, root, level=0) if target_path is None and _resolve_python_namespace_dir( raw, current_path, root, level=0 ) is not None: # Namespace packages have no file node for the named # package, but their imported names can still bind to # ambiguous submodules. Carry a private marker so the # corpus-wide guard can suppress false symbol/call edges; # it is removed before any graph resolver sees the edges. edges.append({ "source": file_nid, "target": "", "relation": "_python_import_marker", "source_file": str_path, "_python_import_module": raw, "_python_import_bindings": _python_import_bindings(node, source), "_python_import_marker_only": True, }) return if target_path is not None: try: if target_path.resolve() == current_path: # Do not turn an import of the current file into a # self-loop. target_path = None except OSError: pass tgt_nid = _make_id(str(target_path)) if target_path is not None else _make_id(raw) edge = { "source": file_nid, "target": tgt_nid, "relation": "imports_from", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, } if not raw.startswith("."): edge["_python_import_module"] = raw edge["_python_import_bindings"] = _python_import_bindings(node, source) # Stamp the resolved target file (mirroring _import_js, #1814) so # the #2169 remap pass can canonicalize this edge's target on an # incremental run where the target file itself is not in the # batch — without it the target keeps an absolute-path-derived id # that matches no node in the merged graph and dangles (#2213). # Existence-gated: a speculative import of a nonexistent sibling # must stay dangling, exactly as before. The stamp is transient # and popped before graph.json ships. if target_path is not None: try: if target_path.is_file(): edge["target_file"] = str(target_path) except OSError: pass if type_only: edge["type_only"] = True edges.append(edge) def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: is_reexport = node.type == "export_statement" # Only handle export_statement if it has a `from` clause (re-export). # Pure exports like `export const x = 1` or `export { localVar }` have no source module. if is_reexport: has_from = any(child.type == "from" or (_read_text(child, source) == "from") for child in node.children if child.type in ("from", "identifier")) if not has_from: return # `import type {...} from` / `export type {...} from` are erased by the # TypeScript compiler: no runtime emit, no module-graph edge. The # dependency is still real for "what references this type", so the edge is # stamped `type_only` rather than dropped, and Import Cycles excludes it # the way it already excludes deferred `import(...)` (#1241) - on the # reporter's corpus all 3 reported cycles closed only through these # (#3123). The grammar keeps `import type from './x'` (a default binding # NAMED type) distinct: there `type` sits inside the import_clause, not as # a bare keyword child of the statement. A mixed `import { type B, C }` # stays a runtime edge - C is a runtime import. is_type_only = any( child.type == "type" and not child.is_named for child in node.children ) resolved_path: "Path | None" = None module_string = None for child in node.children: if child.type == "string": module_string = child break if child.type == "import_require_clause": # TS import-equals form: `import x = require("./m")`. The module # string sits inside the clause, not on the import_statement # itself, so the direct-child scan above never sees it. module_string = next( (sub for sub in child.children if sub.type == "string"), None ) break if module_string is not None: raw = _read_text(module_string, source).strip("'\"` ") resolved = _resolve_js_import_target(raw, str_path) if resolved is not None: tgt_nid, resolved_path = resolved # `_resolve_js_import_path` returns the attempted path when no # local file exists. Static ES imports must treat that as unresolved # rather than minting a checkout-specific target ID (#2457). try: is_file = resolved_path is not None and resolved_path.is_file() except OSError: is_file = False if resolved_path is not None and not is_file: tgt_nid = _make_id("ref", raw) resolved_path = None edge = { "source": file_nid, "target": tgt_nid, "relation": "imports_from", "context": "re-export" if is_reexport else "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, } # Stamp the resolved target file so a same-basename cross-extension # sibling (foo.ts importing/re-exporting ./foo.mjs) keys its target salt # by the TARGET's file rather than the importer's. Both files collapse to # the base id `foo`; without this the salted lookup mis-points the target # back onto the importer's own variant, a phantom self-loop (#1814). if resolved_path is not None: edge["target_file"] = str(resolved_path) if is_type_only: edge["type_only"] = True edges.append(edge) # Emit symbol-level edges for named imports/re-exports from local/aliased files. # e.g. `import { Foo, type Bar } from './bar'` → file → Foo, file → Bar (EXTRACTED) # e.g. `export { Foo } from './bar'` → file → Foo (re_exports edge) # Uses the same _make_id(target_stem, name) key that _extract_generic emits when # defining the symbol, so these edges wire importers directly to existing symbol nodes. if resolved_path is not None: target_stem = _file_stem(resolved_path) line = node.start_point[0] + 1 if is_reexport: # Handle: export { foo, bar } from './module' # export { default as baz } from './module' for child in node.children: if child.type == "export_clause": for spec in child.children: if spec.type == "export_specifier": # The exported name is the local name from the source module name_node = spec.child_by_field_name("name") if name_node: sym = _read_text(name_node, source) if sym == "default": continue # skip default re-exports for ID matching edges.append({ **({"type_only": True} if is_type_only else {}), "source": file_nid, "target": _make_id(target_stem, sym), "relation": "re_exports", "context": "re-export", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, # Which file this symbol target was synthesized # from, so the id-remap post-pass can repoint a # target the candidates rewrite never learns — # a barrel defines no symbols (#1983). Transient, # stripped at build like the #1814 stamp. "target_file": str(resolved_path), }) else: # Handle: import { Foo, type Bar } from './bar' for child in node.children: if child.type == "import_clause": for sub in child.children: if sub.type == "named_imports": for spec in sub.children: if spec.type == "import_specifier": name_node = spec.child_by_field_name("name") if name_node: sym = _read_text(name_node, source) edges.append({ "source": file_nid, "target": _make_id(target_stem, sym), "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, # See the re_exports stamp above (#1983). "target_file": str(resolved_path), }) def _import_java(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: def _walk_scoped(n) -> str: parts: list[str] = [] cur = n while cur: if cur.type == "scoped_identifier": name_node = cur.child_by_field_name("name") if name_node: parts.append(_read_text(name_node, source)) cur = cur.child_by_field_name("scope") elif cur.type == "identifier": parts.append(_read_text(cur, source)) break else: break parts.reverse() return ".".join(parts) for child in node.children: if child.type in ("scoped_identifier", "identifier"): path_str = _walk_scoped(child) module_name = path_str.split(".")[-1].strip("*").strip(".") or ( path_str.split(".")[-2] if len(path_str.split(".")) > 1 else path_str ) if module_name: tgt_nid = _make_id(module_name) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) break def _import_c(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: for child in node.children: if child.type in ("string_literal", "system_lib_string", "string"): raw = _read_text(child, source).strip('"<> ') # Quoted includes: try to resolve to a real file so the target ID # matches the node ID _extract_generic creates for that file. if child.type != "system_lib_string": resolved = _resolve_c_include_path(raw, str_path) if resolved is not None: tgt_nid = _make_id(str(resolved)) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, # Stamp the resolved target, mirroring _import_python (#1814): # without it, an include whose header lives outside this # batch's paths keeps the raw absolute-path id no later pass # ever learns to relativize (#2243). "target_file": str(resolved), }) break module_name = raw.split("/")[-1].split(".")[0] if module_name: tgt_nid = _make_id(module_name) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) break def _import_csharp(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: text = _read_text(node, source).strip().rstrip(";") if text.startswith("global "): text = text[len("global "):].strip() if not text.startswith("using"): return body = text[len("using"):].strip() using_kind, alias, target_fqn = "namespace", None, body if body.startswith("static "): using_kind, target_fqn = "static", body[len("static "):].strip() elif "=" in body: lhs, rhs = body.split("=", 1) using_kind, alias, target_fqn = "alias", lhs.strip(), rhs.strip() if not target_fqn: return edges.append({ "source": file_nid, "target": _make_id(target_fqn), "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, "metadata": sanitize_metadata({k: v for k, v in {"using_kind": using_kind, "alias": alias, "target_fqn": target_fqn, "scope_kind": "namespace" if scope_stack else "file", "scope_id": scope_stack[-1] if scope_stack else None}.items() if v is not None}), }) def _import_kotlin(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: # Grammar 1.1.0 (PyPI tree_sitter_kotlin) emits an `import` node whose # children are the `import` keyword and a `qualified_identifier` (the dotted # path), optionally followed by `.` `*` (wildcard) or `as` + `identifier` # (alias). There is no `path` field. Older forks emit `import_header` with a # `path` field or a bare `identifier` child; keep those branches so the # extractor works across grammar generations (#2526, adapted from PR #2531 # by @Mustaqeem66). path_node = node.child_by_field_name("path") if path_node is None: path_node = next( (c for c in node.children if c.type == "qualified_identifier"), None ) if path_node is not None: raw = _read_text(path_node, source).strip() else: raw = next( (_read_text(c, source).strip() for c in node.children if c.type == "identifier"), "", ) if not raw: return # Wildcard (`import a.b.*`): imports a whole package, not a symbol. The last # path segment is a PACKAGE name, so a symbol-level edge would dangle on (or # collide with) an unrelated node that happens to share the package's name. if raw.endswith(".*") or raw == "*" or any(c.type == "*" for c in node.children): return # Alias (`import a.b.C as D`): the alias is the identifier child after `as`. alias = None saw_as = False for child in node.children: if not saw_as: saw_as = child.type == "as" elif child.type in ("identifier", "simple_identifier"): alias = _read_text(child, source).strip() or None break module_name = raw.split(".")[-1].strip() if not module_name: return # Target is the bare last segment for now; _resolve_kotlin_import_targets # rewrites it to the real node id via the target_fqn stamped here, once the # per-file package index exists. Unresolved targets stay dangling like other # languages' external imports. edges.append({ "source": file_nid, "target": _make_id(module_name), "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, "metadata": sanitize_metadata({k: v for k, v in {"target_fqn": raw, "alias": alias}.items() if v is not None}), }) def _import_scala(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: for child in node.children: if child.type in ("stable_id", "identifier"): raw = _read_text(child, source) module_name = raw.split(".")[-1].strip("{} ") if module_name and module_name != "_": tgt_nid = _make_id(module_name) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) break def _import_php(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: # `node` is a namespace_use_clause: `Client` or `Client as HttpClient`, # whose children are the qualified/bare name, then (if present) `as` # and the alias name, in that order. An aliased clause used to always # target the bare imported name ("Client"), ignoring the alias # entirely -- so two files importing the SAME external class under # DIFFERENT local aliases (a real pattern: disambiguating two # same-named classes from different namespaces) produced two # DIFFERENT stub targets for one class, and _resolve_php_type_references # (which resolves a stub via the file's own alias -> FQN map, keyed by # the alias when the import has one) could never find this one under # its bare-name-derived key, leaving the edge stuck on an unresolved, # never-repointed stub (#3421). The alias, when present, is also what # the rest of the file actually references (a parameter typed # `HttpClient`, not `Client`), so preferring it here keeps this edge's # target consistent with those reference edges too. saw_as = False module_name = None for child in node.children: if child.type == "as": saw_as = True continue if child.type in ("qualified_name", "name", "identifier"): bare = _read_text(child, source).split("\\")[-1].strip() if saw_as: module_name = bare break # the alias name always comes last; nothing more to see if module_name is None: module_name = bare if module_name: tgt_nid = _make_id(module_name) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) # ── C/C++ function name helpers ─────────────────────────────────────────────── def _get_c_func_name(node, source: bytes) -> str | None: """Recursively unwrap declarator to find the innermost identifier (C).""" if node.type == "identifier": return _read_text(node, source) decl = node.child_by_field_name("declarator") if decl: return _get_c_func_name(decl, source) for child in node.children: if child.type == "identifier": return _read_text(child, source) return None # ── JS/TS extra walk for arrow functions ────────────────────────────────────── # Node types whose value is a callable, for the JS/TS assignment / class-field # / function-expression forms below. Older tree-sitter-javascript grammars # label a function expression `function`; current ones use `function_expression`. # ── TS extra walk for namespace / module declarations ───────────────────────── # ── C# extra walk for namespace declarations ────────────────────────────────── # ── Swift extra walk for enum cases ────────────────────────────────────────── # ── Java extra walk for enum constants ─────────────────────────────────────── # ── Language configs ────────────────────────────────────────────────────────── _PYTHON_CONFIG = LanguageConfig( ts_module="tree_sitter_python", class_types=frozenset({"class_definition"}), function_types=frozenset({"function_definition"}), import_types=frozenset({"import_statement", "import_from_statement"}), call_types=frozenset({"call"}), call_function_field="function", call_accessor_node_types=frozenset({"attribute"}), call_accessor_field="attribute", call_accessor_object_field="object", function_boundary_types=frozenset({"function_definition"}), import_handler=_import_python, ) _JS_CONFIG = LanguageConfig( ts_module="tree_sitter_javascript", class_types=frozenset({"class_declaration"}), function_types=frozenset({"function_declaration", "generator_function_declaration", "method_definition"}), import_types=frozenset({"import_statement", "export_statement"}), call_types=frozenset({"call_expression", "new_expression", "jsx_opening_element", "jsx_self_closing_element"}), call_function_field="function", call_accessor_node_types=frozenset({"member_expression"}), call_accessor_field="property", call_accessor_object_field="object", # `function_expression` belongs here so UNTRACKED inline/nested expressions # reach walk_calls' existing closure handler, which already names the type # (`_JS_CLOSURE_TYPES`). Without it the gate never opens, so such an # expression's parameters and locals never fold into extra_locals for its # subtree and read as by-name references (#2241 family). A top-level # `const f = function (…) {}` is tracked via its declarator and was already # fine; the inline/nested forms are what this covers. function_boundary_types=frozenset({"function_declaration", "generator_function_declaration", "arrow_function", "method_definition", "function_expression", "generator_function"}), import_handler=_import_js, ) _TS_CONFIG = LanguageConfig( ts_module="tree_sitter_typescript", ts_language_fn="language_typescript", class_types=frozenset({ "class_declaration", "abstract_class_declaration", # TS abstract class "interface_declaration", # parity with Java/C# "enum_declaration", # named enums "type_alias_declaration", # named type aliases }), # `abstract_method_signature`: an abstract class's method is a contract # declaration exactly like an `interface`'s `method_signature` (already # captured) — a subclass must implement it. Without it the abstract method # was dropped, so the base of an `abstract class` sat in the graph with only # its concrete methods and no node for the overridden contract. function_types=frozenset({"function_declaration", "generator_function_declaration", "method_definition", "method_signature", "abstract_method_signature"}), import_types=frozenset({"import_statement", "export_statement"}), call_types=frozenset({"call_expression", "new_expression"}), call_function_field="function", call_accessor_node_types=frozenset({"member_expression"}), call_accessor_field="property", call_accessor_object_field="object", # `function_expression`: see the note on the JS config above. function_boundary_types=frozenset({"function_declaration", "generator_function_declaration", "arrow_function", "method_definition", "function_expression", "generator_function"}), import_handler=_import_js, ) # .tsx files must use the TSX grammar (JSX-aware), not the plain TypeScript grammar. # tree-sitter-typescript ships two languages: language_typescript (for .ts) and # language_tsx (for .tsx). Parsing .tsx with language_typescript silently fails on # JSX expressions, dropping any call_expression nested inside JSX (e.g. {fmtDate(x)}). _TSX_CONFIG = LanguageConfig( ts_module="tree_sitter_typescript", ts_language_fn="language_tsx", class_types=_TS_CONFIG.class_types, function_types=_TS_CONFIG.function_types, import_types=_TS_CONFIG.import_types, # JSX component usage (``) renders Comp; extracted like a call. call_types=_TS_CONFIG.call_types | {"jsx_opening_element", "jsx_self_closing_element"}, call_function_field=_TS_CONFIG.call_function_field, call_accessor_node_types=_TS_CONFIG.call_accessor_node_types, call_accessor_field=_TS_CONFIG.call_accessor_field, call_accessor_object_field=_TS_CONFIG.call_accessor_object_field, function_boundary_types=_TS_CONFIG.function_boundary_types, import_handler=_TS_CONFIG.import_handler, ) _JAVA_CONFIG = LanguageConfig( ts_module="tree_sitter_java", # record_declaration shares class_declaration's name/body/interfaces fields, # so it becomes a first-class type node instead of an isolated file (#1373). # Enums and annotation declarations use the same name/body contract. class_types=frozenset({ "class_declaration", "interface_declaration", "record_declaration", "enum_declaration", "annotation_type_declaration", }), function_types=frozenset({"method_declaration", "constructor_declaration"}), import_types=frozenset({"import_declaration"}), # object_creation_expression (`new Foo(...)`) is handled by a dedicated Java # branch in walk_calls below — its callee is in the `type` field, not `name`. call_types=frozenset({"method_invocation", "object_creation_expression"}), call_function_field="name", call_accessor_node_types=frozenset(), function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}), import_handler=_import_java, ) _GROOVY_CONFIG = LanguageConfig( ts_module="tree_sitter_groovy", # enum_declaration shares the name/body contract, so a Groovy enum becomes a # first-class type node with its constants (via _java_extra_walk) instead of # being dropped along with everything it declares (#Java enum parity). class_types=frozenset({"class_declaration", "interface_declaration", "enum_declaration"}), function_types=frozenset({"method_declaration", "constructor_declaration"}), import_types=frozenset({"import_declaration"}), call_types=frozenset({"method_invocation"}), call_function_field="name", call_accessor_node_types=frozenset(), function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}), import_handler=_import_java, ) _C_CONFIG = LanguageConfig( ts_module="tree_sitter_c", class_types=frozenset(), function_types=frozenset({"function_definition"}), import_types=frozenset({"preproc_include"}), call_types=frozenset({"call_expression"}), call_function_field="function", call_accessor_node_types=frozenset({"field_expression"}), call_accessor_field="field", function_boundary_types=frozenset({"function_definition"}), import_handler=_import_c, resolve_function_name_fn=_get_c_func_name, ) _CPP_CONFIG = LanguageConfig( ts_module="tree_sitter_cpp", # enum_specifier is a class-like container: an `enum` / `enum class` owns a # set of enumerators. Its name and body sit on the same `name`/`body` fields # as struct_specifier (type_identifier + enumerator_list), so it gets a node # and a body walk; the enumerators are emitted by _cpp_extra_walk (the C++ # parity of Java #1719 / Swift / Scala enums). union_specifier is likewise a # class-like container with the identical name/body fields (type_identifier + # field_declaration_list), so a `union { ... }` becomes a type node with its # data members instead of being dropped along with everything it declares. class_types=frozenset({ "class_specifier", "struct_specifier", "enum_specifier", "union_specifier", }), function_types=frozenset({"function_definition"}), import_types=frozenset({"preproc_include"}), call_types=frozenset({"call_expression"}), call_function_field="function", call_accessor_node_types=frozenset({"field_expression", "qualified_identifier"}), call_accessor_field="field", function_boundary_types=frozenset({"function_definition"}), import_handler=_import_c, resolve_function_name_fn=_get_cpp_func_name, ) def _ruby_sanitize_method_name(name: str) -> str: """Encode trailing Ruby method suffixes (!, ?, =) into safe node ID components (#3077).""" if not name: return name if name.endswith("!"): return f"{name[:-1]}_bang" if name.endswith("?"): return f"{name[:-1]}_pred" if name.endswith("="): return f"{name[:-1]}_eq" return name _RUBY_CONFIG = LanguageConfig( ts_module="tree_sitter_ruby", # `module Foo` is a container node just like `class Foo` in tree-sitter's # Ruby grammar (name in a `constant` child, body in `body_statement`), so it # gets a node and its methods attach via `method` (#1640). Without it, plain # utility/`module_function` modules produced no node and their methods hung # off the file via `contains` with dot-less labels. class_types=frozenset({"class", "module"}), function_types=frozenset({"method", "singleton_method"}), import_types=frozenset(), call_types=frozenset({"call"}), call_function_field="method", call_accessor_node_types=frozenset(), name_fallback_child_types=("constant", "scope_resolution", "identifier"), body_fallback_child_types=("body_statement",), function_boundary_types=frozenset({"method", "singleton_method"}), sanitize_symbol_name_fn=_ruby_sanitize_method_name, ) _CSHARP_CONFIG = LanguageConfig( ts_module="tree_sitter_c_sharp", class_types=frozenset({ "class_declaration", "interface_declaration", "enum_declaration", "struct_declaration", "record_declaration", }), function_types=frozenset({"method_declaration"}), import_types=frozenset({"using_directive"}), # `object_creation_expression` joins the invocation node so `new Foo(...)` # links the constructing method to Foo, the way Java has since #1373. call_types=frozenset({"invocation_expression", "object_creation_expression"}), call_function_field="function", call_accessor_node_types=frozenset({"member_access_expression"}), call_accessor_field="name", body_fallback_child_types=("declaration_list",), function_boundary_types=frozenset({"method_declaration"}), import_handler=_import_csharp, ) _KOTLIN_CONFIG = LanguageConfig( ts_module="tree_sitter_kotlin", class_types=frozenset({"class_declaration", "object_declaration"}), function_types=frozenset({"function_declaration"}), # Grammar 1.1.0 (PyPI tree_sitter_kotlin) names the import node `import`; # older forks use `import_header`. Accept both (#2526). import_types=frozenset({"import_header", "import"}), call_types=frozenset({"call_expression"}), call_function_field="", call_accessor_node_types=frozenset({"navigation_expression"}), call_accessor_field="", # Different tree-sitter-kotlin grammar versions name plain identifier # nodes differently: PyPI's `tree_sitter_kotlin` uses `identifier`, # older forks use `simple_identifier`. Accept both so the extractor # works across grammar generations. name_fallback_child_types=("simple_identifier", "identifier"), body_fallback_child_types=("function_body", "class_body", "enum_class_body"), function_boundary_types=frozenset({"function_declaration"}), import_handler=_import_kotlin, ) _SCALA_CONFIG = LanguageConfig( ts_module="tree_sitter_scala", # traits are class-like containers with their own heritage (extends / with), # so they need a node and the heritage walk just like classes and objects. # Scala 3 `enum` is a class-like container too: it owns methods and a set of # cases, so it needs a node and a body walk like the others (its cases are # emitted by _scala_extra_walk, the parity of Java #1719 / Kotlin #1738). # A Scala 3 `given ... with` instance is a class-like container: its `def`s # belong to the given instance, not to file scope (where they leak and collide # with same-named trait methods, #4130). Its body is a `with_template_body`. class_types=frozenset({"class_definition", "object_definition", "trait_definition", "enum_definition", "given_definition"}), # `function_declaration` is a bodyless `def area: Double` — a deferred # (abstract) method. In a `trait` or `abstract class` it is the contract a # subclass must implement, exactly like Java/C#/TS abstract methods. Only # `function_definition` (a `def` WITH a body) was a function type, so every # abstract member was dropped: a pure-interface trait had no method nodes at # all, and a concrete method calling a deferred one had no target to link to. function_types=frozenset({"function_definition", "function_declaration"}), import_types=frozenset({"import_declaration"}), call_types=frozenset({"call_expression"}), call_function_field="", call_accessor_node_types=frozenset({"field_expression"}), call_accessor_field="field", name_fallback_child_types=("identifier",), # an enum wraps its members in `enum_body` rather than a `template_body`, # so the body walk needs it to reach the enum's methods and cases; a # `given ... with` instance wraps its members in a `with_template_body`. body_fallback_child_types=("template_body", "enum_body", "with_template_body"), function_boundary_types=frozenset({"function_definition"}), import_handler=_import_scala, ) _PHP_CONFIG = LanguageConfig( ts_module="tree_sitter_php", ts_language_fn="language_php", # interfaces/enums/traits are class-like containers whose heritage # (extends/implements) and members must be captured, same as classes. class_types=frozenset({ "class_declaration", "interface_declaration", "enum_declaration", "trait_declaration", }), function_types=frozenset({"function_definition", "method_declaration", "anonymous_function", "arrow_function"}), import_types=frozenset({"namespace_use_clause"}), # object_creation_expression joins the dispatch set so `new Foo(...)` links # the constructing method to Foo (engine has a dedicated PHP branch: the # class sits in a bare name/qualified_name child, not a field) - the PHP # twin of Java #1373 / C# #2997 (#3115). call_types=frozenset({"function_call_expression", "member_call_expression", "scoped_call_expression", "class_constant_access_expression", "object_creation_expression"}), static_prop_types=frozenset({"scoped_property_access_expression"}), helper_fn_names=frozenset({"config"}), container_bind_methods=frozenset({"bind", "singleton", "scoped", "instance"}), event_listener_properties=frozenset({"listen", "subscribe"}), call_function_field="function", call_accessor_node_types=frozenset({"member_call_expression"}), call_accessor_field="name", name_fallback_child_types=("name",), # enums wrap their members in an enum_declaration_list rather than a # declaration_list, so the body walk needs it to reach enum methods/cases. body_fallback_child_types=("declaration_list", "compound_statement", "enum_declaration_list"), function_boundary_types=frozenset({"function_definition", "method_declaration", "anonymous_function", "arrow_function"}), import_handler=_import_php, ) def _import_lua(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: """Extract require('module') from Lua variable_declaration nodes.""" text = _read_text(node, source) import re m = re.search(r"""require\s*[\('"]\s*['"]?([^'")\s]+)""", text) if m: raw_module = m.group(1) if raw_module: tgt_nid = _resolve_lua_import_target(raw_module, str_path) if tgt_nid: edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "confidence_score": 1.0, "source_file": str_path, "source_location": str(node.start_point[0] + 1), "weight": 1.0, }) _LUA_CONFIG = LanguageConfig( ts_module="tree_sitter_lua", ts_language_fn="language", class_types=frozenset(), function_types=frozenset({"function_declaration"}), import_types=frozenset({"variable_declaration"}), call_types=frozenset({"function_call"}), call_function_field="name", call_accessor_node_types=frozenset({"method_index_expression"}), # A colon call `obj:method()` parses as a method_index_expression whose callee # lives in the `method` field and whose receiver lives in the `table` field — # not the `name`/`object` fields the dot-access languages use (#3991). call_accessor_field="method", call_accessor_object_field="table", name_fallback_child_types=("identifier", "method_index_expression"), body_fallback_child_types=("block",), function_boundary_types=frozenset({"function_declaration"}), import_handler=_import_lua, ) def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> list[tuple[str, str]]: """Emit module-level ``imports`` edges and report the imported modules. A Swift ``import CoreKit`` names a module, not a file path, so — unlike the file-resolving JS/TS handlers — there is no existing node for the edge to point at. The returned ``(id, label)`` pairs let the extractor materialize a ``type=module`` anchor node so the edge survives; without it ``build_from_json`` prunes every Swift import edge as a dangling/external reference (#1327). """ modules: list[tuple[str, str]] = [] for child in node.children: if child.type == "identifier": raw = _read_text(child, source) tgt_nid = _make_id(raw) edges.append({ "source": file_nid, "target": tgt_nid, "relation": "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) modules.append((tgt_nid, raw)) break return modules _SWIFT_CONFIG = LanguageConfig( ts_module="tree_sitter_swift", class_types=frozenset({"class_declaration", "protocol_declaration"}), # `protocol_function_declaration` is the body-less method requirement inside a # `protocol { ... }`; tree-sitter-swift gives it its own node type rather than # reusing `function_declaration`, so without it a protocol's method contract # is dropped and the protocol becomes an empty node. function_types=frozenset({"function_declaration", "protocol_function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}), import_types=frozenset({"import_declaration"}), call_types=frozenset({"call_expression"}), call_function_field="", call_accessor_node_types=frozenset({"navigation_expression"}), call_accessor_field="", name_fallback_child_types=("simple_identifier", "type_identifier", "user_type"), body_fallback_child_types=("class_body", "protocol_body", "function_body", "enum_class_body"), function_boundary_types=frozenset({"function_declaration", "protocol_function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}), import_handler=_import_swift, ) # ── Ruby local type inference (for member-call resolution) ───────────────────── # `Const = (...)` shapes that define a lightweight class named after the # constant. tree-sitter parses each as an `assignment`, not a `class`, so the # generic class branch never saw them (#1640). # ── Generic extractor ───────────────────────────────────────────────────────── # ── Python rationale extraction ─────────────────────────────────────────────── _RATIONALE_PREFIXES = ("# NOTE:", "# IMPORTANT:", "# HACK:", "# WHY:", "# RATIONALE:", "# TODO:", "# FIXME:") def _shorten_rationale_label(text: str, width: int = 80) -> str: """Collapse whitespace and truncate ``text`` to ``width`` chars for a rationale node label, cutting on a word boundary rather than mid-word. Shared by the Python and JS/TS rationale extractors (#2206). ``textwrap.shorten`` collapses to just the placeholder when the first "word" alone exceeds ``width`` (e.g. a docstring/comment that opens with an unbroken URL) -- that would emit a content-free label, so fall back to a plain character truncation of the normalized text in that case. """ label = textwrap.shorten(text, width=width, placeholder="…") if label in ("", "…"): flat = " ".join(text.split()) label = flat if len(flat) <= width else flat[: width - 1] + "…" return label def _is_autogenerated_python(source: bytes) -> bool: """Return True if this Python file is auto-generated and its module docstring is noise. Covers: Alembic/Flask-Migrate revisions, Django migrations, protobuf/gRPC/OpenAPI stubs. Module docstrings in these files are change annotations or boilerplate, not rationale. """ head = source[:2048].decode("utf-8", errors="replace") # Generic generated-file markers (protobuf, gRPC, OpenAPI codegen, etc.) if any(m in head for m in ("DO NOT EDIT", "@generated", "Generated by the protocol buffer")): return True # Alembic / Flask-Migrate revision files if (re.search(r"^revision\s*[:=]", head, re.MULTILINE) and "def upgrade(" in head and "down_revision" in head): return True # Django migrations if "class Migration(migrations.Migration)" in head and "operations" in head: return True return False def _extract_python_rationale(path: Path, result: dict) -> None: """Post-pass: extract docstrings and rationale comments from Python source. Mutates result in-place by appending to result['nodes'] and result['edges']. """ try: import tree_sitter_python as tspython from tree_sitter import Language, Parser language = Language(tspython.language()) parser = Parser(language) source = _read_source_bytes(path, warn=False) tree = parser.parse(source) root = tree.root_node except Exception: return stem = _file_stem(path) str_path = str(path) nodes = result["nodes"] edges = result["edges"] seen_ids = {n["id"] for n in nodes} file_nid = _make_id(str(path)) def _get_docstring(body_node) -> tuple[str, int] | None: """A docstring is the first STATEMENT in a module/class/function body. A leading `comment` node — the shebang line essentially every executable script starts with, a coding-declaration or license header, or any ordinary comment — is not a statement: tree-sitter still parses it as a sibling child of the body, but Python's own docstring rule skips right over it. The old unconditional `break` after the first loop iteration stopped at that comment instead of looking past it, so a module docstring behind a shebang (or any leading comment) was silently never found (#3312). """ if not body_node: return None for child in body_node.children: if child.type == "comment": continue if child.type == "expression_statement": for sub in child.children: if sub.type in ("string", "concatenated_string"): text = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace") text = text.strip("\"'").strip('"""').strip("'''").strip() if len(text) > 20: return text, child.start_point[0] + 1 break return None def _add_rationale(text: str, line: int, parent_nid: str) -> None: # Normalize whitespace before truncating, not after: slicing raw text # first can land mid-word, leave a run of literal spaces where a # newline + indentation used to be, or end on a "." that turns into # an Obsidian "..md" filename once export.py appends the extension. label = _shorten_rationale_label(text) rid = _make_id(stem, "rationale", str(line)) if rid not in seen_ids: seen_ids.add(rid) nodes.append({ "id": rid, "label": label, "file_type": "rationale", "source_file": str_path, "source_location": f"L{line}", }) edges.append({ "source": rid, "target": parent_nid, "relation": "rationale_for", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, }) # Module-level docstring — skip for auto-generated files (Alembic, Django # migrations, protobuf stubs, etc.) whose module docstrings are revision # annotations, not architectural rationale. if not _is_autogenerated_python(source): ds = _get_docstring(root) if ds: _add_rationale(ds[0], ds[1], file_nid) # Class and function docstrings def walk_docstrings(node, parent_nid: str) -> None: t = node.type if t == "class_definition": name_node = node.child_by_field_name("name") body = node.child_by_field_name("body") if name_node and body: class_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace") nid = _make_id(parent_nid, class_name) if parent_nid != file_nid else _make_id(stem, class_name) ds = _get_docstring(body) if ds: _add_rationale(ds[0], ds[1], nid) for child in body.children: walk_docstrings(child, nid) return if t == "function_definition": name_node = node.child_by_field_name("name") body = node.child_by_field_name("body") if name_node and body: func_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace") nid = _make_id(parent_nid, func_name) if parent_nid != file_nid else _make_id(stem, func_name) ds = _get_docstring(body) if ds: _add_rationale(ds[0], ds[1], nid) return for child in node.children: walk_docstrings(child, parent_nid) walk_docstrings(root, file_nid) # Rationale comments (# NOTE:, # IMPORTANT:, etc.) source_text = source.decode("utf-8", errors="replace") for lineno, line_text in enumerate(source_text.splitlines(), start=1): stripped = line_text.strip() if any(stripped.startswith(p) for p in _RATIONALE_PREFIXES): _add_rationale(stripped, lineno, file_nid) # ── TypeScript import type normalization (#3154) ────────────────────────────── # tree-sitter-typescript misparses `import(...)` types used inside explicit call- # expression type arguments (e.g. `f()` or # `f()`) as binary comparison expressions (`<` and `>`). # The trailing `();` generates an ERROR node, leaving an open `binary_expression` # that absorbs subsequent declarations as anonymous `function_expression` or `class` # expressions, silently dropping them from extraction. Normalizing `import(...)` # within generic call type arguments `<...>(...)` to a valid type identifier of # identical byte length keeps AST parsing clean while preserving source offsets. _TS_IMPORT_CALL_RE = re.compile( rb"\bimport\s*\(\s*['\"][^'\"\r\n]+['\"]\s*\)" ) def _ts_import_is_code(root: Any, start: int) -> bool: """Return whether an import-call match starts in executable source. Regex matching is only used to locate a literal specifier; comments, strings, regular expressions, and template text must never be fed into the structural masking pass. A template substitution is executable again, so it is the one exception to the ``template_string`` guard. """ node = root.descendant_for_byte_range(start, start + 1) if node is None: # A malformed tree can leave a byte range uncovered. Treat it as code # and let the structural pass decide whether it belongs to a type list; # never turn an uncertain lexical result into a crash. return True in_template_substitution = False while node is not None: if node.type == "comment" or node.type in ("string", "regex", "regex_pattern"): return False if node.type == "template_substitution": in_template_substitution = True elif node.type == "template_string" and not in_template_substitution: return False node = node.parent return True def _ts_type_argument_ranges(root: Any, *, call_only: bool) -> list[tuple[int, int]]: """Collect byte ranges tree-sitter already parsed as type arguments.""" ranges: list[tuple[int, int]] = [] stack = [root] while stack: node = stack.pop() if node.type == "type_arguments": parent = node.parent if not call_only or ( parent is not None and parent.type in ("call_expression", "new_expression") ): ranges.append((node.start_byte, node.end_byte)) stack.extend(node.children) return ranges # Sorted type-argument ranges prepared for lookup: (starts, spans, prefix_max_end). # See _ts_build_range_index. A real alias, not a string: quoting the right-hand # side would make this a `str` value, which type checkers reject as a type. _TsRangeIndex = tuple[list[int], list[tuple[int, int]], list[int]] def _ts_ranges_containing( ranges: _TsRangeIndex, offset: int ) -> list[tuple[int, int]]: """Return the ranges in ``ranges`` that contain ``offset``. Scanning the full range list for every match made :func:`_normalize_ts_import_types` O(matches x ranges); on a large monorepo file with thousands of generic calls and thousands of ``import(...)`` types that quadratic blowup pinned a worker at 100% CPU with no output and no file I/O, so extraction never finished (#3359). Every range containing ``offset`` must start at or before it, so binary-search to the last such start and walk left. A range that ends before ``offset`` is not itself a hit but must not stop the walk -- a closed sibling can sit inside an enclosing range that does contain ``offset``. The index's running maximum of ends (``prefix_max_end``) gives the correct stop: once no range at or before this position reaches past ``offset``, none of the remaining ones can. """ starts, spans, prefix_max_end = ranges hits: list[tuple[int, int]] = [] index = bisect_right(starts, offset) - 1 while index >= 0 and prefix_max_end[index] > offset: start, end = spans[index] if end > offset: hits.append((start, end)) index -= 1 hits.reverse() return hits def _ts_build_range_index(root: Any, *, call_only: bool) -> _TsRangeIndex: """Index :func:`_ts_type_argument_ranges` for :func:`_ts_ranges_containing`. Returns the ranges sorted by start, split into a bare ``starts`` list for :func:`bisect_right`, plus a prefix-maximum of ``end`` values so a lookup can tell when no earlier range can still reach a given offset. """ spans = sorted(_ts_type_argument_ranges(root, call_only=call_only)) starts = [start for start, _ in spans] prefix_max_end: list[int] = [] running = -1 for _, end in spans: running = max(running, end) prefix_max_end.append(running) return starts, spans, prefix_max_end def _ts_error_nodes(root: Any) -> list[Any]: """Return parser error nodes without depending on a grammar's error name.""" errors: list[Any] = [] stack = [root] while stack: node = stack.pop() if node.type == "ERROR" or node.is_error: errors.append(node) stack.extend(node.children) return errors def _ts_mask_candidate_is_malformed( original_root: Any, masked_range: tuple[int, int], errors: list[Any], ) -> bool: """Tell whether a masked generic is backed by an actual parse failure. A valid runtime comparison can have the same token shape as a generic call after the import is replaced (``a < import("x") > (a)``). Tree-sitter exposes the opening ``<`` as a binary operator in the original tree for both forms, so the structural second pass alone cannot distinguish them. Only repair that ambiguity when the original parser has an error adjacent to the candidate's closing angle; that is the signature of the known ``import(...)``-type grammar failure. Valid comparisons, including ones nested in another call's arguments, remain byte-for-byte untouched. """ start, end = masked_range opener = original_root.descendant_for_byte_range(start, start + 1) if opener is None or opener.type != "<": # A grammar that already gives us a structural type_arguments node is # safe to normalize; no binary/comparison ambiguity is present. return True if opener.parent is None or opener.parent.type != "binary_expression": return True # A malformed generic's ERROR starts at the closing `>` (or at the call # punctuation immediately following it). Keep this window deliberately # narrow so an unrelated syntax error elsewhere cannot authorize masking a # valid runtime comparison. for error in errors: if error.start_byte <= end + 2 and error.end_byte >= end - 1: return True return False _TS_EXPORT_TYPE_STAR_RE = re.compile(rb"\bexport\s+(type)\s*\*") def _normalize_ts_export_type_star(source: bytes) -> bytes | None: """Rewrite ``export type * from "..."`` to a plain ``export * from "..."``. TS 5.0's type-only star re-export has no equivalent in tree-sitter-typescript 0.23 (the latest release as of #3942): the grammar raises an ERROR on the ``type`` keyword in this position and the whole file is dropped, losing every symbol, not just the one statement. The construct is erased at compile time, the same as ``import type`` / ``export type { X }``, so the graph only needs the re-export edge, not the type-only distinction — blank out ``type`` (keeping every other byte and offset identical) so the grammar parses the ordinary, already-supported ``export * from "..."`` form instead. Unlike ``import(...)``, this prefix is never ambiguous with a runtime expression, so no AST-based disambiguation pass is needed here. """ matches = list(_TS_EXPORT_TYPE_STAR_RE.finditer(source)) if not matches: return None masked = bytearray(source) for match in matches: start, end = match.span(1) masked[start:end] = b" " * (end - start) return bytes(masked) def _normalize_ts_import_types(source: bytes, *, tsx: bool = False) -> bytes | None: """Rewrite only syntactic TypeScript ``import(...)`` type arguments. The first implementation of #3154 used a ``<...>`` regular expression. In semicolon-less code that expression could span two comparison operators, so a *runtime* dynamic import was blanked before parsing. A temporary, byte-preserving mask lets tree-sitter identify the actual ``type_arguments`` node without asking it to parse the known-invalid ``import(...)`` call-site form. We then rewrite only placeholders inside call/new-expression type arguments. Keeping every replacement the same byte length preserves source offsets and line locations. """ raw_matches = list(_TS_IMPORT_CALL_RE.finditer(source)) if not raw_matches: return None # tree-sitter is already a required TypeScript dependency for extraction. # If it cannot be loaded here, leave the source untouched; the normal AST # path will report its own dependency/parser error rather than applying an # unsafe textual guess. try: import tree_sitter_typescript as ts_typescript from tree_sitter import Language, Parser language_factory = ( ts_typescript.language_tsx if tsx else ts_typescript.language_typescript ) parser = Parser(Language(language_factory())) original_root = parser.parse(source).root_node except Exception: return None matches = [ match for match in raw_matches if _ts_import_is_code(original_root, match.start()) ] if not matches: return None # If the original tree already has a type_arguments node around a match, # its syntax is parseable as written. This includes the grammar's deliberate # comparison-vs-generic ambiguity (`a < b, import("...") > (d)`); retaining # that source is essential because it is a runtime expression, not a type. original_type_ranges = _ts_build_range_index(original_root, call_only=False) matches = [ match for match in matches if not _ts_ranges_containing(original_type_ranges, match.start()) ] if not matches: return None def placeholder(match: "re.Match[bytes]") -> bytes: # Keep CR/LF bytes intact. The first byte becomes an ordinary type # identifier and the remaining bytes are padding, so every downstream # byte offset remains identical to the user's source. matched = match.group(0) return b"T" + re.sub(rb"[^\r\n]", b" ", matched[1:]) masked = bytearray(source) for match in matches: masked[match.start():match.end()] = placeholder(match) root = parser.parse(bytes(masked)).root_node # Only call/new-expression type arguments need this workaround. Ordinary # type annotations already parse ``import(...)`` correctly and should keep # their native AST shape. A range from an outer call's type_arguments also # covers nested generic arguments. type_argument_ranges = _ts_build_range_index(root, call_only=True) errors = _ts_error_nodes(original_root) # `type_argument_ranges` is an index tuple, so test the spans it holds -- # the tuple itself is always truthy. if not type_argument_ranges[1]: return None norm = bytearray(source) changed = False for match in matches: containing_ranges = _ts_ranges_containing(type_argument_ranges, match.start()) if containing_ranges and any( _ts_mask_candidate_is_malformed(original_root, candidate, errors) for candidate in containing_ranges ): norm[match.start():match.end()] = placeholder(match) changed = True return bytes(norm) if changed else None # ── Public API ──────────────────────────────────────────────────────────────── def extract_python(path: Path, *, root: Path | None = None) -> dict: """Extract classes, functions, and imports from a .py file via tree-sitter AST.""" result = _extract_generic(path, _PYTHON_CONFIG, scan_root=root) if "error" not in result: _extract_python_rationale(path, result) return result def extract_js(path: Path) -> dict: """Extract classes, functions, arrow functions, and imports from a .js/.ts/.tsx/.mts/.cts file.""" suffix = path.suffix.lower() is_ts = suffix in (".ts", ".tsx", ".mts", ".cts") if suffix == ".tsx": config = _TSX_CONFIG elif suffix in (".ts", ".mts", ".cts"): config = _TS_CONFIG else: config = _JS_CONFIG source_override = None if is_ts: try: source = _read_source_bytes(path) working = _normalize_ts_export_type_star(source) or source normalized = _normalize_ts_import_types(working, tsx=suffix == ".tsx") source_override = normalized if normalized is not None else ( working if working is not source else None ) except OSError: pass result = _extract_generic(path, config, source_override=source_override) if "error" not in result: _extract_js_rationale(path, result) _rescue_js_dynamic_imports(path, result) return result def _rescue_js_dynamic_imports(path: Path, result: dict) -> None: """Recover ``import('…')`` edges the AST pass does not emit for plain JS/TS. tree-sitter models ``await import('x')`` as a ``call_expression``, not an ``import_statement``, so the specifier only reaches the graph when ``walk_calls`` visits that call — which it never does at module scope (only function bodies are walked for calls). The Svelte/Astro/Vue extractors already patch the same gap by regex because their AST pass fails wholesale; plain ``.ts``/``.js`` was left out on the reasoning that its AST pass "works". It works for STATIC imports; dynamic ones outside a walked body fell through silently (#2575), and because they cluster under hub modules the loss compounds with ``affected`` traversal depth. Dedupe: a dynamic import the AST pass DID capture is already in the graph as an ``imports_from`` edge marked ``deferred`` (``_dynamic_import_js``). Re-emitting it here as a second ``dynamic_import`` edge would state the same fact twice, so a match whose resolved target already has a deferred edge FROM THIS FILE'S NODE is skipped. The source check matters: the AST pass anchors the edge on the enclosing function when the ``import()`` is written inside one, and that is a different fact from "this file depends on that module" — the only one file-level traversal can use (#2584). Regex false positives in comments/strings are the precedented trade of the Svelte/Vue rescues; a ``//``-prefix guard covers the common case. """ try: import re as _re src = _read_source_text(path, warn=False) if not _re.search(r"(? target`, and `caller_nid` is this # file's node only when the `import()` sits at module scope. Written inside a # function it is that function's node — a different fact, at a granularity # `affected` does not walk. Matching on target alone treated the two as one and # skipped the rescue, so a dynamic import inside a function ended up with no # file-level edge at all. The reverse walk then reached the enclosing function # and stopped: the only edge pointing at it is `contains`, deliberately kept out # of DEFAULT_AFFECTED_RELATIONS. # # Measured on a ~700-file TS repo: `affected --depth 3` returned 39 of 49 truly # affected files (recall 0.80, precision 1.00) and deeper traversal did not help, # which is a dead end rather than a depth limit. It stayed hidden because the # usual case still resolves — when the next importer imports that exact symbol # by name there IS an edge into the function. Switch that importer to # `import * as ns` or a side-effect `import './dyn'` and the same graph goes # silent. if (e.get("deferred") and e.get("relation") == "imports_from" and e.get("source") == file_node_id): deferred_ids.add(e.get("target")) tf = e.get("target_file") if tf: try: deferred_files.add(str(Path(tf).resolve())) except OSError: deferred_files.add(str(tf)) # `(?ADR edges never form even when the code cites the ADR. _JS_RATIONALE_PREFIXES = ( "// NOTE:", "// IMPORTANT:", "// HACK:", "// WHY:", "// RATIONALE:", "// TODO:", "// FIXME:", "* NOTE:", "* IMPORTANT:", "* HACK:", "* WHY:", "* RATIONALE:", "* TODO:", "* FIXME:", ) # Doc-reference tokens worth first-classing as graph nodes. Deliberately # conservative: ADR-NNNN (Architecture Decision Records, any zero padding) # and RFC NNNN / RFC-NNNN. _JS_DOC_REF_RE = re.compile(r"\b(ADR[- ]?\d{1,5}|RFC[- ]?\d{1,5})\b", re.IGNORECASE) # Only look for doc references inside comments, not string literals or code. _JS_COMMENT_LINE_RE = re.compile(r"^\s*(//|/\*|\*)") def _extract_js_rationale(path: Path, result: dict) -> None: """Post-pass: extract rationale comments and doc references from JS/TS source. Mutates result in-place by appending to result['nodes'] and result['edges']. """ try: source_text = _read_source_text(path, warn=False) except Exception: return stem = _file_stem(path) str_path = str(path) nodes = result["nodes"] edges = result["edges"] seen_ids = {n["id"] for n in nodes} file_nid = _make_id(str(path)) seen_doc_refs: set[str] = set() def _add_rationale(text: str, line: int) -> None: # Normalize whitespace before truncating, not after: slicing raw text # first can land mid-word, leave a run of literal spaces where a # newline + indentation used to be, or end on a "." that turns into # an Obsidian "..md" filename once export.py appends the extension. label = _shorten_rationale_label(text) rid = _make_id(stem, "rationale", str(line)) if rid not in seen_ids: seen_ids.add(rid) nodes.append({ "id": rid, "label": label, "file_type": "rationale", "source_file": str_path, "source_location": f"L{line}", }) edges.append({ "source": rid, "target": file_nid, "relation": "rationale_for", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, }) def _add_doc_ref(token: str, line: int) -> None: # Normalize "adr 11" / "ADR-0011" spellings to a canonical "ADR-0011" # style label so references to the same document collapse to one node. kind, num = re.match(r"([A-Za-z]+)[- ]?(\d+)", token).groups() kind = kind.upper() label = f"{kind}-{num.zfill(4)}" if kind == "ADR" else f"{kind}-{num}" if label in seen_doc_refs: return seen_doc_refs.add(label) rid = _make_id("docref", label) if rid not in seen_ids: seen_ids.add(rid) nodes.append({ "id": rid, "label": label, "file_type": "doc_ref", "source_file": str_path, "source_location": f"L{line}", }) edges.append({ "source": file_nid, "target": rid, "relation": "cites", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, }) for lineno, line_text in enumerate(source_text.splitlines(), start=1): stripped = line_text.strip() if any(stripped.startswith(p) for p in _JS_RATIONALE_PREFIXES): _add_rationale(stripped.lstrip("/* "), lineno) if _JS_COMMENT_LINE_RE.match(line_text): for m in _JS_DOC_REF_RE.finditer(stripped): _add_doc_ref(m.group(1), lineno) def _resolve_rescued_specifier( path: Path, raw: str, aliases, base_url, ) -> "tuple[str, str, Path | None] | None": """Resolve a regex-rescued import specifier the way ``_import_js`` does. Returns ``(node_id, stub_source_file, resolved_file)`` — ``resolved_file`` is the target as a real on-disk file, or None when the specifier is external or dangling. Returns None when no target can be minted at all (empty bare-import segment). Split out of :func:`_emit_rescued_import` so :func:`_rescue_js_dynamic_imports` can resolve a match FIRST and skip specifiers the AST pass already emitted, without duplicating the resolution rules. """ if raw.startswith("."): resolved = _resolve_js_module_path( Path(os.path.normpath(path.parent / raw)) ) resolved_file = resolved if resolved is not None and resolved.is_file() else None return _make_id(str(resolved)), str(resolved), resolved_file # Check tsconfig.json path aliases (e.g. "$lib/" -> "src/lib/", # "@/" -> "src/") before treating as external. Mirrors _import_js # logic so alias imports resolve to the same file node IDs the # extractor creates (#701). resolved_alias = _resolve_tsconfig_alias(raw, aliases, base_url=base_url) if resolved_alias is not None: resolved_alias = _resolve_js_module_path(resolved_alias) resolved_file = (resolved_alias if resolved_alias is not None and resolved_alias.is_file() else None) return _make_id(str(resolved_alias)), str(resolved_alias), resolved_file # Unmapped `@/` project-root convention fallback (#3357). if raw.startswith("@/"): resolved_unmapped = _resolve_js_module_path(raw, path.parent) if resolved_unmapped is not None and resolved_unmapped.is_file(): return _make_id(str(resolved_unmapped)), str(resolved_unmapped), resolved_unmapped # Bare/scoped import (node_modules) - use last segment; # build_from_json drops as external if no matching node exists. module_name = raw.split("/")[-1] if not module_name: return None return _make_id(module_name), raw, None def _emit_rescued_import( result: dict, existing_ids: set, file_node_id: str, path: Path, raw: str, relation: str, aliases, base_url, ) -> None: """Shared edge/stub emit for the Svelte/Astro/Vue regex-rescue import passes. Resolves the specifier the same way ``_import_js`` does — relative paths and tsconfig aliases both go through :func:`_resolve_js_module_path` so extensionless specifiers probe real on-disk extensions (``../lib/content`` -> ``content.ts``) instead of a naive ``.js``->``.ts`` suffix swap. When the resolved target is a real file on disk, mirror ``_import_js``: emit ONLY the edge, stamped with ``target_file``, and mint no stub node. The #2169 canonicalization loop in :func:`extract` reads the stamp and repoints the edge at the real file node's canonical id. Minting a stub here would carry an absolute-path-derived id when the input path is absolute — a ghost node (e.g. ``private_tmp_..._src_lib_content``) duplicating the real ``src_lib_content`` node and clobbering its label on dedupe (#2195). Stub nodes are still minted for unresolved specifiers (externals, not-yet-created files) so prior behavior is preserved. """ resolution = _resolve_rescued_specifier(path, raw, aliases, base_url) if resolution is None: return node_id, stub_source_file, resolved_file = resolution edge = { "source": file_node_id, "target": node_id, "relation": relation, "confidence": "EXTRACTED", "source_file": str(path), } if resolved_file is not None: # Real file on disk: edge only (no stub node), stamped so the #2169 # canonicalization pass repoints it at the real node (#2195). edge["target_file"] = str(resolved_file) result.setdefault("edges", []).append(edge) return if node_id in existing_ids: # Edge target already a real node - just add the edge, don't add a node. result.setdefault("edges", []).append(edge) return result.setdefault("nodes", []).append({ "id": node_id, "label": raw, "file_type": "code", "source_file": stub_source_file, "confidence": "EXTRACTED", }) result.setdefault("edges", []).append(edge) existing_ids.add(node_id) _HTML_SCRIPT_TAG_RE = re.compile( r"\"'])*)>([\s\S]*?)", re.IGNORECASE, ) _NON_JS_SCRIPT_TYPE_RE = re.compile( r"""\btype\s*=\s*["']?(?!module\b|text/javascript\b|application/javascript\b)""", re.IGNORECASE, ) def _svelte_mask_non_script(src: str) -> tuple[str, str | None]: """Blank everything in a ``.svelte`` file except JS ```` with no body masks to an empty region, which parses as empty JS and contributes nothing — no special-casing needed. Returns ``(masked_source, had_script)``; the caller skips the JS pass entirely when ``had_script`` is False. """ def _blank(s: str) -> str: return re.sub(r"[^\r\n]", " ", s) out: list[str] = [] pos = 0 had_script = False for m in _PHP_SCRIPT_RE.finditer(src): had_script = True out.append(_blank(src[pos:m.start(1)])) # everything up to and including and any trailing part of the match pos = m.end() if not had_script: return src, False out.append(_blank(src[pos:])) return "".join(out), True # tree-sitter-php (the pinned <0.25 and 0.25.1 alike) has no PHP 8.5 `(void)` # cast and cannot parse a cast applied directly to `match (...)`: the statement # becomes an ERROR node, can lose its call, and the file is reported as partially # extracted (#4202). Blanking just the cast keeps the call or match after it. _PHP_UNPARSED_CAST_RE = re.compile( rb"\(\s*void\s*\)|\(\s*(?:string|int|integer|float|double|real|bool|boolean" rb"|array|object|binary)\s*\)(?=\s*match\s*\()", re.IGNORECASE, ) _PHP_NON_CODE = frozenset({"comment", "string", "encapsed_string", "heredoc", "nowdoc", "text"}) def _php_blank_unparsed_casts(source: bytes) -> bytes: """Blank `(void)` casts and casts on `match` to spaces, outside strings and comments, keeping every byte offset and line number (#4202).""" matches = list(_PHP_UNPARSED_CAST_RE.finditer(source)) if not matches: return source import tree_sitter_php from tree_sitter import Language, Parser root = Parser(Language(tree_sitter_php.language_php())).parse(source).root_node out = bytearray(source) for m in matches: node = root.descendant_for_byte_range(m.start(), m.start() + 1) while node is not None and node.type not in _PHP_NON_CODE: node = node.parent if node is None: out[m.start():m.end()] = re.sub(rb"[^\r\n]", b" ", m.group()) return bytes(out) def extract_php(path: Path) -> dict: """Extract classes, functions, methods, namespace uses, and calls from a .php file. Also parses any inline ``