mirror of
https://github.com/safishamsi/graphify.git
synced 2026-07-12 18:37:12 +00:00
Co-Authored-By: spindle79 <spindle79@users.noreply.github.com>
This commit is contained in:
Safi
+1
-1
@@ -24,7 +24,7 @@ class FileType(str, Enum):
|
||||
|
||||
_MANIFEST_PATH = "graphify-out/manifest.json"
|
||||
|
||||
CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.mjs', '.ejs', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.dart', '.v', '.sv', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk'}
|
||||
CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.mjs', '.ejs', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk'}
|
||||
DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'}
|
||||
PAPER_EXTENSIONS = {'.pdf'}
|
||||
IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
|
||||
|
||||
@@ -2123,6 +2123,142 @@ def extract_svelte(path: Path) -> dict:
|
||||
return result
|
||||
|
||||
|
||||
def extract_astro(path: Path) -> dict:
|
||||
"""Extract imports from .astro files: frontmatter (TS) + template regex fallback.
|
||||
|
||||
Astro files start with a ``---\\n...\\n---`` frontmatter block of TypeScript
|
||||
setup code (where almost all imports live), followed by an HTML-with-expressions
|
||||
template body, and optionally ``<script>`` blocks for client-side JS. Tree-sitter
|
||||
only sees the file usefully through the frontmatter — feeding the whole file to
|
||||
the JS parser produces a top-level ERROR node because the template is not valid
|
||||
JS, so ``import_statement`` nodes are never reached and static imports are
|
||||
silently dropped (#850). Mirrors :func:`extract_svelte` — same regex-rescue
|
||||
approach, scanning the frontmatter block and any client-side ``<script>`` blocks
|
||||
for static and dynamic imports.
|
||||
"""
|
||||
result = _extract_generic(path, _JS_CONFIG)
|
||||
try:
|
||||
import re as _re
|
||||
src = path.read_text(encoding="utf-8", errors="replace")
|
||||
existing_ids = {n["id"] for n in result.get("nodes", [])}
|
||||
file_node_id = _make_id(str(path))
|
||||
aliases = _load_tsconfig_aliases(path.parent)
|
||||
# Dynamic imports anywhere in the file: `import('./X.astro')` is legal in
|
||||
# frontmatter setup code and inside expression slots.
|
||||
for m in _re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src):
|
||||
raw = m.group(1)
|
||||
if not raw:
|
||||
continue
|
||||
if raw.startswith("."):
|
||||
resolved = Path(os.path.normpath(path.parent / raw))
|
||||
resolved = _resolve_js_module_path(resolved)
|
||||
node_id = _make_id(str(resolved))
|
||||
stub_source_file = str(resolved)
|
||||
else:
|
||||
resolved_alias = None
|
||||
for alias_prefix, alias_base in aliases.items():
|
||||
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
||||
rest = raw[len(alias_prefix):].lstrip("/")
|
||||
resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
|
||||
break
|
||||
if resolved_alias is not None:
|
||||
resolved_alias = _resolve_js_module_path(resolved_alias)
|
||||
node_id = _make_id(str(resolved_alias))
|
||||
stub_source_file = str(resolved_alias)
|
||||
else:
|
||||
module_name = raw.split("/")[-1]
|
||||
if not module_name:
|
||||
continue
|
||||
node_id = _make_id(module_name)
|
||||
stub_source_file = raw
|
||||
if node_id in existing_ids:
|
||||
result.setdefault("edges", []).append({
|
||||
"source": file_node_id, "target": node_id,
|
||||
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
||||
"source_file": str(path),
|
||||
})
|
||||
continue
|
||||
result.setdefault("nodes", []).append({
|
||||
"id": node_id, "label": raw,
|
||||
"file_type": "code", "source_file": stub_source_file,
|
||||
"confidence": "EXTRACTED",
|
||||
})
|
||||
result.setdefault("edges", []).append({
|
||||
"source": file_node_id, "target": node_id,
|
||||
"relation": "dynamic_import", "confidence": "EXTRACTED",
|
||||
"source_file": str(path),
|
||||
})
|
||||
existing_ids.add(node_id)
|
||||
# Static imports: scan the `---...---` frontmatter at the file head plus any
|
||||
# client-side <script> blocks. Both are TS/JS regions but live inside a file
|
||||
# the JS tree-sitter parser cannot validate as a whole.
|
||||
frontmatter_re = _re.compile(
|
||||
r"\A\s*---\s*\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n|\Z)"
|
||||
)
|
||||
script_re = _re.compile(
|
||||
r"<script\b[^>]*>([\s\S]*?)</script\s*>", _re.IGNORECASE
|
||||
)
|
||||
static_import_re = _re.compile(
|
||||
r"""import\s+(?:[^'"`;]+?\s+from\s+)?['"]([^'"]+)['"]"""
|
||||
)
|
||||
regions: list[str] = []
|
||||
fm = frontmatter_re.search(src)
|
||||
if fm:
|
||||
regions.append(fm.group(1))
|
||||
for script_match in script_re.finditer(src):
|
||||
regions.append(script_match.group(1))
|
||||
for region in regions:
|
||||
for m in static_import_re.finditer(region):
|
||||
raw = m.group(1)
|
||||
if not raw:
|
||||
continue
|
||||
if raw.startswith("."):
|
||||
resolved = Path(os.path.normpath(path.parent / raw))
|
||||
if resolved.suffix == ".js":
|
||||
resolved = resolved.with_suffix(".ts")
|
||||
elif resolved.suffix == ".jsx":
|
||||
resolved = resolved.with_suffix(".tsx")
|
||||
node_id = _make_id(str(resolved))
|
||||
stub_source_file = str(resolved)
|
||||
else:
|
||||
resolved_alias = None
|
||||
for alias_prefix, alias_base in aliases.items():
|
||||
if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
|
||||
rest = raw[len(alias_prefix):].lstrip("/")
|
||||
resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
|
||||
break
|
||||
if resolved_alias is not None:
|
||||
node_id = _make_id(str(resolved_alias))
|
||||
stub_source_file = str(resolved_alias)
|
||||
else:
|
||||
module_name = raw.split("/")[-1]
|
||||
if not module_name:
|
||||
continue
|
||||
node_id = _make_id(module_name)
|
||||
stub_source_file = raw
|
||||
if node_id in existing_ids:
|
||||
result.setdefault("edges", []).append({
|
||||
"source": file_node_id, "target": node_id,
|
||||
"relation": "imports_from", "confidence": "EXTRACTED",
|
||||
"source_file": str(path),
|
||||
})
|
||||
continue
|
||||
result.setdefault("nodes", []).append({
|
||||
"id": node_id, "label": raw,
|
||||
"file_type": "code", "source_file": stub_source_file,
|
||||
"confidence": "EXTRACTED",
|
||||
})
|
||||
result.setdefault("edges", []).append({
|
||||
"source": file_node_id, "target": node_id,
|
||||
"relation": "imports_from", "confidence": "EXTRACTED",
|
||||
"source_file": str(path),
|
||||
})
|
||||
existing_ids.add(node_id)
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def extract_java(path: Path) -> dict:
|
||||
"""Extract classes, interfaces, methods, constructors, and imports from a .java file."""
|
||||
return _extract_generic(path, _JAVA_CONFIG)
|
||||
@@ -5490,6 +5626,7 @@ _DISPATCH: dict[str, Any] = {
|
||||
".F08": extract_fortran,
|
||||
".vue": extract_js,
|
||||
".svelte": extract_svelte,
|
||||
".astro": extract_astro,
|
||||
".dart": extract_dart,
|
||||
".v": extract_verilog,
|
||||
".sv": extract_verilog,
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Tests for `.astro` extraction (#850).
|
||||
|
||||
Astro files have a TypeScript frontmatter block (`---...---`) at the top where
|
||||
nearly all imports live, followed by an HTML-with-expressions template and
|
||||
optionally `<script>` blocks. Tree-sitter-javascript fed the whole file produces
|
||||
a top-level ERROR node because the template is not valid JS, so the JS AST pass
|
||||
recovers nothing. The :func:`extract_astro` regex pass salvages imports from the
|
||||
frontmatter and any `<script>` blocks — same strategy as :func:`extract_svelte`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from graphify.detect import CODE_EXTENSIONS
|
||||
from graphify.extract import (
|
||||
_make_id,
|
||||
extract_astro,
|
||||
)
|
||||
|
||||
|
||||
def _write(path: Path, body: str) -> Path:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(body, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _import_targets(result: dict, *, relation: str | None = None) -> set[str]:
|
||||
return {
|
||||
str(e.get("target") or "")
|
||||
for e in result.get("edges", [])
|
||||
if relation is None or e.get("relation") == relation
|
||||
}
|
||||
|
||||
|
||||
def test_astro_is_in_code_extensions():
|
||||
"""Without this, detect.py silently drops `.astro` from the AST pass (#850)."""
|
||||
assert ".astro" in CODE_EXTENSIONS
|
||||
|
||||
|
||||
def test_extract_astro_picks_up_frontmatter_static_imports(tmp_path):
|
||||
page = _write(
|
||||
tmp_path / "src/pages/index.astro",
|
||||
"""---
|
||||
import Layout from '../layouts/Layout.astro';
|
||||
import Hero from '../components/Hero.astro';
|
||||
const { title } = Astro.props;
|
||||
---
|
||||
|
||||
<Layout title={title}>
|
||||
<Hero />
|
||||
</Layout>
|
||||
""",
|
||||
)
|
||||
# Sibling files so the resolver lands on real node ids, not phantoms.
|
||||
layout = _write(tmp_path / "src/layouts/Layout.astro", "---\n---\n<slot />\n")
|
||||
hero = _write(tmp_path / "src/components/Hero.astro", "---\n---\n<h1>hi</h1>\n")
|
||||
|
||||
result = extract_astro(page)
|
||||
targets = _import_targets(result, relation="imports_from")
|
||||
assert _make_id(str(layout)) in targets
|
||||
assert _make_id(str(hero)) in targets
|
||||
|
||||
|
||||
def test_extract_astro_handles_dynamic_import_in_frontmatter(tmp_path):
|
||||
page = _write(
|
||||
tmp_path / "src/pages/lazy.astro",
|
||||
"""---
|
||||
const Mod = await import('./Other.astro');
|
||||
---
|
||||
|
||||
<div>{Mod.default}</div>
|
||||
""",
|
||||
)
|
||||
other = _write(tmp_path / "src/pages/Other.astro", "---\n---\n<p>o</p>\n")
|
||||
|
||||
result = extract_astro(page)
|
||||
targets = _import_targets(result, relation="dynamic_import")
|
||||
assert _make_id(str(other)) in targets
|
||||
|
||||
|
||||
def test_extract_astro_picks_up_client_side_script_imports(tmp_path):
|
||||
page = _write(
|
||||
tmp_path / "src/pages/with-script.astro",
|
||||
"""---
|
||||
import Layout from '../layouts/Layout.astro';
|
||||
---
|
||||
|
||||
<Layout>
|
||||
<button id="b">click</button>
|
||||
</Layout>
|
||||
|
||||
<script>
|
||||
import { hydrate } from '../client/hydrate.ts';
|
||||
hydrate(document.getElementById('b'));
|
||||
</script>
|
||||
""",
|
||||
)
|
||||
layout = _write(tmp_path / "src/layouts/Layout.astro", "---\n---\n<slot />\n")
|
||||
hydrate = _write(tmp_path / "src/client/hydrate.ts", "export function hydrate(){}\n")
|
||||
|
||||
result = extract_astro(page)
|
||||
targets = _import_targets(result, relation="imports_from")
|
||||
assert _make_id(str(layout)) in targets
|
||||
assert _make_id(str(hydrate)) in targets
|
||||
|
||||
|
||||
def test_extract_astro_no_frontmatter_does_not_crash(tmp_path):
|
||||
"""Astro permits frontmatter-less files (pure-HTML pages). Must not raise."""
|
||||
page = _write(
|
||||
tmp_path / "src/pages/plain.astro",
|
||||
"<h1>no frontmatter here</h1>\n",
|
||||
)
|
||||
result = extract_astro(page)
|
||||
# Empty/no-imports result is acceptable; the extractor must just not crash.
|
||||
assert isinstance(result, dict)
|
||||
assert _import_targets(result, relation="imports_from") == set()
|
||||
|
||||
|
||||
def test_extract_astro_handles_tsconfig_path_alias(tmp_path):
|
||||
_write(
|
||||
tmp_path / "tsconfig.json",
|
||||
"""{
|
||||
"compilerOptions": {
|
||||
"baseUrl": ".",
|
||||
"paths": { "@components/*": ["src/components/*"] }
|
||||
}
|
||||
}
|
||||
""",
|
||||
)
|
||||
page = _write(
|
||||
tmp_path / "src/pages/alias.astro",
|
||||
"""---
|
||||
import Hero from '@components/Hero.astro';
|
||||
---
|
||||
|
||||
<Hero />
|
||||
""",
|
||||
)
|
||||
hero = _write(tmp_path / "src/components/Hero.astro", "---\n---\n<h1>h</h1>\n")
|
||||
|
||||
result = extract_astro(page)
|
||||
targets = _import_targets(result, relation="imports_from")
|
||||
assert _make_id(str(hero)) in targets
|
||||
Reference in New Issue
Block a user