mirror of
https://github.com/safishamsi/graphify.git
synced 2026-07-13 19:07:10 +00:00
Prototypes: repo-scoped import-first resolution + cross-language API-contract edges
Flag-gated experiments (GRAPHIFY_REPO_SCOPED_RESOLUTION, GRAPHIFY_API_CONTRACT_EDGES), validated but intentionally NOT on v8. Parked here so the work is recoverable. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Safi
@@ -9255,6 +9255,185 @@ def _resolve_swift_member_calls(
|
||||
})
|
||||
|
||||
|
||||
# ── Cross-language API-contract edges (client HTTP call ↔ server route) ───────
|
||||
# A `fetch('/api/users/:id')` in a TS frontend and `@app.route('/api/users/<id>')`
|
||||
# in a Python backend are the same endpoint but share no import and no language —
|
||||
# an edge no import- or scope-based resolver can produce. These passes detect both
|
||||
# sides, normalize the path templates, and link them.
|
||||
|
||||
# Server route declarations: group 1 = HTTP verb (or a "match-any" keyword),
|
||||
# group 2 = the path template.
|
||||
_API_ROUTE_RES = (
|
||||
# Flask / FastAPI / Starlette: @app.get("/x"), @router.post("/x"), @app.route("/x")
|
||||
re.compile(r"""@\s*[\w.]+\.(get|post|put|delete|patch|route)\s*\(\s*["']([^"']+)["']""", re.I),
|
||||
# Express / Koa / Fastify: app.get("/x"), router.post(`/x`)
|
||||
re.compile(r"""\b(?:app|router|api|server)\.(get|post|put|delete|patch|all)\s*\(\s*["'`]([^"'`]+)["'`]""", re.I),
|
||||
# Spring: @GetMapping("/x"), @RequestMapping("/x")
|
||||
re.compile(r"""@(Get|Post|Put|Delete|Patch|Request)Mapping\s*\(\s*(?:value\s*=\s*)?["']([^"']+)["']""", re.I),
|
||||
)
|
||||
|
||||
# Client HTTP calls. `fetch` has no verb group (verb lives in an options object,
|
||||
# default GET); axios/requests-style have an optional/required verb group.
|
||||
_API_FETCH_RE = re.compile(r"""\bfetch\s*\(\s*["'`]([^"'`]+)["'`]""", re.I)
|
||||
_API_AXIOS_RE = re.compile(r"""\baxios(?:\.(get|post|put|delete|patch))?\s*\(\s*["'`]([^"'`]+)["'`]""", re.I)
|
||||
_API_REQUESTS_RE = re.compile(r"""\b(?:requests|httpx|session|http_client)\.(get|post|put|delete|patch)\s*\(\s*["']([^"']+)["']""", re.I)
|
||||
|
||||
_API_ANY_VERB = "ANY"
|
||||
_API_SCAN_SUFFIXES = frozenset({".py", ".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".java", ".go", ".rb"})
|
||||
|
||||
|
||||
def _api_verb(raw: str) -> str:
|
||||
v = raw.upper()
|
||||
return _API_ANY_VERB if v in ("ROUTE", "ALL", "REQUEST") else v
|
||||
|
||||
|
||||
def _api_verbs_compatible(a: str, b: str) -> bool:
|
||||
return a == _API_ANY_VERB or b == _API_ANY_VERB or a == b
|
||||
|
||||
|
||||
def _api_normalize_path(raw: str) -> str:
|
||||
"""Canonicalize an HTTP path template so a client call and its route match.
|
||||
|
||||
Strips scheme/host/query/fragment and trailing slash, then replaces every
|
||||
parameter segment — Flask ``<id>``/``<int:id>``, FastAPI/Spring ``{id}``,
|
||||
Express/React ``:id``, JS template ``${...}``, and bare numeric ids — with
|
||||
``{}``.
|
||||
"""
|
||||
s = raw.strip()
|
||||
m = re.match(r"^[a-zA-Z][\w+.-]*://[^/]+(/.*)$", s)
|
||||
if m:
|
||||
s = m.group(1)
|
||||
if not s.startswith("/"):
|
||||
idx = s.find("/")
|
||||
s = s[idx:] if idx != -1 else "/" + s
|
||||
s = s.split("?", 1)[0].split("#", 1)[0]
|
||||
out = []
|
||||
for seg in s.split("/"):
|
||||
if seg == "":
|
||||
out.append("")
|
||||
elif (seg[0] in ":<" or (seg.startswith("{") and seg.endswith("}"))
|
||||
or "${" in seg or seg.isdigit()):
|
||||
out.append("{}")
|
||||
else:
|
||||
out.append(seg.lower())
|
||||
norm = "/".join(out)
|
||||
return norm[:-1] if len(norm) > 1 and norm.endswith("/") else norm
|
||||
|
||||
|
||||
def _api_path_is_specific(norm: str) -> bool:
|
||||
"""Require at least one literal (non-parameter) segment so we don't match on
|
||||
`/` or `/{}` and over-connect."""
|
||||
return any(seg and seg != "{}" for seg in norm.split("/"))
|
||||
|
||||
|
||||
def _scan_api_routes(line: str):
|
||||
for rx in _API_ROUTE_RES:
|
||||
for m in rx.finditer(line):
|
||||
yield _api_verb(m.group(1)), m.group(2)
|
||||
|
||||
|
||||
def _scan_api_clients(line: str):
|
||||
for m in _API_FETCH_RE.finditer(line):
|
||||
yield _API_ANY_VERB, m.group(1)
|
||||
for m in _API_AXIOS_RE.finditer(line):
|
||||
yield (_api_verb(m.group(1)) if m.group(1) else _API_ANY_VERB), m.group(2)
|
||||
for m in _API_REQUESTS_RE.finditer(line):
|
||||
yield _api_verb(m.group(1)), m.group(2)
|
||||
|
||||
|
||||
def _resolve_api_contract_edges(paths, all_nodes, all_edges) -> int:
|
||||
"""Link client HTTP calls to server route handlers across files and languages.
|
||||
|
||||
Conservative: only unique, cross-file, specific-path matches are linked, and
|
||||
the edge is INFERRED with ``resolution="api_contract"`` so it is clearly a
|
||||
runtime-contract inference, not a static reference. Returns the edge count.
|
||||
"""
|
||||
funcs_by_file: dict[str, list[tuple[int, str]]] = {}
|
||||
file_node_by_file: dict[str, str] = {}
|
||||
for n in all_nodes:
|
||||
sf = n.get("source_file")
|
||||
if not sf:
|
||||
continue
|
||||
loc = str(n.get("source_location") or "").lstrip("L")
|
||||
try:
|
||||
line = int(loc) if loc else 0
|
||||
except ValueError:
|
||||
line = 0
|
||||
label = n.get("label", "")
|
||||
if label.endswith("()"):
|
||||
funcs_by_file.setdefault(sf, []).append((line, n["id"]))
|
||||
elif label == Path(sf).name:
|
||||
file_node_by_file[sf] = n["id"]
|
||||
for sf in funcs_by_file:
|
||||
funcs_by_file[sf].sort()
|
||||
|
||||
def _enclosing(sf: str, line: int) -> str | None:
|
||||
best = None
|
||||
for fl, nid in funcs_by_file.get(sf, []):
|
||||
if fl <= line:
|
||||
best = nid
|
||||
else:
|
||||
break
|
||||
return best or file_node_by_file.get(sf)
|
||||
|
||||
def _handler_below(sf: str, line: int) -> str | None:
|
||||
below = [(fl, nid) for fl, nid in funcs_by_file.get(sf, []) if fl >= line]
|
||||
if below:
|
||||
return below[0][1]
|
||||
return _enclosing(sf, line)
|
||||
|
||||
routes: dict[str, list[tuple[str, str, str]]] = {} # norm -> [(verb, file, anchor)]
|
||||
clients: list[tuple[str, str, str, str]] = [] # (norm, verb, file, anchor)
|
||||
for p in paths:
|
||||
if p.suffix.lower() not in _API_SCAN_SUFFIXES:
|
||||
continue
|
||||
sf = str(p)
|
||||
try:
|
||||
text = p.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
continue
|
||||
for i, lt in enumerate(text.splitlines(), start=1):
|
||||
for verb, raw in _scan_api_routes(lt):
|
||||
norm = _api_normalize_path(raw)
|
||||
if _api_path_is_specific(norm):
|
||||
anchor = _handler_below(sf, i)
|
||||
if anchor:
|
||||
routes.setdefault(norm, []).append((verb, sf, anchor))
|
||||
for verb, raw in _scan_api_clients(lt):
|
||||
norm = _api_normalize_path(raw)
|
||||
if _api_path_is_specific(norm):
|
||||
anchor = _enclosing(sf, i)
|
||||
if anchor:
|
||||
clients.append((norm, verb, sf, anchor))
|
||||
|
||||
existing = {(e["source"], e["target"]) for e in all_edges}
|
||||
n_added = 0
|
||||
for norm, cverb, cfile, csrc in clients:
|
||||
targets = {
|
||||
rnid for (rverb, rfile, rnid) in routes.get(norm, [])
|
||||
if rfile != cfile and _api_verbs_compatible(cverb, rverb)
|
||||
}
|
||||
if len(targets) != 1: # unambiguous, cross-file match only
|
||||
continue
|
||||
tgt = next(iter(targets))
|
||||
if tgt == csrc or (csrc, tgt) in existing:
|
||||
continue
|
||||
existing.add((csrc, tgt))
|
||||
n_added += 1
|
||||
all_edges.append({
|
||||
"source": csrc,
|
||||
"target": tgt,
|
||||
"relation": "calls",
|
||||
"context": "api_call",
|
||||
"confidence": "INFERRED",
|
||||
"confidence_score": 0.7,
|
||||
"resolution": "api_contract",
|
||||
"source_file": cfile,
|
||||
"weight": 1.0,
|
||||
})
|
||||
return n_added
|
||||
|
||||
|
||||
def extract_objc(path: Path) -> dict:
|
||||
"""Extract interfaces, implementations, protocols, methods, and imports from .m/.mm/.h files."""
|
||||
try:
|
||||
@@ -12805,6 +12984,53 @@ def extract(
|
||||
sf_rel = sf_path
|
||||
nid_to_file_nid[n["id"]] = _file_node_id(sf_rel)
|
||||
|
||||
# PROTOTYPE (improvement #1): repo-scoped import-first resolution. A "repo" is
|
||||
# the nearest ancestor dir containing a package manifest; files with no
|
||||
# manifest ancestor all share the scan root, so single-repo behaviour is
|
||||
# unchanged (same_repo == every candidate). Opt-in via the env flag so the
|
||||
# default path is untouched while we evaluate it.
|
||||
_repo_scoped = os.environ.get("GRAPHIFY_REPO_SCOPED_RESOLUTION", "").strip() == "1"
|
||||
nid_to_repo: dict[str, str] = {}
|
||||
if _repo_scoped:
|
||||
_MANIFESTS = ("setup.py", "pyproject.toml", "setup.cfg", "requirements.txt",
|
||||
"package.json", "go.mod", "Cargo.toml", "pom.xml", "build.gradle")
|
||||
_repo_roots: set[Path] = set()
|
||||
_checked: set[Path] = set()
|
||||
for p in paths:
|
||||
d = p.parent.resolve()
|
||||
while True:
|
||||
if d not in _checked:
|
||||
_checked.add(d)
|
||||
try:
|
||||
if any((d / m).exists() for m in _MANIFESTS):
|
||||
_repo_roots.add(d)
|
||||
except OSError:
|
||||
pass
|
||||
if d == root or d.parent == d:
|
||||
break
|
||||
d = d.parent
|
||||
|
||||
def _repo_of(sf: str) -> str:
|
||||
if not sf:
|
||||
return str(root)
|
||||
fp = Path(sf)
|
||||
fp = (root / fp) if not fp.is_absolute() else fp
|
||||
try:
|
||||
fp = fp.resolve()
|
||||
except OSError:
|
||||
pass
|
||||
best: Path | None = None
|
||||
for r in _repo_roots:
|
||||
if r == fp or r in fp.parents:
|
||||
if best is None or len(str(r)) > len(str(best)):
|
||||
best = r
|
||||
return str(best) if best is not None else str(root)
|
||||
|
||||
for n in all_nodes:
|
||||
sf = n.get("source_file")
|
||||
if sf:
|
||||
nid_to_repo[n["id"]] = _repo_of(sf)
|
||||
|
||||
existing_pairs = {(e["source"], e["target"]) for e in all_edges}
|
||||
for rc in all_raw_calls:
|
||||
callee = rc.get("callee", "")
|
||||
@@ -12835,6 +13061,21 @@ def extract(
|
||||
or (candidate_file_nid is not None and candidate_file_nid in imported_modules)
|
||||
)
|
||||
|
||||
# PROTOTYPE (improvement #1): resolve within the caller's repo first.
|
||||
# Prefer same-repo definitions (a colliding name resolves to the local
|
||||
# one, not another repo's); a cross-repo definition links only with
|
||||
# explicit import/dependency evidence (kills phantom cross-repo edges to
|
||||
# an accidentally-unique name). No-op for single-repo corpora.
|
||||
if _repo_scoped and nid_to_repo:
|
||||
caller_repo = nid_to_repo.get(caller)
|
||||
same_repo = [c for c in candidates if nid_to_repo.get(c) == caller_repo]
|
||||
if same_repo:
|
||||
candidates = same_repo
|
||||
else:
|
||||
candidates = [c for c in candidates if _has_import_evidence(c)]
|
||||
if not candidates:
|
||||
continue
|
||||
|
||||
if len(candidates) == 1:
|
||||
tgt = candidates[0]
|
||||
has_import_evidence = _has_import_evidence(tgt)
|
||||
@@ -12893,6 +13134,16 @@ def extract(
|
||||
import logging
|
||||
logging.getLogger(__name__).warning("Swift member-call resolution failed, skipping: %s", exc)
|
||||
|
||||
# Cross-language API-contract edges: link client HTTP calls to server route
|
||||
# handlers across files/languages (frontend ↔ backend). Opt-in while we
|
||||
# evaluate; conservative (unique, cross-file, specific-path matches only).
|
||||
if os.environ.get("GRAPHIFY_API_CONTRACT_EDGES", "").strip() == "1":
|
||||
try:
|
||||
_resolve_api_contract_edges(paths, all_nodes, all_edges)
|
||||
except Exception as exc:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning("API-contract edge resolution failed, skipping: %s", exc)
|
||||
|
||||
# Relativize source_file fields so paths are portable across machines (#555)
|
||||
for item in all_nodes + all_edges:
|
||||
sf = item.get("source_file")
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from graphify.extract import (
|
||||
extract,
|
||||
_api_normalize_path,
|
||||
_api_verbs_compatible,
|
||||
_api_path_is_specific,
|
||||
)
|
||||
|
||||
|
||||
# ── path normalization (unit) ───────────────────────────────────────────────
|
||||
|
||||
def test_param_styles_normalize_equal():
|
||||
# Flask, FastAPI/Spring, Express/React, and a JS template literal must all
|
||||
# canonicalize to the same path so a client matches its route.
|
||||
forms = [
|
||||
"/api/users/<int:id>", # Flask
|
||||
"/api/users/{id}", # FastAPI / Spring
|
||||
"/api/users/:id", # Express / React
|
||||
"/api/users/${id}", # JS template
|
||||
"/api/users/42", # concrete numeric id
|
||||
"https://host/api/users/7?x=1#f", # scheme/host/query/fragment stripped
|
||||
]
|
||||
norms = {_api_normalize_path(f) for f in forms}
|
||||
assert norms == {"/api/users/{}"}
|
||||
|
||||
|
||||
def test_specificity_guard():
|
||||
assert not _api_path_is_specific("/")
|
||||
assert not _api_path_is_specific("/{}")
|
||||
assert _api_path_is_specific("/api/users/{}")
|
||||
|
||||
|
||||
def test_verb_compatibility():
|
||||
assert _api_verbs_compatible("GET", "GET")
|
||||
assert _api_verbs_compatible("ANY", "POST") # fetch/route default
|
||||
assert not _api_verbs_compatible("GET", "POST")
|
||||
|
||||
|
||||
# ── end-to-end resolution ───────────────────────────────────────────────────
|
||||
|
||||
def _write(p: Path, text: str) -> Path:
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(text, encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
def _api_edges(result: dict) -> set[tuple[str, str]]:
|
||||
by_id = {n["id"]: n.get("label", "") for n in result["nodes"]}
|
||||
return {
|
||||
(by_id.get(e["source"], ""), by_id.get(e["target"], ""))
|
||||
for e in result["edges"]
|
||||
if e.get("resolution") == "api_contract"
|
||||
}
|
||||
|
||||
|
||||
def _two_service_fixture(base: Path):
|
||||
_write(base / "frontend/src/api.ts",
|
||||
"export async function loadUser(id: number) {\n"
|
||||
" const res = await fetch(`/api/users/${id}`);\n"
|
||||
" return res.json();\n"
|
||||
"}\n\n"
|
||||
"export async function createUser(data: any) {\n"
|
||||
" return axios.post('/api/users', data);\n"
|
||||
"}\n")
|
||||
_write(base / "backend/app.py",
|
||||
"from flask import Flask\napp = Flask(__name__)\n\n"
|
||||
"@app.get('/api/users/<int:id>')\n"
|
||||
"def get_user(id):\n return {'id': id}\n\n"
|
||||
"@app.post('/api/users')\n"
|
||||
"def create_user():\n return {'created': True}\n")
|
||||
return sorted([p for p in base.rglob("*") if p.suffix in (".ts", ".py")])
|
||||
|
||||
|
||||
def test_cross_language_api_edges_resolve(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("GRAPHIFY_API_CONTRACT_EDGES", "1")
|
||||
files = _two_service_fixture(tmp_path / "src")
|
||||
result = extract(files, cache_root=tmp_path / "src")
|
||||
edges = _api_edges(result)
|
||||
assert ("loadUser()", "get_user()") in edges # GET /api/users/{}
|
||||
assert ("createUser()", "create_user()") in edges # POST /api/users
|
||||
|
||||
|
||||
def test_api_edges_off_by_default(tmp_path, monkeypatch):
|
||||
monkeypatch.delenv("GRAPHIFY_API_CONTRACT_EDGES", raising=False)
|
||||
files = _two_service_fixture(tmp_path / "src")
|
||||
result = extract(files, cache_root=tmp_path / "src")
|
||||
assert _api_edges(result) == set()
|
||||
|
||||
|
||||
def test_same_file_call_and_route_not_linked(tmp_path, monkeypatch):
|
||||
# A fetch and a route in the SAME file are not an API contract across services.
|
||||
monkeypatch.setenv("GRAPHIFY_API_CONTRACT_EDGES", "1")
|
||||
_write(tmp_path / "src/app.py",
|
||||
"from flask import Flask\napp = Flask(__name__)\n\n"
|
||||
"@app.get('/api/ping')\n"
|
||||
"def ping():\n return 'ok'\n\n"
|
||||
"def caller():\n return requests.get('/api/ping')\n")
|
||||
files = sorted((tmp_path / "src").rglob("*.py"))
|
||||
result = extract(files, cache_root=tmp_path / "src")
|
||||
assert _api_edges(result) == set()
|
||||
|
||||
|
||||
def test_ambiguous_route_path_not_linked(tmp_path, monkeypatch):
|
||||
# Same path served by two different backend files -> ambiguous -> skip.
|
||||
monkeypatch.setenv("GRAPHIFY_API_CONTRACT_EDGES", "1")
|
||||
_write(tmp_path / "src/frontend/c.ts",
|
||||
"function go() { return fetch('/api/data'); }\n")
|
||||
_write(tmp_path / "src/svcA/a.py",
|
||||
"@app.get('/api/data')\ndef a():\n return 1\n")
|
||||
_write(tmp_path / "src/svcB/b.py",
|
||||
"@app.get('/api/data')\ndef b():\n return 2\n")
|
||||
files = sorted([p for p in (tmp_path / "src").rglob("*") if p.suffix in (".ts", ".py")])
|
||||
result = extract(files, cache_root=tmp_path / "src")
|
||||
assert _api_edges(result) == set()
|
||||
Reference in New Issue
Block a user