mirror of
https://github.com/safishamsi/graphify.git
synced 2026-09-22 21:45:58 +00:00
Parse package manifests into canonical package nodes + depends_on edges (#1377)
apm.yml was a .yml document handled by the LLM, so the same package got a different file-anchored node id from its own manifest than from each dependent's dependency reference and split into duplicate nodes. New manifest_ingest module parses apm.yml/pyproject.toml/go.mod/pom.xml deterministically into ONE package node per package, keyed by name via ids.make_id, plus depends_on edges; routed to the AST path (CODE) so the LLM never sees them. Package nodes are exempt from the file-stem prefix remap so the canonical id is stable across manifests and dedup collapses references to a single hub node. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Safi
@@ -383,6 +383,13 @@ def _shebang_file_type(path: Path) -> FileType | None:
|
||||
|
||||
|
||||
def classify_file(path: Path) -> FileType | None:
|
||||
# Package manifests (apm.yml, pyproject.toml, go.mod, pom.xml) are parsed
|
||||
# deterministically, so route them to the AST path (CODE) rather than the LLM
|
||||
# document path — otherwise apm.yml (a .yml "document") would be LLM-extracted
|
||||
# and a package would split into duplicate file-anchored nodes (#1377).
|
||||
from graphify.manifest_ingest import is_package_manifest_path
|
||||
if is_package_manifest_path(path):
|
||||
return FileType.CODE
|
||||
# Compound extensions must be checked before simple suffix lookup
|
||||
if path.name.lower().endswith(".blade.php"):
|
||||
return FileType.CODE
|
||||
|
||||
@@ -13,6 +13,7 @@ from typing import Any, Callable
|
||||
from .cache import load_cached, save_cached
|
||||
from .ids import make_id
|
||||
from .mcp_ingest import extract_mcp_config, is_mcp_config_path
|
||||
from .manifest_ingest import extract_package_manifest, is_package_manifest_path
|
||||
|
||||
_RECURSION_LIMIT = 10_000
|
||||
|
||||
@@ -12379,6 +12380,11 @@ def _get_extractor(path: Path) -> Any | None:
|
||||
# (servers, commands, packages, env vars) instead of opaque JSON keys.
|
||||
if is_mcp_config_path(path):
|
||||
return extract_mcp_config
|
||||
# Package manifests (apm.yml, pyproject.toml, go.mod, pom.xml) → a canonical
|
||||
# package node + depends_on edges, by filename before generic suffix dispatch
|
||||
# (#1377). apm.yml would otherwise be a .yml document handled by the LLM.
|
||||
if is_package_manifest_path(path):
|
||||
return extract_package_manifest
|
||||
return _DISPATCH.get(path.suffix)
|
||||
|
||||
|
||||
@@ -12684,6 +12690,12 @@ def extract(
|
||||
sf = n.get("source_file")
|
||||
if not sf:
|
||||
continue
|
||||
# Package nodes carry a canonical name-keyed id (pkg_<name>) that must
|
||||
# stay identical across every manifest that references the package, so
|
||||
# they are exempt from the file-stem prefix remap (#1377), like the
|
||||
# type=module anchors (#1327).
|
||||
if n.get("type") == "package":
|
||||
continue
|
||||
try:
|
||||
entry = prefix_remap.get(Path(sf).resolve())
|
||||
except Exception:
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
"""Deterministic package-manifest ingestion (#1377).
|
||||
|
||||
Package manifests (``apm.yml``, ``pyproject.toml``, ``go.mod``, ``pom.xml``)
|
||||
declare a package and its dependencies. Left to the LLM document path, the same
|
||||
package gets a different file-anchored node id from its own manifest than from
|
||||
each dependent's dependency reference, so it splits into duplicate nodes. This
|
||||
module parses manifests deterministically and emits ONE canonical package node
|
||||
per package -- keyed by NAME via :func:`graphify.ids.make_id` -- plus
|
||||
``depends_on`` edges, so a package referenced from N manifests collapses to a
|
||||
single hub node (the dependency stub and the package's own definition node share
|
||||
the canonical id and merge at build time).
|
||||
|
||||
Mirrors ``mcp_ingest``: recognized by filename, routed to the deterministic AST
|
||||
path (never the LLM), so a manifest is extracted exactly once.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from graphify.ids import make_id
|
||||
|
||||
__all__ = ["is_package_manifest_path", "extract_package_manifest", "PACKAGE_MANIFEST_NAMES"]
|
||||
|
||||
# manifest filename (lowercased) -> ecosystem tag
|
||||
PACKAGE_MANIFEST_NAMES: dict[str, str] = {
|
||||
"apm.yml": "apm",
|
||||
"apm.yaml": "apm",
|
||||
"pyproject.toml": "python",
|
||||
"go.mod": "go",
|
||||
"pom.xml": "maven",
|
||||
}
|
||||
|
||||
_MAX_MANIFEST_BYTES = 2_000_000 # 2 MB cap — manifests are small; this rejects junk
|
||||
|
||||
|
||||
def is_package_manifest_path(path: Path) -> bool:
|
||||
"""True if ``path`` is a recognized package manifest (by filename)."""
|
||||
return path.name.lower() in PACKAGE_MANIFEST_NAMES
|
||||
|
||||
|
||||
def _pkg_id(name: str) -> str:
|
||||
"""Canonical package node id, keyed by package NAME so every reference to the
|
||||
same package -- its own manifest and any dependent's dependency line -- maps
|
||||
to one node."""
|
||||
return make_id("pkg", name)
|
||||
|
||||
|
||||
def extract_package_manifest(path: Path) -> dict[str, Any]:
|
||||
"""Parse a package manifest into a canonical package node + ``depends_on`` edges."""
|
||||
try:
|
||||
if path.stat().st_size > _MAX_MANIFEST_BYTES:
|
||||
return {"nodes": [], "edges": [], "error": "manifest too large to index"}
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError as exc:
|
||||
return {"nodes": [], "edges": [], "error": f"manifest read error: {exc}"}
|
||||
|
||||
eco = PACKAGE_MANIFEST_NAMES[path.name.lower()]
|
||||
try:
|
||||
info = _PARSERS[eco](text)
|
||||
except Exception as exc: # noqa: BLE001 — a malformed manifest must not abort extraction
|
||||
return {"nodes": [], "edges": [], "error": f"manifest parse error: {exc}"}
|
||||
if not info or not info.get("name"):
|
||||
return {"nodes": [], "edges": []}
|
||||
|
||||
name = info["name"]
|
||||
str_path = str(path)
|
||||
pkg_nid = _pkg_id(name)
|
||||
node: dict[str, Any] = {
|
||||
"id": pkg_nid,
|
||||
"label": name,
|
||||
"file_type": "code", # valid schema type; `type` distinguishes packages
|
||||
"type": "package",
|
||||
"ecosystem": eco,
|
||||
"source_file": str_path,
|
||||
"source_location": "L1",
|
||||
}
|
||||
if info.get("version"):
|
||||
node["version"] = info["version"]
|
||||
nodes: list[dict] = [node]
|
||||
edges: list[dict] = []
|
||||
|
||||
seen: set[str] = set()
|
||||
for dep in info.get("deps", []):
|
||||
if not dep:
|
||||
continue
|
||||
dep_nid = _pkg_id(dep)
|
||||
if dep_nid == pkg_nid or dep_nid in seen:
|
||||
continue
|
||||
seen.add(dep_nid)
|
||||
# The edge targets the dependency's canonical package id. If that package's
|
||||
# own manifest is in the corpus, the edge resolves to its (single) node; if
|
||||
# the dependency is external, build_from_json prunes the dangling edge. We
|
||||
# deliberately do NOT emit a stub node — a stub with an empty source_file
|
||||
# would risk clobbering the real node's source_file under id-dedup.
|
||||
edges.append({
|
||||
"source": pkg_nid,
|
||||
"target": dep_nid,
|
||||
"relation": "depends_on",
|
||||
"context": "dependency",
|
||||
"confidence": "EXTRACTED",
|
||||
"confidence_score": 1.0,
|
||||
"source_file": str_path,
|
||||
"source_location": "L1",
|
||||
"weight": 1.0,
|
||||
})
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
|
||||
|
||||
# ── per-ecosystem parsers: text -> {"name", "version"?, "deps": [str]} | None ──
|
||||
|
||||
def _coerce_deps(value: Any) -> list[str]:
|
||||
"""A dependency block may be a list of names or a name->spec map."""
|
||||
if isinstance(value, dict):
|
||||
return [str(k) for k in value]
|
||||
if isinstance(value, list):
|
||||
out: list[str] = []
|
||||
for item in value:
|
||||
if isinstance(item, str):
|
||||
out.append(item)
|
||||
elif isinstance(item, dict) and item:
|
||||
out.append(str(next(iter(item))))
|
||||
return out
|
||||
return []
|
||||
|
||||
|
||||
def _parse_apm(text: str) -> dict | None:
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
return _parse_apm_fallback(text)
|
||||
data = yaml.safe_load(text)
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return {
|
||||
"name": data.get("name"),
|
||||
"version": data.get("version"),
|
||||
"deps": _coerce_deps(data.get("dependencies")),
|
||||
}
|
||||
|
||||
|
||||
def _parse_apm_fallback(text: str) -> dict | None:
|
||||
"""Minimal line parser for apm.yml when PyYAML is unavailable: a top-level
|
||||
``name:`` plus a simple ``dependencies:`` block (list items or a name map)."""
|
||||
name = None
|
||||
deps: list[str] = []
|
||||
in_deps = False
|
||||
for line in text.splitlines():
|
||||
if not in_deps:
|
||||
m = re.match(r'^name:\s*["\']?([^"\'\s#]+)', line)
|
||||
if m:
|
||||
name = m.group(1)
|
||||
continue
|
||||
if re.match(r'^dependencies:\s*$', line):
|
||||
in_deps = True
|
||||
continue
|
||||
if in_deps:
|
||||
dm = (re.match(r'^\s*-\s*["\']?([^"\'\s#:]+)', line)
|
||||
or re.match(r'^\s{2,}([A-Za-z0-9._/@-]+)\s*:', line))
|
||||
if dm:
|
||||
deps.append(dm.group(1))
|
||||
elif re.match(r'^\S', line): # next top-level key ends the block
|
||||
in_deps = False
|
||||
return {"name": name, "version": None, "deps": deps} if name else None
|
||||
|
||||
|
||||
def _pep508_name(spec: str) -> str:
|
||||
"""`requests>=2.0` -> `requests`; `pkg[extra]==1; python_version<'3.9'` -> `pkg`."""
|
||||
return re.split(r'[\s<>=!~;\[\(]', spec.strip(), maxsplit=1)[0]
|
||||
|
||||
|
||||
def _parse_pyproject(text: str) -> dict | None:
|
||||
try:
|
||||
import tomllib as _toml
|
||||
except ImportError:
|
||||
try:
|
||||
import tomli as _toml # type: ignore
|
||||
except ImportError:
|
||||
return None
|
||||
data = _toml.loads(text)
|
||||
proj = data.get("project", {}) if isinstance(data.get("project"), dict) else {}
|
||||
poetry = (data.get("tool", {}) or {}).get("poetry", {}) if isinstance(data.get("tool"), dict) else {}
|
||||
name = proj.get("name") or (poetry.get("name") if isinstance(poetry, dict) else None)
|
||||
if not name:
|
||||
return None
|
||||
deps: list[str] = [_pep508_name(s) for s in (proj.get("dependencies") or []) if isinstance(s, str)]
|
||||
if isinstance(poetry, dict):
|
||||
for dep in (poetry.get("dependencies") or {}):
|
||||
if str(dep).lower() != "python":
|
||||
deps.append(str(dep))
|
||||
return {"name": name, "version": proj.get("version") or (poetry.get("version") if isinstance(poetry, dict) else None), "deps": deps}
|
||||
|
||||
|
||||
def _parse_gomod(text: str) -> dict | None:
|
||||
name = None
|
||||
deps: list[str] = []
|
||||
in_block = False
|
||||
for line in text.splitlines():
|
||||
s = line.strip()
|
||||
if name is None:
|
||||
m = re.match(r'^module\s+(\S+)', s)
|
||||
if m:
|
||||
name = m.group(1)
|
||||
continue
|
||||
if re.match(r'^require\s*\(', s):
|
||||
in_block = True
|
||||
continue
|
||||
if in_block:
|
||||
if s.startswith(')'):
|
||||
in_block = False
|
||||
continue
|
||||
dm = re.match(r'^(\S+)\s+v\S+', s)
|
||||
if dm:
|
||||
deps.append(dm.group(1))
|
||||
else:
|
||||
dm = re.match(r'^require\s+(\S+)\s+v\S+', s)
|
||||
if dm:
|
||||
deps.append(dm.group(1))
|
||||
return {"name": name, "version": None, "deps": deps} if name else None
|
||||
|
||||
|
||||
def _parse_pom(text: str) -> dict | None:
|
||||
# Drop the default namespace so findtext/findall don't need the {uri} prefix.
|
||||
text = re.sub(r'\sxmlns="[^"]*"', '', text, count=1)
|
||||
root = ET.fromstring(text)
|
||||
aid = root.findtext("artifactId")
|
||||
gid = root.findtext("groupId")
|
||||
if not aid:
|
||||
return None
|
||||
name = f"{gid}:{aid}" if gid else aid
|
||||
deps: list[str] = []
|
||||
for dep in root.findall(".//dependencies/dependency"):
|
||||
da = dep.findtext("artifactId")
|
||||
dg = dep.findtext("groupId")
|
||||
if da:
|
||||
deps.append(f"{dg}:{da}" if dg else da)
|
||||
return {"name": name, "version": root.findtext("version"), "deps": deps}
|
||||
|
||||
|
||||
_PARSERS = {
|
||||
"apm": _parse_apm,
|
||||
"python": _parse_pyproject,
|
||||
"go": _parse_gomod,
|
||||
"maven": _parse_pom,
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from graphify.build import build_from_json
|
||||
from graphify.detect import FileType, classify_file
|
||||
from graphify.extract import extract
|
||||
from graphify.manifest_ingest import (
|
||||
extract_package_manifest,
|
||||
is_package_manifest_path,
|
||||
)
|
||||
|
||||
|
||||
def _write(p: Path, text: str) -> Path:
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(text, encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
# ── routing: manifests are deterministic (CODE), not LLM documents ───────────
|
||||
|
||||
def test_manifests_classify_as_code_not_document(tmp_path):
|
||||
for name in ("apm.yml", "pyproject.toml", "go.mod", "pom.xml"):
|
||||
p = _write(tmp_path / name, "x")
|
||||
assert is_package_manifest_path(p)
|
||||
assert classify_file(p) is FileType.CODE, name
|
||||
# a generic yaml stays a document
|
||||
assert classify_file(_write(tmp_path / "config.yaml", "a: 1")) is FileType.DOCUMENT
|
||||
|
||||
|
||||
# ── per-format parsing ───────────────────────────────────────────────────────
|
||||
|
||||
def _pkg_nodes(result):
|
||||
return [n for n in result["nodes"] if n.get("type") == "package"]
|
||||
|
||||
|
||||
def test_apm_parses_name_and_deps(tmp_path):
|
||||
p = _write(tmp_path / "apm.yml",
|
||||
"name: my-pkg\nversion: 1.2.3\ndependencies:\n - dep-a\n - dep-b\n")
|
||||
r = extract_package_manifest(p)
|
||||
pkg = _pkg_nodes(r)[0]
|
||||
assert pkg["label"] == "my-pkg" and pkg["version"] == "1.2.3"
|
||||
deps = {e["target"] for e in r["edges"] if e["relation"] == "depends_on"}
|
||||
assert {"pkg_dep_a", "pkg_dep_b"} <= deps
|
||||
|
||||
|
||||
def test_pyproject_parses_pep508_deps(tmp_path):
|
||||
p = _write(tmp_path / "pyproject.toml",
|
||||
'[project]\nname = "cool-lib"\nversion = "0.1"\n'
|
||||
'dependencies = ["requests>=2.0", "rich[jupyter]==13.0", "tomli; python_version<\'3.11\'"]\n')
|
||||
r = extract_package_manifest(p)
|
||||
assert _pkg_nodes(r)[0]["label"] == "cool-lib"
|
||||
deps = {e["target"] for e in r["edges"]}
|
||||
assert {"pkg_requests", "pkg_rich", "pkg_tomli"} <= deps # versions/extras/markers stripped
|
||||
|
||||
|
||||
def test_gomod_parses_module_and_requires(tmp_path):
|
||||
p = _write(tmp_path / "go.mod",
|
||||
"module example.com/me/app\n\ngo 1.22\n\nrequire (\n"
|
||||
"\tgithub.com/x/y v1.2.3\n\tgithub.com/a/b v0.4.0\n)\n")
|
||||
r = extract_package_manifest(p)
|
||||
assert _pkg_nodes(r)[0]["label"] == "example.com/me/app"
|
||||
deps = {e["target"] for e in r["edges"]}
|
||||
assert "pkg_github_com_x_y" in deps and "pkg_github_com_a_b" in deps
|
||||
|
||||
|
||||
def test_pom_parses_artifact_and_deps(tmp_path):
|
||||
p = _write(tmp_path / "pom.xml",
|
||||
'<project xmlns="http://maven.apache.org/POM/4.0.0">\n'
|
||||
' <groupId>com.acme</groupId>\n <artifactId>widget</artifactId>\n <version>2.0</version>\n'
|
||||
' <dependencies>\n <dependency><groupId>org.lib</groupId><artifactId>core</artifactId></dependency>\n'
|
||||
' </dependencies>\n</project>\n')
|
||||
r = extract_package_manifest(p)
|
||||
assert _pkg_nodes(r)[0]["label"] == "com.acme:widget"
|
||||
assert any(e["target"] == "pkg_org_lib_core" for e in r["edges"])
|
||||
|
||||
|
||||
# ── #1377: a package referenced by N manifests is ONE node ───────────────────
|
||||
|
||||
def test_apm_dependency_collapses_to_single_canonical_node(tmp_path):
|
||||
base = tmp_path / "packages"
|
||||
_write(base / "core/apm.yml", "name: coding-standards-core\nversion: 1.0.4\n")
|
||||
_write(base / "csharp/apm.yml",
|
||||
"name: coding-standards-csharp\ndependencies:\n - coding-standards-core\n")
|
||||
_write(base / "python/apm.yml",
|
||||
'name: coding-standards-python\ndependencies:\n coding-standards-core: ">=1.0"\n')
|
||||
files = sorted(base.rglob("apm.yml"))
|
||||
result = extract(files, cache_root=tmp_path)
|
||||
|
||||
core = [n for n in result["nodes"]
|
||||
if n.get("type") == "package" and n["label"] == "coding-standards-core"]
|
||||
assert len(core) == 1, "core package must be a single canonical node"
|
||||
assert core[0]["id"] == "pkg_coding_standards_core" and core[0]["source_file"]
|
||||
|
||||
g = build_from_json(result)
|
||||
core_ids = [n for n, d in g.nodes(data=True) if d.get("label") == "coding-standards-core"]
|
||||
dep_edges = [(u, v) for u, v, d in g.edges(data=True) if d.get("relation") == "depends_on"]
|
||||
assert len(core_ids) == 1
|
||||
assert len(dep_edges) == 2 # both dependents point at the one core node
|
||||
|
||||
|
||||
def test_external_dependency_edge_pruned_not_orphaned(tmp_path):
|
||||
# A dep whose manifest isn't in the corpus: the edge dangles and build prunes it.
|
||||
p = _write(tmp_path / "apm.yml", "name: leaf\ndependencies:\n - some-external-pkg\n")
|
||||
result = extract([p], cache_root=tmp_path)
|
||||
g = build_from_json(result)
|
||||
assert "pkg_some_external_pkg" not in set(g.nodes()) # no fabricated external node
|
||||
assert [n for n, d in g.nodes(data=True) if d.get("label") == "leaf"]
|
||||
|
||||
|
||||
def test_malformed_manifest_does_not_crash(tmp_path):
|
||||
p = _write(tmp_path / "pom.xml", "<project><not closed")
|
||||
r = extract_package_manifest(p) # parse error -> empty, no exception
|
||||
assert r["nodes"] == [] and r["edges"] == []
|
||||
Reference in New Issue
Block a user