feat(markdown): resolve code-span mentions into references edges

The markdown extractor's docstring promised a heading --references--> node edge
when a backtick Name matches a known pattern, but nothing implemented it, so an
AST build produced no document-to-code edge at all.

extract_markdown now collects inline code spans outside fences and frontmatter as
raw_calls tagged language "markdown", owned by the heading they sit under (the
page before the first heading). A new markdown_mentions language resolver runs
after the id-remap passes and turns them into references edges: the explicit
path::Symbol[::member] form resolves through the cited file's symbols and is
EXTRACTED 1.0; a bare or dotted name resolves only when exactly one callable code
node carries that label and is INFERRED 0.95. Ambiguous names, builtins, missing
files and non-callable matches emit nothing. The shared cross-file call pass
skips markdown raw_calls so prose never emits calls edges.

Tests cover span classification, raw_call ownership, end-to-end resolution,
ambiguity, and cache round-tripping.
This commit is contained in:
Ryan Peterson committed 2026-09-14 15:12:17 -04:00
1 parent fe66389083
commit 2d833a0a3c
4 files changed
+482 -2

No files matched your search

+16
View File
@@ -25,6 +25,7 @@ from .resolver_registry import (
from .ruby_resolution import resolve_ruby_member_calls
from .csharp_dispatch import resolve_csharp_interface_dispatch
from .pascal_resolution import resolve_pascal_inherited_calls
from .markdown_resolution import MARKDOWN_MENTION_SUFFIXES, resolve_markdown_mentions
# --- migrated to graphify/extractors/ (see graphify/extractors/MIGRATION.md) ---
from graphify.extractors.base import ( # noqa: F401
@@ -4806,6 +4807,15 @@ register_language_resolver(
"csharp_interface_dispatch", frozenset({".cs"}), resolve_csharp_interface_dispatch
)
)
# Markdown code-span mentions (`Widget`, `mod.py::Widget::render`) become
# heading --references--> symbol edges once every file is extracted and ids are
# final. Lives in graphify.markdown_resolution; the shared call pass above skips
# these raw_calls so a mention is never mistaken for a call.
register_language_resolver(
LanguageResolver(
"markdown_mentions", MARKDOWN_MENTION_SUFFIXES, resolve_markdown_mentions
)
)
# Inline markdown link: [text](target "optional title"). The negative lookbehind
@@ -7375,6 +7385,12 @@ def extract(
# in the corpus — exactly what #2141 must not do.
if rc.get("language") == "bash":
continue
# A Markdown code span names a symbol, it does not call one. The
# markdown_mentions resolver turns it into a `references` edge with the
# cited file as evidence; a global name match here would mint a `calls`
# edge from a heading to whatever shares the name.
if rc.get("language") == "markdown":
continue
# A Go predeclared function is never a cross-file call: the extractor
# already drops bare `append(s, x)` (extractors/go.py), so this is the
# backstop for Go raw_calls minted on any other path. Language-gated
+94 -2
View File
@@ -6,6 +6,7 @@ import os
import unicodedata
from pathlib import Path
from graphify.detect import CODE_EXTENSIONS
from graphify.extractors.base import _file_stem, _make_id
from graphify.security import sanitize_metadata
@@ -18,6 +19,22 @@ _MD_WIKILINK_RE = re.compile(r'(?<!\!)\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]')
_MD_LINKABLE_EXTS = {".md", ".mdx", ".qmd", ".markdown", ".rst", ".txt"}
# Inline code spans, single or double backtick delimited. Fenced blocks are
# skipped before this runs, so a span here is prose citing a symbol by name.
_MD_CODE_SPAN_RE = re.compile(r'(?<!`)(`+)(?!`)(.+?)(?<!`)\1(?!`)')
# The two mention grammars a code span can carry. A path-qualified mention
# (``src/pkg/mod.py::Widget::render``) names the defining file and a
# ``::``-separated symbol chain, exactly as pytest node ids and cite-lint
# citations do. A bare mention (``Widget``, ``render()``, ``pkg.Widget``) names
# a symbol with no file evidence, so it resolves only on a unique match.
_MD_PATH_MENTION_RE = re.compile(
r'^([A-Za-z0-9_./\-]+\.[A-Za-z0-9]+)::([A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)(?:\(\))?$'
)
_MD_BARE_MENTION_RE = re.compile(
r'^(?:[A-Za-z_]\w*\.)*([A-Za-z_]\w*)(?:\(\))?$'
)
# A YAML frontmatter block is only frontmatter when the opening `---` is the
# very first line of the file. A `---` further down is a horizontal rule and
# must not be mistaken for one. Bounded so a file that opens a fence and never
@@ -235,6 +252,33 @@ def _resolve_markdown_link(raw: str, source_dir: Path,
return Path(os.path.normpath(str(hit)))
return resolved
def _code_span_mention(span: str) -> "tuple[str | None, list[str]] | None":
"""Classify one inline code span as a symbol mention, or None.
Returns ``(path, names)``: ``path`` is the cited file for the
``path::Name`` form and None for a bare or dotted mention; ``names`` is the
symbol chain, outermost first. A trailing ``()`` is dropped from the last
name so ``render()`` and ``render`` cite the same symbol. Spans that read
as a file (``setup.py``), a shell command, an expression or prose are not
mentions and yield None.
"""
text = span.strip()
if not text or " " in text:
return None
m = _MD_PATH_MENTION_RE.match(text)
if m:
return m.group(1), m.group(2).split("::")
m = _MD_BARE_MENTION_RE.match(text)
if not m:
return None
if "." in text:
# ``a.b.Name`` is a qualified symbol; ``setup.py`` is a file. A dotted
# span whose last segment is a code extension is the latter.
if "." + m.group(1) in CODE_EXTENSIONS:
return None
return None, [m.group(1)]
def extract_markdown(path: Path) -> dict:
"""Extract structural nodes and edges from a Markdown file.
@@ -253,7 +297,6 @@ def extract_markdown(path: Path) -> dict:
Produces edges for:
- file --contains--> heading
- parent heading --contains--> child heading (nesting by level)
- heading --references--> other node (when backtick `Name` matches a known pattern)
- file --references--> linked document, for inline ``[text](./other.md)``,
reference-style ``[label]: ./other.md`` and ``[[wikilink]]`` links, so a
hub doc (``index.md`` / ``table-of-contents.md``) becomes a real hub node
@@ -262,6 +305,19 @@ def extract_markdown(path: Path) -> dict:
own node, so the edge merges into that node (no ghost node). External
URLs, in-page anchors, images and non-document targets are skipped.
Inline code spans on heading and body lines (backtick-quoted ``Widget``,
``render()``, ``pkg.Widget`` or ``src/mod.py::Widget::render``) are
collected as ``raw_calls`` tagged ``language: "markdown"`` rather than
edges: the symbol they cite lives in another file, so the match is made
once every file is extracted and ids are final, by the
``markdown_mentions`` language resolver (see
``graphify.markdown_resolution``). That pass emits
heading --references--> code symbol (page --references--> symbol for a
mention above the first heading): EXTRACTED for the path-qualified form,
INFERRED for a bare name that matches exactly one code symbol. The shared
cross-file call pass skips these raw_calls, so a mention never becomes a
``calls`` edge.
Fenced code blocks (``` ... ```) are skipped during parsing so their
contents don't get treated as headings, but no node is emitted for
them — they were always orphans (only a single contains edge to the
@@ -283,7 +339,33 @@ def extract_markdown(path: Path) -> dict:
str_path = str(path)
nodes: list[dict] = []
edges: list[dict] = []
raw_calls: list[dict] = []
seen_ids: set[str] = set()
seen_mentions: set[tuple[str, "str | None", tuple[str, ...]]] = set()
def add_mention(caller_nid: str, span: str, line: int) -> None:
mention = _code_span_mention(span)
if mention is None:
return
cited_path, names = mention
key = (caller_nid, cited_path, tuple(names))
if key in seen_mentions:
return
seen_mentions.add(key)
raw_call = {
"caller_nid": caller_nid,
"callee": names[-1],
"is_member_call": False,
"language": "markdown",
"context": span.strip(),
"source_file": str_path,
"source_location": f"L{line}",
}
if cited_path is not None:
raw_call["path"] = cited_path
if len(names) > 1:
raw_call["qualifiers"] = names[:-1]
raw_calls.append(raw_call)
def add_node(nid: str, label: str, line: int, file_type: str = "document",
node_kind: str = "heading", extra: "dict | None" = None) -> None:
@@ -403,6 +485,16 @@ def extract_markdown(path: Path) -> dict:
add_edge(parent, h_nid, "contains", line_num)
heading_stack.append((level, h_nid))
# A code span in the heading itself is that heading's mention.
for m in _MD_CODE_SPAN_RE.finditer(title):
add_mention(h_nid, m.group(2), line_num)
continue
return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0}
# Body prose: a code span cites a symbol on behalf of the enclosing
# heading (the page when the text precedes the first heading).
owner = heading_stack[-1][1] if heading_stack else file_nid
for m in _MD_CODE_SPAN_RE.finditer(line_text):
add_mention(owner, m.group(2), line_num)
return {"nodes": nodes, "edges": edges, "raw_calls": raw_calls,
"input_tokens": 0, "output_tokens": 0}
+175
View File
@@ -0,0 +1,175 @@
"""Cross-file resolution of code-span mentions in Markdown documents.
``extract_markdown`` reports every inline code span that reads as a symbol
name (``Widget``, ``render()``, ``pkg.Widget``, ``src/mod.py::Widget::render``)
as a ``raw_calls`` entry tagged ``language: "markdown"`` instead of guessing a
target id: the symbol is defined in some other file, and only once every file
is extracted and the id-remap passes have run can the mention be matched to
the node that actually exists. Riding the ``raw_calls`` channel means those
passes rewrite the mention's ``caller_nid`` alongside every other raw call,
and the extraction cache round-trips it for free.
This resolver runs from the language-resolver registry (see
``resolver_registry``) with the merged corpus in hand and turns each mention
into a ``references`` edge from the heading (or page) that cites the symbol
to the code node that defines it:
* a path-qualified mention resolves against the cited file only. The file is
matched by its path relative to the document, then as a segment-aligned
suffix of a scanned file, and must be unique; the symbol chain is walked
through ``contains`` / ``method`` edges so ``mod.py::Widget::render`` is
the ``render`` owned by ``Widget``. The document names the file and the
symbol verbatim, so the edge is EXTRACTED;
* a bare mention resolves only when exactly one callable or type in the
corpus carries that label (case-sensitive; language built-ins excluded).
Only nodes the extractors marked ``_callable`` qualify, for the reason the
indirect-call pass gives (#1566): a by-name match must land on a real
function, method or class, never on a same-named data symbol such as a
JSON key. No tie-breaking: an ambiguous name is a guess, and a guess is
not an edge. The match is INFERRED (0.95, a named cross-file reference).
The shared cross-file call pass in ``extract`` skips ``markdown`` raw calls,
so a mention never surfaces as a ``calls`` edge.
"""
from __future__ import annotations
import os
from typing import Any
from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS
MARKDOWN_MENTION_SUFFIXES = frozenset({".md", ".mdx", ".qmd", ".skill"})
_CONTAINMENT_RELATIONS = frozenset({"contains", "method"})
def _symbol_label(node: dict[str, Any]) -> str:
"""A node's label as a document would write it: no ``()``, no leading dot."""
return str(node.get("label", "")).strip("()").lstrip(".")
def _is_file_node(node: dict[str, Any]) -> bool:
label = str(node.get("label", ""))
source_file = str(node.get("source_file", ""))
return bool(source_file) and label == os.path.basename(source_file)
def _posix(path: str) -> str:
return path.replace("\\", "/")
def _match_cited_file(cited: str, doc_file: str, source_files: set[str]) -> str | None:
"""The scanned source file a ``path::Name`` mention names, or None.
Tries the path relative to the citing document first (``../src/mod.py``
from ``docs/guide.md``), then an exact match, then a unique
segment-aligned suffix (``src/mod.py`` naming ``/repo/src/mod.py``).
"""
cited_posix = _posix(cited)
doc_dir = os.path.dirname(doc_file)
relative = _posix(os.path.normpath(os.path.join(doc_dir, cited)))
for candidate in (relative, cited_posix):
if candidate in source_files:
return candidate
suffix = "/" + cited_posix.lstrip("./")
matches = [sf for sf in source_files if sf.endswith(suffix)]
return matches[0] if len(matches) == 1 else None
def _markdown_raw_calls(per_file: list[dict]) -> list[dict]:
calls: list[dict] = []
for result in per_file:
if not isinstance(result, dict):
continue
for rc in result.get("raw_calls", []) or []:
if isinstance(rc, dict) and rc.get("language") == "markdown":
calls.append(rc)
return calls
def resolve_markdown_mentions(
per_file: list[dict],
all_nodes: list[dict],
all_edges: list[dict],
) -> None:
"""Turn ``language: "markdown"`` raw calls into ``references`` edges in place."""
mentions = _markdown_raw_calls(per_file)
if not mentions:
return
code_nodes = [
n for n in all_nodes
if n.get("id") and n.get("file_type") == "code" and not _is_file_node(n)
]
by_label: dict[str, list[str]] = {}
by_file: dict[str, dict[str, list[str]]] = {}
for n in code_nodes:
label = _symbol_label(n)
if not label:
continue
if n.get("_callable"):
by_label.setdefault(label, []).append(n["id"])
source_file = _posix(str(n.get("source_file", "")))
if source_file:
by_file.setdefault(source_file, {}).setdefault(label, []).append(n["id"])
source_files = set(by_file)
parents: dict[str, set[str]] = {}
for e in all_edges:
if e.get("relation") in _CONTAINMENT_RELATIONS:
parents.setdefault(str(e.get("target")), set()).add(str(e.get("source")))
node_ids = {n["id"] for n in all_nodes if n.get("id")}
existing = {
(e.get("source"), e.get("target"))
for e in all_edges if e.get("relation") == "references"
}
def _resolve_chain(file_labels: dict[str, list[str]], names: list[str]) -> str | None:
candidates = list(file_labels.get(names[-1], []))
for depth, owner_name in enumerate(reversed(names[:-1]), start=1):
owners = set(file_labels.get(owner_name, []))
narrowed = []
for c in candidates:
chain = {c}
for _ in range(depth):
chain = {p for nid in chain for p in parents.get(nid, ())}
if chain & owners:
narrowed.append(c)
candidates = narrowed
return candidates[0] if len(candidates) == 1 else None
for rc in mentions:
caller = str(rc.get("caller_nid", ""))
callee = str(rc.get("callee", "")).strip()
if not caller or not callee or caller not in node_ids:
continue
names = list(rc.get("qualifiers") or []) + [callee]
cited = rc.get("path")
if cited:
source_file = _match_cited_file(
str(cited), _posix(str(rc.get("source_file", ""))), source_files
)
if source_file is None:
continue
target = _resolve_chain(by_file[source_file], names)
confidence, score = "EXTRACTED", 1.0
else:
if callee in _LANGUAGE_BUILTIN_GLOBALS:
continue
candidates = by_label.get(callee, [])
target = candidates[0] if len(candidates) == 1 else None
confidence, score = "INFERRED", 0.95
if target is None or target == caller or (caller, target) in existing:
continue
existing.add((caller, target))
all_edges.append({
"source": caller,
"target": target,
"relation": "references",
"context": rc.get("context", ""),
"confidence": confidence,
"confidence_score": score,
"source_file": rc.get("source_file", ""),
"source_location": rc.get("source_location"),
"weight": 1.0,
})
+197
View File
@@ -0,0 +1,197 @@
"""Markdown code-span mentions become heading --references--> symbol edges.
``extract_markdown``'s docstring promised ``heading --references--> other node``
for a backtick `Name`, but nothing implemented it: a docs corpus and its code
corpus shared no edge at all. Mentions now ride the ``raw_calls`` channel
(tagged ``language: "markdown"``) and the ``markdown_mentions`` language
resolver matches them against the merged corpus after the id-remap passes.
"""
from __future__ import annotations
import os
from pathlib import Path
from graphify.extract import extract
from graphify.extractors.markdown import _code_span_mention, extract_markdown
_WIDGET_PY = '''\
class Widget:
def render(self):
return "w"
def helper():
return Widget()
'''
_GADGET_PY = '''\
class Gadget:
def render(self):
return "g"
def helper():
return Gadget()
'''
_GUIDE_MD = '''\
# Guide
Start with `Widget`; `str` and `print()` are built-ins.
## Rendering
`Widget` draws through `Widget.render()`; `helper()` is defined twice.
```python
print(Widget)
```
## Pinned
See `src/widget.py::Widget::render` and `src/gadget.py::helper`.
Also `../src/widget.py::Widget` and the missing `src/nowhere.py::Widget`.
'''
def _extract(tmp_path, files: dict[str, str]):
for name, body in files.items():
p = tmp_path / name
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(body)
old = os.getcwd()
try:
os.chdir(tmp_path)
return extract([Path(n) for n in files],
cache_root=tmp_path / ".cache", parallel=False)
finally:
os.chdir(old)
def _node(r, label, source_contains=""):
return next(n for n in r["nodes"]
if n["label"] == label and source_contains in str(n.get("source_file", "")))
def _references(r):
return {(e["source"], e["target"]): e for e in r["edges"]
if e["relation"] == "references"}
def test_code_span_mentions_are_classified():
assert _code_span_mention("Widget") == (None, ["Widget"])
assert _code_span_mention("render()") == (None, ["render"])
assert _code_span_mention("pkg.sub.Widget") == (None, ["Widget"])
pinned = ("src/widget.py", ["Widget", "render"])
assert _code_span_mention("src/widget.py::Widget::render") == pinned
assert _code_span_mention("src/widget.py::Widget::render()") == pinned
# Files, commands, expressions and prose are not symbol mentions.
for span in ("setup.py", "git revert", "x = 1", "a-b", "--flag", "", "src/widget.py"):
assert _code_span_mention(span) is None, span
def test_extract_markdown_reports_mentions_as_markdown_raw_calls(tmp_path):
doc = tmp_path / "docs" / "guide.md"
doc.parent.mkdir()
doc.write_text(_GUIDE_MD)
r = extract_markdown(doc)
assert r["edges"] == [e for e in r["edges"] if e["relation"] == "contains"]
calls = r["raw_calls"]
assert calls and all(rc["language"] == "markdown" for rc in calls)
assert all(rc["is_member_call"] is False for rc in calls)
heading_ids = {n["label"]: n["id"] for n in r["nodes"]}
owners = {(rc["context"], rc["caller_nid"]) for rc in calls}
# Body prose belongs to the enclosing heading; the first paragraph to the H1.
assert ("Widget", heading_ids["Guide"]) in owners
assert ("Widget", heading_ids["Rendering"]) in owners
dotted = next(rc for rc in calls if rc["context"] == "Widget.render()")
assert (dotted["caller_nid"], dotted["callee"]) == (heading_ids["Rendering"], "render")
pinned = next(rc for rc in calls if rc["context"] == "src/widget.py::Widget::render")
assert (pinned["path"], pinned["qualifiers"], pinned["callee"]) == (
"src/widget.py", ["Widget"], "render")
assert pinned["caller_nid"] == heading_ids["Pinned"]
assert pinned["source_location"] == "L15"
# Fenced code is not prose: `print(Widget)` inside the block adds nothing.
assert [rc for rc in calls if rc["source_location"] == "L10"] == []
# One raw call per (owner, mention), however often the heading repeats it.
assert len([rc for rc in calls if rc["callee"] == "Widget"
and rc["caller_nid"] == heading_ids["Rendering"]]) == 1
def test_extract_markdown_heading_code_span_belongs_to_that_heading(tmp_path):
doc = tmp_path / "api.md"
doc.write_text("# API\n\n## `Widget`\n\nText.\n")
r = extract_markdown(doc)
(rc,) = r["raw_calls"]
heading = next(n for n in r["nodes"] if n["label"] == "`Widget`")
assert (rc["caller_nid"], rc["callee"], rc["path"] if "path" in rc else None) == (
heading["id"], "Widget", None)
def test_mentions_resolve_to_references_edges_end_to_end(tmp_path):
r = _extract(tmp_path, {
"src/widget.py": _WIDGET_PY,
"src/gadget.py": _GADGET_PY,
"docs/guide.md": _GUIDE_MD,
})
refs = _references(r)
guide = _node(r, "Guide")
rendering = _node(r, "Rendering")
pinned = _node(r, "Pinned")
widget = _node(r, "Widget", "widget.py")
widget_render = next(
n for n in r["nodes"] if n["label"] == ".render()" and "widget" in n["id"])
gadget_helper = _node(r, "helper()", "gadget.py")
# Bare unique name: INFERRED reference from the citing heading.
assert refs[(guide["id"], widget["id"])]["confidence"] == "INFERRED"
assert refs[(guide["id"], widget["id"])]["confidence_score"] == 0.95
assert (rendering["id"], widget["id"]) in refs
# Dotted `Widget.render()` resolves by its last segment: `render` is
# ambiguous (two classes define it), so no edge; same for `helper()`.
assert {t for (s, t) in refs if s == rendering["id"]} == {widget["id"]}
# Path-qualified mentions are EXTRACTED and scoped to the cited file.
assert refs[(pinned["id"], widget_render["id"])]["confidence"] == "EXTRACTED"
assert refs[(pinned["id"], widget_render["id"])]["confidence_score"] == 1.0
assert refs[(pinned["id"], gadget_helper["id"])]["confidence"] == "EXTRACTED"
# `../src/widget.py::Widget` resolves relative to the document.
assert (pinned["id"], widget["id"]) in refs
# A cited file that is not in the corpus yields nothing.
assert {t for (s, t) in refs if s == pinned["id"]} == {
widget_render["id"], gadget_helper["id"], widget["id"]}
# Built-ins never resolve, and no mention becomes a call.
node_ids = {n["id"] for n in r["nodes"]}
assert all(s in node_ids and t in node_ids for (s, t) in refs)
doc_ids = {n["id"] for n in r["nodes"] if n.get("file_type") == "document"}
assert not [e for e in r["edges"]
if e["relation"] in ("calls", "indirect_call") and e["source"] in doc_ids]
def test_ambiguous_bare_name_yields_no_edge(tmp_path):
r = _extract(tmp_path, {
"a/thing.py": "class Thing:\n pass\n",
"b/thing.py": "class Thing:\n pass\n",
"notes.md": "# Notes\n\nUse `Thing`.\n",
})
notes = _node(r, "Notes")
assert {t for (s, t) in _references(r) if s == notes["id"]} == set()
def test_mentions_survive_the_extraction_cache(tmp_path):
files = {"src/widget.py": _WIDGET_PY, "docs/guide.md": _GUIDE_MD}
first = _extract(tmp_path, files)
second = _extract(tmp_path, files)
def _pairs(r):
return {(s, t) for (s, t) in _references(r)}
assert _pairs(first) == _pairs(second)
guide = _node(second, "Guide")
widget = _node(second, "Widget", "widget.py")
assert (guide["id"], widget["id"]) in _pairs(second)