From 2d833a0a3c43ddbed28c8ae2a43071d7cc472ff5 Mon Sep 17 00:00:00 2001 From: Ryan Peterson Date: Mon, 14 Sep 2026 15:12:17 -0400 Subject: [PATCH] feat(markdown): resolve code-span mentions into references edges The markdown extractor's docstring promised a heading --references--> node edge when a backtick Name matches a known pattern, but nothing implemented it, so an AST build produced no document-to-code edge at all. extract_markdown now collects inline code spans outside fences and frontmatter as raw_calls tagged language "markdown", owned by the heading they sit under (the page before the first heading). A new markdown_mentions language resolver runs after the id-remap passes and turns them into references edges: the explicit path::Symbol[::member] form resolves through the cited file's symbols and is EXTRACTED 1.0; a bare or dotted name resolves only when exactly one callable code node carries that label and is INFERRED 0.95. Ambiguous names, builtins, missing files and non-callable matches emit nothing. The shared cross-file call pass skips markdown raw_calls so prose never emits calls edges. Tests cover span classification, raw_call ownership, end-to-end resolution, ambiguity, and cache round-tripping. --- graphify/extract.py | 16 +++ graphify/extractors/markdown.py | 96 ++++++++++++++- graphify/markdown_resolution.py | 175 ++++++++++++++++++++++++++ tests/test_markdown_code_spans.py | 197 ++++++++++++++++++++++++++++++ 4 files changed, 482 insertions(+), 2 deletions(-) create mode 100644 graphify/markdown_resolution.py create mode 100644 tests/test_markdown_code_spans.py diff --git a/graphify/extract.py b/graphify/extract.py index 90a6ab026..20c3b66d2 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -25,6 +25,7 @@ from .resolver_registry import ( from .ruby_resolution import resolve_ruby_member_calls from .csharp_dispatch import resolve_csharp_interface_dispatch from .pascal_resolution import resolve_pascal_inherited_calls +from .markdown_resolution import MARKDOWN_MENTION_SUFFIXES, resolve_markdown_mentions # --- migrated to graphify/extractors/ (see graphify/extractors/MIGRATION.md) --- from graphify.extractors.base import ( # noqa: F401 @@ -4806,6 +4807,15 @@ register_language_resolver( "csharp_interface_dispatch", frozenset({".cs"}), resolve_csharp_interface_dispatch ) ) +# Markdown code-span mentions (`Widget`, `mod.py::Widget::render`) become +# heading --references--> symbol edges once every file is extracted and ids are +# final. Lives in graphify.markdown_resolution; the shared call pass above skips +# these raw_calls so a mention is never mistaken for a call. +register_language_resolver( + LanguageResolver( + "markdown_mentions", MARKDOWN_MENTION_SUFFIXES, resolve_markdown_mentions + ) +) # Inline markdown link: [text](target "optional title"). The negative lookbehind @@ -7375,6 +7385,12 @@ def extract( # in the corpus — exactly what #2141 must not do. if rc.get("language") == "bash": continue + # A Markdown code span names a symbol, it does not call one. The + # markdown_mentions resolver turns it into a `references` edge with the + # cited file as evidence; a global name match here would mint a `calls` + # edge from a heading to whatever shares the name. + if rc.get("language") == "markdown": + continue # A Go predeclared function is never a cross-file call: the extractor # already drops bare `append(s, x)` (extractors/go.py), so this is the # backstop for Go raw_calls minted on any other path. Language-gated diff --git a/graphify/extractors/markdown.py b/graphify/extractors/markdown.py index 50c1754a7..387074a96 100644 --- a/graphify/extractors/markdown.py +++ b/graphify/extractors/markdown.py @@ -6,6 +6,7 @@ import os import unicodedata from pathlib import Path +from graphify.detect import CODE_EXTENSIONS from graphify.extractors.base import _file_stem, _make_id from graphify.security import sanitize_metadata @@ -18,6 +19,22 @@ _MD_WIKILINK_RE = re.compile(r'(? "tuple[str | None, list[str]] | None": + """Classify one inline code span as a symbol mention, or None. + + Returns ``(path, names)``: ``path`` is the cited file for the + ``path::Name`` form and None for a bare or dotted mention; ``names`` is the + symbol chain, outermost first. A trailing ``()`` is dropped from the last + name so ``render()`` and ``render`` cite the same symbol. Spans that read + as a file (``setup.py``), a shell command, an expression or prose are not + mentions and yield None. + """ + text = span.strip() + if not text or " " in text: + return None + m = _MD_PATH_MENTION_RE.match(text) + if m: + return m.group(1), m.group(2).split("::") + m = _MD_BARE_MENTION_RE.match(text) + if not m: + return None + if "." in text: + # ``a.b.Name`` is a qualified symbol; ``setup.py`` is a file. A dotted + # span whose last segment is a code extension is the latter. + if "." + m.group(1) in CODE_EXTENSIONS: + return None + return None, [m.group(1)] + + def extract_markdown(path: Path) -> dict: """Extract structural nodes and edges from a Markdown file. @@ -253,7 +297,6 @@ def extract_markdown(path: Path) -> dict: Produces edges for: - file --contains--> heading - parent heading --contains--> child heading (nesting by level) - - heading --references--> other node (when backtick `Name` matches a known pattern) - file --references--> linked document, for inline ``[text](./other.md)``, reference-style ``[label]: ./other.md`` and ``[[wikilink]]`` links, so a hub doc (``index.md`` / ``table-of-contents.md``) becomes a real hub node @@ -262,6 +305,19 @@ def extract_markdown(path: Path) -> dict: own node, so the edge merges into that node (no ghost node). External URLs, in-page anchors, images and non-document targets are skipped. + Inline code spans on heading and body lines (backtick-quoted ``Widget``, + ``render()``, ``pkg.Widget`` or ``src/mod.py::Widget::render``) are + collected as ``raw_calls`` tagged ``language: "markdown"`` rather than + edges: the symbol they cite lives in another file, so the match is made + once every file is extracted and ids are final, by the + ``markdown_mentions`` language resolver (see + ``graphify.markdown_resolution``). That pass emits + heading --references--> code symbol (page --references--> symbol for a + mention above the first heading): EXTRACTED for the path-qualified form, + INFERRED for a bare name that matches exactly one code symbol. The shared + cross-file call pass skips these raw_calls, so a mention never becomes a + ``calls`` edge. + Fenced code blocks (``` ... ```) are skipped during parsing so their contents don't get treated as headings, but no node is emitted for them — they were always orphans (only a single contains edge to the @@ -283,7 +339,33 @@ def extract_markdown(path: Path) -> dict: str_path = str(path) nodes: list[dict] = [] edges: list[dict] = [] + raw_calls: list[dict] = [] seen_ids: set[str] = set() + seen_mentions: set[tuple[str, "str | None", tuple[str, ...]]] = set() + + def add_mention(caller_nid: str, span: str, line: int) -> None: + mention = _code_span_mention(span) + if mention is None: + return + cited_path, names = mention + key = (caller_nid, cited_path, tuple(names)) + if key in seen_mentions: + return + seen_mentions.add(key) + raw_call = { + "caller_nid": caller_nid, + "callee": names[-1], + "is_member_call": False, + "language": "markdown", + "context": span.strip(), + "source_file": str_path, + "source_location": f"L{line}", + } + if cited_path is not None: + raw_call["path"] = cited_path + if len(names) > 1: + raw_call["qualifiers"] = names[:-1] + raw_calls.append(raw_call) def add_node(nid: str, label: str, line: int, file_type: str = "document", node_kind: str = "heading", extra: "dict | None" = None) -> None: @@ -403,6 +485,16 @@ def extract_markdown(path: Path) -> dict: add_edge(parent, h_nid, "contains", line_num) heading_stack.append((level, h_nid)) + # A code span in the heading itself is that heading's mention. + for m in _MD_CODE_SPAN_RE.finditer(title): + add_mention(h_nid, m.group(2), line_num) continue - return {"nodes": nodes, "edges": edges, "input_tokens": 0, "output_tokens": 0} + # Body prose: a code span cites a symbol on behalf of the enclosing + # heading (the page when the text precedes the first heading). + owner = heading_stack[-1][1] if heading_stack else file_nid + for m in _MD_CODE_SPAN_RE.finditer(line_text): + add_mention(owner, m.group(2), line_num) + + return {"nodes": nodes, "edges": edges, "raw_calls": raw_calls, + "input_tokens": 0, "output_tokens": 0} diff --git a/graphify/markdown_resolution.py b/graphify/markdown_resolution.py new file mode 100644 index 000000000..bbae40dcb --- /dev/null +++ b/graphify/markdown_resolution.py @@ -0,0 +1,175 @@ +"""Cross-file resolution of code-span mentions in Markdown documents. + +``extract_markdown`` reports every inline code span that reads as a symbol +name (``Widget``, ``render()``, ``pkg.Widget``, ``src/mod.py::Widget::render``) +as a ``raw_calls`` entry tagged ``language: "markdown"`` instead of guessing a +target id: the symbol is defined in some other file, and only once every file +is extracted and the id-remap passes have run can the mention be matched to +the node that actually exists. Riding the ``raw_calls`` channel means those +passes rewrite the mention's ``caller_nid`` alongside every other raw call, +and the extraction cache round-trips it for free. + +This resolver runs from the language-resolver registry (see +``resolver_registry``) with the merged corpus in hand and turns each mention +into a ``references`` edge from the heading (or page) that cites the symbol +to the code node that defines it: + +* a path-qualified mention resolves against the cited file only. The file is + matched by its path relative to the document, then as a segment-aligned + suffix of a scanned file, and must be unique; the symbol chain is walked + through ``contains`` / ``method`` edges so ``mod.py::Widget::render`` is + the ``render`` owned by ``Widget``. The document names the file and the + symbol verbatim, so the edge is EXTRACTED; +* a bare mention resolves only when exactly one callable or type in the + corpus carries that label (case-sensitive; language built-ins excluded). + Only nodes the extractors marked ``_callable`` qualify, for the reason the + indirect-call pass gives (#1566): a by-name match must land on a real + function, method or class, never on a same-named data symbol such as a + JSON key. No tie-breaking: an ambiguous name is a guess, and a guess is + not an edge. The match is INFERRED (0.95, a named cross-file reference). + +The shared cross-file call pass in ``extract`` skips ``markdown`` raw calls, +so a mention never surfaces as a ``calls`` edge. +""" +from __future__ import annotations + +import os +from typing import Any + +from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS + +MARKDOWN_MENTION_SUFFIXES = frozenset({".md", ".mdx", ".qmd", ".skill"}) + +_CONTAINMENT_RELATIONS = frozenset({"contains", "method"}) + + +def _symbol_label(node: dict[str, Any]) -> str: + """A node's label as a document would write it: no ``()``, no leading dot.""" + return str(node.get("label", "")).strip("()").lstrip(".") + + +def _is_file_node(node: dict[str, Any]) -> bool: + label = str(node.get("label", "")) + source_file = str(node.get("source_file", "")) + return bool(source_file) and label == os.path.basename(source_file) + + +def _posix(path: str) -> str: + return path.replace("\\", "/") + + +def _match_cited_file(cited: str, doc_file: str, source_files: set[str]) -> str | None: + """The scanned source file a ``path::Name`` mention names, or None. + + Tries the path relative to the citing document first (``../src/mod.py`` + from ``docs/guide.md``), then an exact match, then a unique + segment-aligned suffix (``src/mod.py`` naming ``/repo/src/mod.py``). + """ + cited_posix = _posix(cited) + doc_dir = os.path.dirname(doc_file) + relative = _posix(os.path.normpath(os.path.join(doc_dir, cited))) + for candidate in (relative, cited_posix): + if candidate in source_files: + return candidate + suffix = "/" + cited_posix.lstrip("./") + matches = [sf for sf in source_files if sf.endswith(suffix)] + return matches[0] if len(matches) == 1 else None + + +def _markdown_raw_calls(per_file: list[dict]) -> list[dict]: + calls: list[dict] = [] + for result in per_file: + if not isinstance(result, dict): + continue + for rc in result.get("raw_calls", []) or []: + if isinstance(rc, dict) and rc.get("language") == "markdown": + calls.append(rc) + return calls + + +def resolve_markdown_mentions( + per_file: list[dict], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Turn ``language: "markdown"`` raw calls into ``references`` edges in place.""" + mentions = _markdown_raw_calls(per_file) + if not mentions: + return + + code_nodes = [ + n for n in all_nodes + if n.get("id") and n.get("file_type") == "code" and not _is_file_node(n) + ] + by_label: dict[str, list[str]] = {} + by_file: dict[str, dict[str, list[str]]] = {} + for n in code_nodes: + label = _symbol_label(n) + if not label: + continue + if n.get("_callable"): + by_label.setdefault(label, []).append(n["id"]) + source_file = _posix(str(n.get("source_file", ""))) + if source_file: + by_file.setdefault(source_file, {}).setdefault(label, []).append(n["id"]) + source_files = set(by_file) + + parents: dict[str, set[str]] = {} + for e in all_edges: + if e.get("relation") in _CONTAINMENT_RELATIONS: + parents.setdefault(str(e.get("target")), set()).add(str(e.get("source"))) + node_ids = {n["id"] for n in all_nodes if n.get("id")} + existing = { + (e.get("source"), e.get("target")) + for e in all_edges if e.get("relation") == "references" + } + + def _resolve_chain(file_labels: dict[str, list[str]], names: list[str]) -> str | None: + candidates = list(file_labels.get(names[-1], [])) + for depth, owner_name in enumerate(reversed(names[:-1]), start=1): + owners = set(file_labels.get(owner_name, [])) + narrowed = [] + for c in candidates: + chain = {c} + for _ in range(depth): + chain = {p for nid in chain for p in parents.get(nid, ())} + if chain & owners: + narrowed.append(c) + candidates = narrowed + return candidates[0] if len(candidates) == 1 else None + + for rc in mentions: + caller = str(rc.get("caller_nid", "")) + callee = str(rc.get("callee", "")).strip() + if not caller or not callee or caller not in node_ids: + continue + names = list(rc.get("qualifiers") or []) + [callee] + cited = rc.get("path") + if cited: + source_file = _match_cited_file( + str(cited), _posix(str(rc.get("source_file", ""))), source_files + ) + if source_file is None: + continue + target = _resolve_chain(by_file[source_file], names) + confidence, score = "EXTRACTED", 1.0 + else: + if callee in _LANGUAGE_BUILTIN_GLOBALS: + continue + candidates = by_label.get(callee, []) + target = candidates[0] if len(candidates) == 1 else None + confidence, score = "INFERRED", 0.95 + if target is None or target == caller or (caller, target) in existing: + continue + existing.add((caller, target)) + all_edges.append({ + "source": caller, + "target": target, + "relation": "references", + "context": rc.get("context", ""), + "confidence": confidence, + "confidence_score": score, + "source_file": rc.get("source_file", ""), + "source_location": rc.get("source_location"), + "weight": 1.0, + }) diff --git a/tests/test_markdown_code_spans.py b/tests/test_markdown_code_spans.py new file mode 100644 index 000000000..d081f84a7 --- /dev/null +++ b/tests/test_markdown_code_spans.py @@ -0,0 +1,197 @@ +"""Markdown code-span mentions become heading --references--> symbol edges. + +``extract_markdown``'s docstring promised ``heading --references--> other node`` +for a backtick `Name`, but nothing implemented it: a docs corpus and its code +corpus shared no edge at all. Mentions now ride the ``raw_calls`` channel +(tagged ``language: "markdown"``) and the ``markdown_mentions`` language +resolver matches them against the merged corpus after the id-remap passes. +""" +from __future__ import annotations + +import os +from pathlib import Path + +from graphify.extract import extract +from graphify.extractors.markdown import _code_span_mention, extract_markdown + +_WIDGET_PY = '''\ +class Widget: + def render(self): + return "w" + + +def helper(): + return Widget() +''' + +_GADGET_PY = '''\ +class Gadget: + def render(self): + return "g" + + +def helper(): + return Gadget() +''' + +_GUIDE_MD = '''\ +# Guide + +Start with `Widget`; `str` and `print()` are built-ins. + +## Rendering + +`Widget` draws through `Widget.render()`; `helper()` is defined twice. + +```python +print(Widget) +``` + +## Pinned + +See `src/widget.py::Widget::render` and `src/gadget.py::helper`. +Also `../src/widget.py::Widget` and the missing `src/nowhere.py::Widget`. +''' + + +def _extract(tmp_path, files: dict[str, str]): + for name, body in files.items(): + p = tmp_path / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(body) + old = os.getcwd() + try: + os.chdir(tmp_path) + return extract([Path(n) for n in files], + cache_root=tmp_path / ".cache", parallel=False) + finally: + os.chdir(old) + + +def _node(r, label, source_contains=""): + return next(n for n in r["nodes"] + if n["label"] == label and source_contains in str(n.get("source_file", ""))) + + +def _references(r): + return {(e["source"], e["target"]): e for e in r["edges"] + if e["relation"] == "references"} + + +def test_code_span_mentions_are_classified(): + assert _code_span_mention("Widget") == (None, ["Widget"]) + assert _code_span_mention("render()") == (None, ["render"]) + assert _code_span_mention("pkg.sub.Widget") == (None, ["Widget"]) + pinned = ("src/widget.py", ["Widget", "render"]) + assert _code_span_mention("src/widget.py::Widget::render") == pinned + assert _code_span_mention("src/widget.py::Widget::render()") == pinned + # Files, commands, expressions and prose are not symbol mentions. + for span in ("setup.py", "git revert", "x = 1", "a-b", "--flag", "", "src/widget.py"): + assert _code_span_mention(span) is None, span + + +def test_extract_markdown_reports_mentions_as_markdown_raw_calls(tmp_path): + doc = tmp_path / "docs" / "guide.md" + doc.parent.mkdir() + doc.write_text(_GUIDE_MD) + + r = extract_markdown(doc) + + assert r["edges"] == [e for e in r["edges"] if e["relation"] == "contains"] + calls = r["raw_calls"] + assert calls and all(rc["language"] == "markdown" for rc in calls) + assert all(rc["is_member_call"] is False for rc in calls) + heading_ids = {n["label"]: n["id"] for n in r["nodes"]} + owners = {(rc["context"], rc["caller_nid"]) for rc in calls} + # Body prose belongs to the enclosing heading; the first paragraph to the H1. + assert ("Widget", heading_ids["Guide"]) in owners + assert ("Widget", heading_ids["Rendering"]) in owners + dotted = next(rc for rc in calls if rc["context"] == "Widget.render()") + assert (dotted["caller_nid"], dotted["callee"]) == (heading_ids["Rendering"], "render") + pinned = next(rc for rc in calls if rc["context"] == "src/widget.py::Widget::render") + assert (pinned["path"], pinned["qualifiers"], pinned["callee"]) == ( + "src/widget.py", ["Widget"], "render") + assert pinned["caller_nid"] == heading_ids["Pinned"] + assert pinned["source_location"] == "L15" + # Fenced code is not prose: `print(Widget)` inside the block adds nothing. + assert [rc for rc in calls if rc["source_location"] == "L10"] == [] + # One raw call per (owner, mention), however often the heading repeats it. + assert len([rc for rc in calls if rc["callee"] == "Widget" + and rc["caller_nid"] == heading_ids["Rendering"]]) == 1 + + +def test_extract_markdown_heading_code_span_belongs_to_that_heading(tmp_path): + doc = tmp_path / "api.md" + doc.write_text("# API\n\n## `Widget`\n\nText.\n") + + r = extract_markdown(doc) + + (rc,) = r["raw_calls"] + heading = next(n for n in r["nodes"] if n["label"] == "`Widget`") + assert (rc["caller_nid"], rc["callee"], rc["path"] if "path" in rc else None) == ( + heading["id"], "Widget", None) + + +def test_mentions_resolve_to_references_edges_end_to_end(tmp_path): + r = _extract(tmp_path, { + "src/widget.py": _WIDGET_PY, + "src/gadget.py": _GADGET_PY, + "docs/guide.md": _GUIDE_MD, + }) + + refs = _references(r) + guide = _node(r, "Guide") + rendering = _node(r, "Rendering") + pinned = _node(r, "Pinned") + widget = _node(r, "Widget", "widget.py") + widget_render = next( + n for n in r["nodes"] if n["label"] == ".render()" and "widget" in n["id"]) + gadget_helper = _node(r, "helper()", "gadget.py") + + # Bare unique name: INFERRED reference from the citing heading. + assert refs[(guide["id"], widget["id"])]["confidence"] == "INFERRED" + assert refs[(guide["id"], widget["id"])]["confidence_score"] == 0.95 + assert (rendering["id"], widget["id"]) in refs + # Dotted `Widget.render()` resolves by its last segment: `render` is + # ambiguous (two classes define it), so no edge; same for `helper()`. + assert {t for (s, t) in refs if s == rendering["id"]} == {widget["id"]} + # Path-qualified mentions are EXTRACTED and scoped to the cited file. + assert refs[(pinned["id"], widget_render["id"])]["confidence"] == "EXTRACTED" + assert refs[(pinned["id"], widget_render["id"])]["confidence_score"] == 1.0 + assert refs[(pinned["id"], gadget_helper["id"])]["confidence"] == "EXTRACTED" + # `../src/widget.py::Widget` resolves relative to the document. + assert (pinned["id"], widget["id"]) in refs + # A cited file that is not in the corpus yields nothing. + assert {t for (s, t) in refs if s == pinned["id"]} == { + widget_render["id"], gadget_helper["id"], widget["id"]} + # Built-ins never resolve, and no mention becomes a call. + node_ids = {n["id"] for n in r["nodes"]} + assert all(s in node_ids and t in node_ids for (s, t) in refs) + doc_ids = {n["id"] for n in r["nodes"] if n.get("file_type") == "document"} + assert not [e for e in r["edges"] + if e["relation"] in ("calls", "indirect_call") and e["source"] in doc_ids] + + +def test_ambiguous_bare_name_yields_no_edge(tmp_path): + r = _extract(tmp_path, { + "a/thing.py": "class Thing:\n pass\n", + "b/thing.py": "class Thing:\n pass\n", + "notes.md": "# Notes\n\nUse `Thing`.\n", + }) + + notes = _node(r, "Notes") + assert {t for (s, t) in _references(r) if s == notes["id"]} == set() + + +def test_mentions_survive_the_extraction_cache(tmp_path): + files = {"src/widget.py": _WIDGET_PY, "docs/guide.md": _GUIDE_MD} + first = _extract(tmp_path, files) + second = _extract(tmp_path, files) + + def _pairs(r): + return {(s, t) for (s, t) in _references(r)} + + assert _pairs(first) == _pairs(second) + guide = _node(second, "Guide") + widget = _node(second, "Widget", "widget.py") + assert (guide["id"], widget["id"]) in _pairs(second)