mirror of
https://github.com/safishamsi/graphify.git
synced 2026-10-13 23:49:14 +00:00
fix(markdown): honour dotted qualifiers and explicit relative cites in mention resolution
Three findings from the automated review of #3562, all reproduced:
A dotted span with a non-code extension (README.md, pyproject.toml) was classified as a
symbol because only CODE_EXTENSIONS was checked, and every dotted form was reduced to its
last segment, so time.sleep matched a repo's own sleep(). The dotted form now keeps its
whole chain, the file early-exit also checks DOC_EXTENSIONS, and a bare-name match must
carry every qualifier as an owner label or a segment or stem of its source path.
_match_cited_file used lstrip("./"), which strips characters, so a .github/... cite could
never match by suffix. The leading ./ and ../ segments are now stripped as segments, and
the suffix form also matches a root-level file.
An explicit ./ or ../ cite that escaped the corpus fell through to the suffix rule and
could land on an unrelated copy of the file. It now resolves relative to the document
only.
(cherry picked from commit 3f84e7f3e2)
This commit is contained in:
1 parent
7dbc668933
commit
8cfebfdfd2
3 files changed
+143
-26
No files matched your search
@@ -6,7 +6,7 @@ import os
|
||||
import unicodedata
|
||||
|
||||
from pathlib import Path
|
||||
from graphify.detect import CODE_EXTENSIONS
|
||||
from graphify.detect import CODE_EXTENSIONS, DOC_EXTENSIONS
|
||||
from graphify.extractors.base import _file_stem, _make_id
|
||||
from graphify.security import sanitize_metadata
|
||||
|
||||
@@ -27,7 +27,8 @@ _MD_CODE_SPAN_RE = re.compile(r'(?<!`)(`+)(?!`)(.+?)(?<!`)\1(?!`)')
|
||||
# (``src/pkg/mod.py::Widget::render``) names the defining file and a
|
||||
# ``::``-separated symbol chain, exactly as pytest node ids and cite-lint
|
||||
# citations do. A bare mention (``Widget``, ``render()``, ``pkg.Widget``) names
|
||||
# a symbol with no file evidence, so it resolves only on a unique match.
|
||||
# a symbol with no file evidence, so it resolves only on a unique match; the
|
||||
# dotted form keeps its qualifiers (``pkg``) as evidence the resolver checks.
|
||||
_MD_PATH_MENTION_RE = re.compile(
|
||||
r'^([A-Za-z0-9_./\-]+\.[A-Za-z0-9]+)::([A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)(?:\(\))?$'
|
||||
)
|
||||
@@ -257,10 +258,12 @@ def _code_span_mention(span: str) -> "tuple[str | None, list[str]] | None":
|
||||
|
||||
Returns ``(path, names)``: ``path`` is the cited file for the
|
||||
``path::Name`` form and None for a bare or dotted mention; ``names`` is the
|
||||
symbol chain, outermost first. A trailing ``()`` is dropped from the last
|
||||
name so ``render()`` and ``render`` cite the same symbol. Spans that read
|
||||
as a file (``setup.py``), a shell command, an expression or prose are not
|
||||
mentions and yield None.
|
||||
symbol chain, outermost first, so ``pkg.sub.Widget`` yields
|
||||
``["pkg", "sub", "Widget"]`` and the resolver can hold the match to that
|
||||
evidence. A trailing ``()`` is dropped from the last name so ``render()``
|
||||
and ``render`` cite the same symbol. Spans that read as a file
|
||||
(``setup.py``, ``README.md``), a shell command, an expression or prose are
|
||||
not mentions and yield None.
|
||||
"""
|
||||
text = span.strip()
|
||||
if not text or " " in text:
|
||||
@@ -272,10 +275,14 @@ def _code_span_mention(span: str) -> "tuple[str | None, list[str]] | None":
|
||||
if not m:
|
||||
return None
|
||||
if "." in text:
|
||||
# ``a.b.Name`` is a qualified symbol; ``setup.py`` is a file. A dotted
|
||||
# span whose last segment is a code extension is the latter.
|
||||
if "." + m.group(1) in CODE_EXTENSIONS:
|
||||
# ``a.b.Name`` is a qualified symbol; ``setup.py`` and ``README.md``
|
||||
# are files. A dotted span whose last segment is a code or document
|
||||
# extension is the latter. Other file-like spans (``pyproject.toml``)
|
||||
# classify as a mention and are rejected at resolution, where the
|
||||
# qualifier ``pyproject`` matches no callable's file or owner.
|
||||
if "." + m.group(1) in CODE_EXTENSIONS or "." + m.group(1) in DOC_EXTENSIONS:
|
||||
return None
|
||||
return None, text.rstrip("()").split(".")
|
||||
return None, [m.group(1)]
|
||||
|
||||
|
||||
@@ -311,7 +318,9 @@ def extract_markdown(path: Path) -> dict:
|
||||
edges: the symbol they cite lives in another file, so the match is made
|
||||
once every file is extracted and ids are final, by the
|
||||
``markdown_mentions`` language resolver (see
|
||||
``graphify.markdown_resolution``). That pass emits
|
||||
``graphify.markdown_resolution``). The dotted form travels with its
|
||||
qualifiers, which the resolver checks against the match's owners and
|
||||
file path. That pass emits
|
||||
heading --references--> code symbol (page --references--> symbol for a
|
||||
mention above the first heading): EXTRACTED for the path-qualified form,
|
||||
INFERRED for a bare name that matches exactly one code symbol. The shared
|
||||
|
||||
@@ -25,8 +25,17 @@ to the code node that defines it:
|
||||
Only nodes the extractors marked ``_callable`` qualify, for the reason the
|
||||
indirect-call pass gives (#1566): a by-name match must land on a real
|
||||
function, method or class, never on a same-named data symbol such as a
|
||||
JSON key. No tie-breaking: an ambiguous name is a guess, and a guess is
|
||||
not an edge. The match is INFERRED (0.95, a named cross-file reference).
|
||||
JSON key. A dotted mention (``pkg.Widget``, ``Widget.render``) keeps its
|
||||
qualifiers, and a candidate survives only when every qualifier is a label
|
||||
on its ``contains`` / ``method`` owner chain or a segment (or stem) of its
|
||||
source path: ``time.sleep`` never lands on a repo's own ``sleep``, and
|
||||
``Widget.render`` picks the ``render`` that ``Widget`` owns. No other
|
||||
tie-breaking: an ambiguous name is a guess, and a guess is not an edge.
|
||||
The match is INFERRED (0.95, a named cross-file reference).
|
||||
|
||||
An explicit relative cite (``./`` or ``../``) resolves against the document's
|
||||
directory only; it never falls through to the suffix rule, so a path that
|
||||
escapes the corpus cannot land on an unrelated copy of the file elsewhere.
|
||||
|
||||
The shared cross-file call pass in ``extract`` skips ``markdown`` raw calls,
|
||||
so a mention never surfaces as a ``calls`` edge.
|
||||
@@ -34,6 +43,7 @@ so a mention never surfaces as a ``calls`` edge.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS
|
||||
@@ -42,6 +52,10 @@ MARKDOWN_MENTION_SUFFIXES = frozenset({".md", ".mdx", ".qmd", ".skill"})
|
||||
|
||||
_CONTAINMENT_RELATIONS = frozenset({"contains", "method"})
|
||||
|
||||
#: Leading ``./`` and ``../`` segments of a cited path, stripped as whole
|
||||
#: segments (``lstrip("./")`` would eat the dot of ``.github/...``).
|
||||
_RELATIVE_PREFIX_RE = re.compile(r"^(\.\.?/)+")
|
||||
|
||||
|
||||
def _symbol_label(node: dict[str, Any]) -> str:
|
||||
"""A node's label as a document would write it: no ``()``, no leading dot."""
|
||||
@@ -63,19 +77,56 @@ def _match_cited_file(cited: str, doc_file: str, source_files: set[str]) -> str
|
||||
|
||||
Tries the path relative to the citing document first (``../src/mod.py``
|
||||
from ``docs/guide.md``), then an exact match, then a unique
|
||||
segment-aligned suffix (``src/mod.py`` naming ``/repo/src/mod.py``).
|
||||
segment-aligned suffix (``src/mod.py`` naming ``/repo/src/mod.py``). A
|
||||
cite that is explicitly relative (``./x`` or ``../x``) stops after the
|
||||
first step: it names one location, and a suffix match elsewhere would be a
|
||||
different file.
|
||||
"""
|
||||
cited_posix = _posix(cited)
|
||||
doc_dir = os.path.dirname(doc_file)
|
||||
relative = _posix(os.path.normpath(os.path.join(doc_dir, cited)))
|
||||
for candidate in (relative, cited_posix):
|
||||
if candidate in source_files:
|
||||
return candidate
|
||||
suffix = "/" + cited_posix.lstrip("./")
|
||||
matches = [sf for sf in source_files if sf.endswith(suffix)]
|
||||
relative = _posix(os.path.normpath(os.path.join(doc_dir, cited_posix)))
|
||||
if relative in source_files:
|
||||
return relative
|
||||
if cited_posix.startswith(("./", "../")):
|
||||
return None
|
||||
if cited_posix in source_files:
|
||||
return cited_posix
|
||||
stripped = _RELATIVE_PREFIX_RE.sub("", cited_posix)
|
||||
matches = [sf for sf in source_files
|
||||
if sf == stripped or sf.endswith("/" + stripped)]
|
||||
return matches[0] if len(matches) == 1 else None
|
||||
|
||||
|
||||
def _evidence(node_id: str, nodes_by_id: dict[str, dict[str, Any]],
|
||||
parents: dict[str, set[str]]) -> set[str]:
|
||||
"""Labels a dotted mention may qualify ``node_id`` with.
|
||||
|
||||
The labels of every node on its ``contains`` / ``method`` owner chain
|
||||
(``Widget`` for ``Widget.render``) plus each segment and stem of its
|
||||
source path (``pkg`` and ``mod`` for ``pkg.mod.Widget``).
|
||||
"""
|
||||
evidence: set[str] = set()
|
||||
frontier = {node_id}
|
||||
seen: set[str] = set()
|
||||
while frontier:
|
||||
nid = frontier.pop()
|
||||
if nid in seen:
|
||||
continue
|
||||
seen.add(nid)
|
||||
owners = parents.get(nid, set())
|
||||
for owner in owners:
|
||||
node = nodes_by_id.get(owner)
|
||||
if node is not None:
|
||||
evidence.add(_symbol_label(node))
|
||||
frontier |= owners
|
||||
node = nodes_by_id.get(node_id, {})
|
||||
for segment in _posix(str(node.get("source_file", ""))).split("/"):
|
||||
if segment:
|
||||
evidence.add(segment)
|
||||
evidence.add(os.path.splitext(segment)[0])
|
||||
return evidence
|
||||
|
||||
|
||||
def _markdown_raw_calls(per_file: list[dict]) -> list[dict]:
|
||||
calls: list[dict] = []
|
||||
for result in per_file:
|
||||
@@ -118,7 +169,8 @@ def resolve_markdown_mentions(
|
||||
for e in all_edges:
|
||||
if e.get("relation") in _CONTAINMENT_RELATIONS:
|
||||
parents.setdefault(str(e.get("target")), set()).add(str(e.get("source")))
|
||||
node_ids = {n["id"] for n in all_nodes if n.get("id")}
|
||||
nodes_by_id = {n["id"]: n for n in all_nodes if n.get("id")}
|
||||
node_ids = set(nodes_by_id)
|
||||
existing = {
|
||||
(e.get("source"), e.get("target"))
|
||||
for e in all_edges if e.get("relation") == "references"
|
||||
@@ -156,7 +208,11 @@ def resolve_markdown_mentions(
|
||||
else:
|
||||
if callee in _LANGUAGE_BUILTIN_GLOBALS:
|
||||
continue
|
||||
candidates = by_label.get(callee, [])
|
||||
qualifiers = set(names[:-1])
|
||||
candidates = [
|
||||
c for c in by_label.get(callee, [])
|
||||
if qualifiers <= _evidence(c, nodes_by_id, parents)
|
||||
]
|
||||
target = candidates[0] if len(candidates) == 1 else None
|
||||
confidence, score = "INFERRED", 0.95
|
||||
if target is None or target == caller or (caller, target) in existing:
|
||||
|
||||
@@ -8,11 +8,13 @@ resolver matches them against the merged corpus after the id-remap passes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from graphify.extract import extract
|
||||
from graphify.extractors.markdown import _code_span_mention, extract_markdown
|
||||
from graphify.markdown_resolution import _match_cited_file
|
||||
|
||||
_WIDGET_PY = '''\
|
||||
class Widget:
|
||||
@@ -81,15 +83,42 @@ def _references(r):
|
||||
def test_code_span_mentions_are_classified():
|
||||
assert _code_span_mention("Widget") == (None, ["Widget"])
|
||||
assert _code_span_mention("render()") == (None, ["render"])
|
||||
assert _code_span_mention("pkg.sub.Widget") == (None, ["Widget"])
|
||||
assert _code_span_mention("pkg.sub.Widget") == (None, ["pkg", "sub", "Widget"])
|
||||
assert _code_span_mention("Widget.render()") == (None, ["Widget", "render"])
|
||||
# A file-like span with an unknown extension classifies as a mention and
|
||||
# is rejected at resolution, where `pyproject` matches no callable.
|
||||
assert _code_span_mention("pyproject.toml") == (None, ["pyproject", "toml"])
|
||||
pinned = ("src/widget.py", ["Widget", "render"])
|
||||
assert _code_span_mention("src/widget.py::Widget::render") == pinned
|
||||
assert _code_span_mention("src/widget.py::Widget::render()") == pinned
|
||||
# Files, commands, expressions and prose are not symbol mentions.
|
||||
for span in ("setup.py", "git revert", "x = 1", "a-b", "--flag", "", "src/widget.py"):
|
||||
for span in ("setup.py", "README.md", "notes.txt", "git revert", "x = 1", "a-b",
|
||||
"--flag", "", "src/widget.py"):
|
||||
assert _code_span_mention(span) is None, span
|
||||
|
||||
|
||||
def test_cited_file_matching():
|
||||
sources = {"src/mod.py", ".github/scripts/check.py", "lib/x.py", "vendor/lib/x.py",
|
||||
"docs/src/mod.py"}
|
||||
# Doc-relative first, then exact, then a unique segment-aligned suffix.
|
||||
assert _match_cited_file("src/mod.py", "docs/guide.md", sources) == "docs/src/mod.py"
|
||||
assert _match_cited_file("src/mod.py", "README.md", sources) == "src/mod.py"
|
||||
assert _match_cited_file("mod.py", "README.md", sources) is None
|
||||
assert _match_cited_file("scripts/check.py", "README.md", sources) == (
|
||||
".github/scripts/check.py")
|
||||
# `./` and `../` are stripped as segments: a hidden directory keeps its dot.
|
||||
assert _match_cited_file(".github/scripts/check.py", "docs/guide.md", sources) == (
|
||||
".github/scripts/check.py")
|
||||
assert _match_cited_file("./src/mod.py", "README.md", sources) == "src/mod.py"
|
||||
# The suffix form also names a root-level file.
|
||||
assert _match_cited_file("lib/x.py", "docs/guide.md", {"lib/x.py"}) == "lib/x.py"
|
||||
# An explicit relative cite resolves against the document only: escaping
|
||||
# the corpus never falls through to another copy of the file.
|
||||
assert _match_cited_file("../lib/x.py", "docs/guide.md", sources) == "lib/x.py"
|
||||
assert _match_cited_file("../lib/x.py", "README.md", sources) is None
|
||||
assert _match_cited_file("../../vendor/lib/x.py", "docs/guide.md", sources) is None
|
||||
|
||||
|
||||
def test_extract_markdown_reports_mentions_as_markdown_raw_calls(tmp_path):
|
||||
doc = tmp_path / "docs" / "guide.md"
|
||||
doc.parent.mkdir()
|
||||
@@ -152,9 +181,11 @@ def test_mentions_resolve_to_references_edges_end_to_end(tmp_path):
|
||||
assert refs[(guide["id"], widget["id"])]["confidence"] == "INFERRED"
|
||||
assert refs[(guide["id"], widget["id"])]["confidence_score"] == 0.95
|
||||
assert (rendering["id"], widget["id"]) in refs
|
||||
# Dotted `Widget.render()` resolves by its last segment: `render` is
|
||||
# ambiguous (two classes define it), so no edge; same for `helper()`.
|
||||
assert {t for (s, t) in refs if s == rendering["id"]} == {widget["id"]}
|
||||
# `render` is defined twice, but the qualifier in `Widget.render()` names
|
||||
# the owner, so the mention resolves; the bare `helper()` stays ambiguous.
|
||||
assert {t for (s, t) in refs if s == rendering["id"]} == {
|
||||
widget["id"], widget_render["id"]}
|
||||
assert refs[(rendering["id"], widget_render["id"])]["confidence"] == "INFERRED"
|
||||
# Path-qualified mentions are EXTRACTED and scoped to the cited file.
|
||||
assert refs[(pinned["id"], widget_render["id"])]["confidence"] == "EXTRACTED"
|
||||
assert refs[(pinned["id"], widget_render["id"])]["confidence_score"] == 1.0
|
||||
@@ -183,6 +214,27 @@ def test_ambiguous_bare_name_yields_no_edge(tmp_path):
|
||||
assert {t for (s, t) in _references(r) if s == notes["id"]} == set()
|
||||
|
||||
|
||||
def test_dotted_mentions_need_qualifier_evidence(tmp_path):
|
||||
r = _extract(tmp_path, {
|
||||
"src/widget.py": _WIDGET_PY + "\n\ndef sleep():\n pass\n",
|
||||
"src/gadget.py": _GADGET_PY + "\n\ndef toml():\n pass\n",
|
||||
"docs/notes.md": (
|
||||
"# Notes\n\n"
|
||||
"`time.sleep` and `pyproject.toml` are not this repo's sleep or toml.\n"
|
||||
"`widget.Widget`, `src.gadget.Gadget` and `Widget.render` are.\n"
|
||||
"`Gadget.helper` is not: `helper` is not owned by `Gadget`.\n"
|
||||
),
|
||||
})
|
||||
|
||||
notes = _node(r, "Notes")
|
||||
cited = {t for (s, t) in _references(r) if s == notes["id"]}
|
||||
widget = _node(r, "Widget", "widget.py")
|
||||
gadget = _node(r, "Gadget", "gadget.py")
|
||||
widget_render = next(
|
||||
n for n in r["nodes"] if n["label"] == ".render()" and "widget" in n["id"])
|
||||
assert cited == {widget["id"], gadget["id"], widget_render["id"]}
|
||||
|
||||
|
||||
def test_mentions_survive_the_extraction_cache(tmp_path):
|
||||
files = {"src/widget.py": _WIDGET_PY, "docs/guide.md": _GUIDE_MD}
|
||||
first = _extract(tmp_path, files)
|
||||
|
||||
Reference in new issue
Block a user