diff --git a/graphify/extractors/markdown.py b/graphify/extractors/markdown.py index e1b24409..8c67d681 100644 --- a/graphify/extractors/markdown.py +++ b/graphify/extractors/markdown.py @@ -6,6 +6,7 @@ import os from pathlib import Path from graphify.extractors.base import _file_stem, _make_id +from graphify.security import sanitize_metadata _MD_INLINE_LINK_RE = re.compile(r'(?]+)>?(?:\s+[^)]*)?\)') @@ -16,6 +17,79 @@ _MD_WIKILINK_RE = re.compile(r'(? tuple[list[str], int]: + """Split leading YAML frontmatter off *lines*. + + Returns ``(frontmatter_lines, body_start_index)``. When the file has no + frontmatter — the common case — returns ``([], 0)`` and the caller parses + from line 0 exactly as before. + """ + if not lines or lines[0].strip() != "---": + return [], 0 + limit = min(len(lines), _MD_FRONTMATTER_MAX_LINES + 1) + for i in range(1, limit): + if lines[i].strip() in _MD_FRONTMATTER_CLOSE: + return lines[1:i], i + 1 + # Unterminated fence: treat the `---` as ordinary content, not frontmatter. + return [], 0 + + +def _parse_frontmatter(fm_lines: list[str]) -> dict: + """Parse frontmatter lines into a plain dict. + + Values are passed through ``sanitize_metadata`` by the caller, so nested + dicts and lists survive (a review workflow's ``coherence_check:`` block, + Obsidian ``aliases:``) while staying bounded and HTML-safe. + """ + if not fm_lines: + return {} + text = "\n".join(fm_lines) + try: + import yaml + except ImportError: + return _parse_frontmatter_fallback(fm_lines) + try: + data = yaml.safe_load(text) + except Exception: + # Malformed YAML in one document must not fail the whole extraction. + return _parse_frontmatter_fallback(fm_lines) + return data if isinstance(data, dict) else {} + + +def _parse_frontmatter_fallback(fm_lines: list[str]) -> dict: + """Flat `key: value` parser for when PyYAML is not installed. + + Nested blocks and list items are skipped rather than guessed at; the keys + that matter for graph filtering (``type``, ``review_status``, ``title``) + are flat scalars in practice. + """ + out: dict = {} + for raw in fm_lines: + if not raw[:1].strip(): + continue # indented -> belongs to a nested block, skip + m = _MD_FM_SCALAR_RE.match(raw.strip()) + if not m: + continue + key, value = m.group(1).strip(), m.group(2).strip() + if not value: + continue # a bare `key:` opens a nested block + out[key] = value.strip('"\'') + return out + def _resolve_markdown_link(raw: str, source_dir: Path) -> "Path | None": """Resolve a markdown link target to the absolute path of a sibling document. @@ -54,8 +128,16 @@ def extract_markdown(path: Path) -> dict: """Extract structural nodes and edges from a Markdown file. Produces nodes for: - - The file itself - - Each heading (# / ## / ### etc.) + - The file itself, tagged ``node_kind: "page"``, carrying any YAML + frontmatter under ``frontmatter`` + - Each heading (# / ## / ### etc.), tagged ``node_kind: "heading"`` + + ``node_kind`` exists because ``file_type`` cannot carry this distinction: + it is a closed enum (build.py rewrites anything outside + ``code|document|paper|image|rationale|concept`` to ``"concept"``) and + ``"document"`` on both endpoints is load-bearing for the twin-merge pass. + Without a separate field, headings — typically the large majority of nodes + in a docs-heavy corpus — cannot be filtered out by a consumer. Produces edges for: - file --contains--> heading @@ -74,6 +156,11 @@ def extract_markdown(path: Path) -> dict: them — they were always orphans (only a single contains edge to the parent doc) and inflated the disconnected-component count (#1077). + Leading YAML frontmatter is parsed onto the page node and excluded from + heading detection (a `#` there is a YAML comment). Links inside it are + still followed: review workflows keep wikilinks in frontmatter and those + are genuine references. + No tree-sitter dependency — pure line-by-line parsing. """ try: @@ -87,11 +174,16 @@ def extract_markdown(path: Path) -> dict: edges: list[dict] = [] seen_ids: set[str] = set() - def add_node(nid: str, label: str, line: int, file_type: str = "document") -> None: + def add_node(nid: str, label: str, line: int, file_type: str = "document", + node_kind: str = "heading", extra: "dict | None" = None) -> None: if nid not in seen_ids: seen_ids.add(nid) - nodes.append({"id": nid, "label": label, "file_type": file_type, - "source_file": str_path, "source_location": f"L{line}"}) + node = {"id": nid, "label": label, "file_type": file_type, + "node_kind": node_kind, + "source_file": str_path, "source_location": f"L{line}"} + if extra: + node.update(extra) + nodes.append(node) def add_edge(src: str, tgt: str, relation: str, line: int, confidence: str = "EXTRACTED", weight: float = 1.0, @@ -103,8 +195,13 @@ def extract_markdown(path: Path) -> dict: edge["target_file"] = target_file edges.append(edge) + lines = source.splitlines() + fm_lines, body_start = _split_frontmatter(lines) + frontmatter = sanitize_metadata(_parse_frontmatter(fm_lines)) + file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) + add_node(file_nid, path.name, 1, node_kind="page", + extra={"frontmatter": frontmatter} if frontmatter else None) source_dir = path.parent # Dedup link edges by resolved target node so a hub doc that links to the @@ -144,7 +241,6 @@ def extract_markdown(path: Path) -> dict: heading_stack: list[tuple[int, str]] = [] in_code_block = False - lines = source.splitlines() for line_num_0, line_text in enumerate(lines): line_num = line_num_0 + 1 @@ -169,6 +265,13 @@ def extract_markdown(path: Path) -> dict: if ref_def: add_link(ref_def.group(1), line_num) + # Inside the frontmatter block a leading `#` is a YAML comment, not an + # H1. Links above are still scanned there on purpose: review workflows + # put wikilinks in frontmatter (e.g. a `consulted:` list), and those are + # real references. Only heading detection is suppressed. + if line_num_0 < body_start: + continue + # Detect headings: # Heading, ## Heading, etc. heading_match = re.match(r'^(#{1,6})\s+(.+)', line_text) if heading_match: diff --git a/tests/test_languages.py b/tests/test_languages.py index add764b4..46928b80 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -3306,3 +3306,119 @@ def test_cl_import_edges_are_not_dangling(): # the import-target stubs are sourceless so the corpus rewire can collapse them stub_labels = {n["label"] for n in r["nodes"] if n.get("source_file") == ""} assert "cl" in stub_labels + + +# ── Markdown: node_kind + frontmatter ──────────────────────────────────────── + +def _md_extract(src: str): + """Write *src* to a temp .md file and extract it.""" + import tempfile, os + with tempfile.NamedTemporaryFile(suffix=".md", mode="w", delete=False) as fh: + fh.write(src) + fpath = fh.name + try: + return extract_markdown(Path(fpath)) + finally: + os.unlink(fpath) + + +def test_markdown_node_kind_separates_pages_from_headings(): + """Headings must be filterable. file_type is 'document' for both, so a + consumer needs node_kind to tell a page from a heading.""" + r = _md_extract("# Title\n\n## Section\n\n### Deeper\n") + kinds = [n["node_kind"] for n in r["nodes"]] + assert kinds.count("page") == 1, f"expected exactly one page node, got {kinds}" + assert kinds.count("heading") == 3, f"expected three heading nodes, got {kinds}" + # file_type stays 'document' on both — build.py's twin-merge depends on it. + assert {n["file_type"] for n in r["nodes"]} == {"document"} + + +def test_markdown_frontmatter_lands_on_page_node(): + r = _md_extract( + "---\n" + "title: Two Tier Model\n" + "type: decision\n" + "review_status: reviewed\n" + "---\n" + "\n" + "# Body Heading\n" + ) + page = [n for n in r["nodes"] if n["node_kind"] == "page"][0] + assert page["frontmatter"]["type"] == "decision" + assert page["frontmatter"]["review_status"] == "reviewed" + heading = [n for n in r["nodes"] if n["node_kind"] == "heading"][0] + assert "frontmatter" not in heading + + +def test_markdown_no_frontmatter_key_when_absent(): + """A plain document must not grow an empty frontmatter dict.""" + r = _md_extract("# Just A Heading\n") + page = [n for n in r["nodes"] if n["node_kind"] == "page"][0] + assert "frontmatter" not in page + + +def test_markdown_yaml_comment_is_not_a_heading(): + """`#` inside frontmatter is a YAML comment, not an H1.""" + r = _md_extract( + "---\n" + "# this is a yaml comment\n" + "title: Real Title\n" + "---\n" + "\n" + "# Actual Heading\n" + ) + labels = _labels(r) + assert not any("yaml comment" in l for l in labels), \ + f"YAML comment parsed as heading: {labels}" + assert any("Actual Heading" in l for l in labels) + + +def test_markdown_horizontal_rule_is_not_frontmatter(): + """A `---` that is not on line 1 is a horizontal rule.""" + r = _md_extract("# Title\n\n---\n\nsome text\n") + page = [n for n in r["nodes"] if n["node_kind"] == "page"][0] + assert "frontmatter" not in page + assert any("Title" in l for l in _labels(r)) + + +def test_markdown_unterminated_frontmatter_fence_is_content(): + """An opening `---` with no closing fence must not swallow the document.""" + r = _md_extract("---\ntitle: Dangling\n\n# Still A Heading\n") + assert any("Still A Heading" in l for l in _labels(r)) + + +def test_markdown_frontmatter_wikilinks_still_produce_edges(): + """Review workflows keep wikilinks in frontmatter (a `consulted:` list); + those are genuine references and must not be dropped.""" + import tempfile, os + d = tempfile.mkdtemp() + try: + (Path(d) / "target.md").write_text("# Target\n") + src = Path(d) / "source.md" + src.write_text("---\nconsulted: [[target]]\n---\n\n# Source\n") + r = extract_markdown(src) + refs = [e for e in r["edges"] if e["relation"] == "references"] + assert refs, "wikilink in frontmatter produced no reference edge" + finally: + import shutil; shutil.rmtree(d) + + +def test_markdown_nested_frontmatter_survives(): + """Nested blocks (a coherence_check: record) must not be flattened away.""" + r = _md_extract( + "---\n" + "title: Nested\n" + "coherence_check:\n" + " verdict: extends\n" + "---\n" + "\n" + "# Body\n" + ) + page = [n for n in r["nodes"] if n["node_kind"] == "page"][0] + assert page["frontmatter"]["coherence_check"]["verdict"] == "extends" + + +def test_markdown_malformed_frontmatter_does_not_raise(): + r = _md_extract("---\n: : : not valid yaml : :\n---\n\n# Body\n") + assert "error" not in r + assert any("Body" in l for l in _labels(r))