feat(markdown): tag page vs heading nodes and extract frontmatter

Adds an additive node_kind ("page"|"heading") attribute so a docs corpus can
distinguish the page node from its heading nodes, and parses leading YAML
frontmatter onto the page node via the bounded sanitize_metadata (values become
capped attributes, not graph nodes). Also fixes a leading YAML `#` comment in
frontmatter being extracted as an H1. Node ids/labels/file_type are unchanged,
so existing markdown graphs are not re-keyed.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

graphify/extractors/markdown.py

@@ -6,6 +6,7 @@ import os
 
 from pathlib import Path
 from graphify.extractors.base import _file_stem, _make_id
+from graphify.security import sanitize_metadata
 
 
 _MD_INLINE_LINK_RE = re.compile(r'(?<!\!)\[[^\]]*\]\(\s*<?([^)\s>]+)>?(?:\s+[^)]*)?\)')
@@ -16,6 +17,79 @@ _MD_WIKILINK_RE = re.compile(r'(?<!\!)\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]')
 
 _MD_LINKABLE_EXTS = {".md", ".mdx", ".qmd", ".markdown", ".rst", ".txt"}
 
+# A YAML frontmatter block is only frontmatter when the opening `---` is the
+# very first line of the file. A `---` further down is a horizontal rule and
+# must not be mistaken for one. Bounded so a file that opens a fence and never
+# closes it cannot swallow the whole document.
+_MD_FRONTMATTER_CLOSE = ("---", "...")
+_MD_FRONTMATTER_MAX_LINES = 200
+
+# Flat `key: value` fallback, used only when PyYAML is unavailable. PyYAML is
+# not a declared dependency of graphify (see manifest_ingest._parse_apm for the
+# same defensive-import pattern), so the extractor must degrade rather than
+# fail.
+_MD_FM_SCALAR_RE = re.compile(r'^([A-Za-z0-9_][A-Za-z0-9_\-. ]*):\s*(.*)$')
+
+
+def _split_frontmatter(lines: list[str]) -> tuple[list[str], int]:
+    """Split leading YAML frontmatter off *lines*.
+
+    Returns ``(frontmatter_lines, body_start_index)``. When the file has no
+    frontmatter — the common case — returns ``([], 0)`` and the caller parses
+    from line 0 exactly as before.
+    """
+    if not lines or lines[0].strip() != "---":
+        return [], 0
+    limit = min(len(lines), _MD_FRONTMATTER_MAX_LINES + 1)
+    for i in range(1, limit):
+        if lines[i].strip() in _MD_FRONTMATTER_CLOSE:
+            return lines[1:i], i + 1
+    # Unterminated fence: treat the `---` as ordinary content, not frontmatter.
+    return [], 0
+
+
+def _parse_frontmatter(fm_lines: list[str]) -> dict:
+    """Parse frontmatter lines into a plain dict.
+
+    Values are passed through ``sanitize_metadata`` by the caller, so nested
+    dicts and lists survive (a review workflow's ``coherence_check:`` block,
+    Obsidian ``aliases:``) while staying bounded and HTML-safe.
+    """
+    if not fm_lines:
+        return {}
+    text = "\n".join(fm_lines)
+    try:
+        import yaml
+    except ImportError:
+        return _parse_frontmatter_fallback(fm_lines)
+    try:
+        data = yaml.safe_load(text)
+    except Exception:
+        # Malformed YAML in one document must not fail the whole extraction.
+        return _parse_frontmatter_fallback(fm_lines)
+    return data if isinstance(data, dict) else {}
+
+
+def _parse_frontmatter_fallback(fm_lines: list[str]) -> dict:
+    """Flat `key: value` parser for when PyYAML is not installed.
+
+    Nested blocks and list items are skipped rather than guessed at; the keys
+    that matter for graph filtering (``type``, ``review_status``, ``title``)
+    are flat scalars in practice.
+    """
+    out: dict = {}
+    for raw in fm_lines:
+        if not raw[:1].strip():
+            continue  # indented -> belongs to a nested block, skip
+        m = _MD_FM_SCALAR_RE.match(raw.strip())
+        if not m:
+            continue
+        key, value = m.group(1).strip(), m.group(2).strip()
+        if not value:
+            continue  # a bare `key:` opens a nested block
+        out[key] = value.strip('"\'')
+    return out
+
 def _resolve_markdown_link(raw: str, source_dir: Path) -> "Path | None":
     """Resolve a markdown link target to the absolute path of a sibling document.
 
@@ -54,8 +128,16 @@ def extract_markdown(path: Path) -> dict:
     """Extract structural nodes and edges from a Markdown file.
 
     Produces nodes for:
-    - The file itself
-    - Each heading (# / ## / ### etc.)
+    - The file itself, tagged ``node_kind: "page"``, carrying any YAML
+      frontmatter under ``frontmatter``
+    - Each heading (# / ## / ### etc.), tagged ``node_kind: "heading"``
+
+    ``node_kind`` exists because ``file_type`` cannot carry this distinction:
+    it is a closed enum (build.py rewrites anything outside
+    ``code|document|paper|image|rationale|concept`` to ``"concept"``) and
+    ``"document"`` on both endpoints is load-bearing for the twin-merge pass.
+    Without a separate field, headings — typically the large majority of nodes
+    in a docs-heavy corpus — cannot be filtered out by a consumer.
 
     Produces edges for:
     - file --contains--> heading
@@ -74,6 +156,11 @@ def extract_markdown(path: Path) -> dict:
     them — they were always orphans (only a single contains edge to the
     parent doc) and inflated the disconnected-component count (#1077).
 
+    Leading YAML frontmatter is parsed onto the page node and excluded from
+    heading detection (a `#` there is a YAML comment). Links inside it are
+    still followed: review workflows keep wikilinks in frontmatter and those
+    are genuine references.
+
     No tree-sitter dependency — pure line-by-line parsing.
     """
     try:
@@ -87,11 +174,16 @@ def extract_markdown(path: Path) -> dict:
     edges: list[dict] = []
     seen_ids: set[str] = set()
 
-    def add_node(nid: str, label: str, line: int, file_type: str = "document") -> None:
+    def add_node(nid: str, label: str, line: int, file_type: str = "document",
+                 node_kind: str = "heading", extra: "dict | None" = None) -> None:
         if nid not in seen_ids:
             seen_ids.add(nid)
-            nodes.append({"id": nid, "label": label, "file_type": file_type,
-                          "source_file": str_path, "source_location": f"L{line}"})
+            node = {"id": nid, "label": label, "file_type": file_type,
+                    "node_kind": node_kind,
+                    "source_file": str_path, "source_location": f"L{line}"}
+            if extra:
+                node.update(extra)
+            nodes.append(node)
 
     def add_edge(src: str, tgt: str, relation: str, line: int,
                  confidence: str = "EXTRACTED", weight: float = 1.0,
@@ -103,8 +195,13 @@ def extract_markdown(path: Path) -> dict:
             edge["target_file"] = target_file
         edges.append(edge)
 
+    lines = source.splitlines()
+    fm_lines, body_start = _split_frontmatter(lines)
+    frontmatter = sanitize_metadata(_parse_frontmatter(fm_lines))
+
     file_nid = _make_id(str(path))
-    add_node(file_nid, path.name, 1)
+    add_node(file_nid, path.name, 1, node_kind="page",
+             extra={"frontmatter": frontmatter} if frontmatter else None)
 
     source_dir = path.parent
     # Dedup link edges by resolved target node so a hub doc that links to the
@@ -144,7 +241,6 @@ def extract_markdown(path: Path) -> dict:
     heading_stack: list[tuple[int, str]] = []
     in_code_block = False
 
-    lines = source.splitlines()
     for line_num_0, line_text in enumerate(lines):
         line_num = line_num_0 + 1
 
@@ -169,6 +265,13 @@ def extract_markdown(path: Path) -> dict:
         if ref_def:
             add_link(ref_def.group(1), line_num)
 
+        # Inside the frontmatter block a leading `#` is a YAML comment, not an
+        # H1. Links above are still scanned there on purpose: review workflows
+        # put wikilinks in frontmatter (e.g. a `consulted:` list), and those are
+        # real references. Only heading detection is suppressed.
+        if line_num_0 < body_start:
+            continue
+
         # Detect headings: # Heading, ## Heading, etc.
         heading_match = re.match(r'^(#{1,6})\s+(.+)', line_text)
         if heading_match:

tests/test_languages.py

@@ -3306,3 +3306,119 @@ def test_cl_import_edges_are_not_dangling():
     # the import-target stubs are sourceless so the corpus rewire can collapse them
     stub_labels = {n["label"] for n in r["nodes"] if n.get("source_file") == ""}
     assert "cl" in stub_labels
+
+
+# ── Markdown: node_kind + frontmatter ────────────────────────────────────────
+
+def _md_extract(src: str):
+    """Write *src* to a temp .md file and extract it."""
+    import tempfile, os
+    with tempfile.NamedTemporaryFile(suffix=".md", mode="w", delete=False) as fh:
+        fh.write(src)
+        fpath = fh.name
+    try:
+        return extract_markdown(Path(fpath))
+    finally:
+        os.unlink(fpath)
+
+
+def test_markdown_node_kind_separates_pages_from_headings():
+    """Headings must be filterable. file_type is 'document' for both, so a
+    consumer needs node_kind to tell a page from a heading."""
+    r = _md_extract("# Title\n\n## Section\n\n### Deeper\n")
+    kinds = [n["node_kind"] for n in r["nodes"]]
+    assert kinds.count("page") == 1, f"expected exactly one page node, got {kinds}"
+    assert kinds.count("heading") == 3, f"expected three heading nodes, got {kinds}"
+    # file_type stays 'document' on both — build.py's twin-merge depends on it.
+    assert {n["file_type"] for n in r["nodes"]} == {"document"}
+
+
+def test_markdown_frontmatter_lands_on_page_node():
+    r = _md_extract(
+        "---\n"
+        "title: Two Tier Model\n"
+        "type: decision\n"
+        "review_status: reviewed\n"
+        "---\n"
+        "\n"
+        "# Body Heading\n"
+    )
+    page = [n for n in r["nodes"] if n["node_kind"] == "page"][0]
+    assert page["frontmatter"]["type"] == "decision"
+    assert page["frontmatter"]["review_status"] == "reviewed"
+    heading = [n for n in r["nodes"] if n["node_kind"] == "heading"][0]
+    assert "frontmatter" not in heading
+
+
+def test_markdown_no_frontmatter_key_when_absent():
+    """A plain document must not grow an empty frontmatter dict."""
+    r = _md_extract("# Just A Heading\n")
+    page = [n for n in r["nodes"] if n["node_kind"] == "page"][0]
+    assert "frontmatter" not in page
+
+
+def test_markdown_yaml_comment_is_not_a_heading():
+    """`#` inside frontmatter is a YAML comment, not an H1."""
+    r = _md_extract(
+        "---\n"
+        "# this is a yaml comment\n"
+        "title: Real Title\n"
+        "---\n"
+        "\n"
+        "# Actual Heading\n"
+    )
+    labels = _labels(r)
+    assert not any("yaml comment" in l for l in labels), \
+        f"YAML comment parsed as heading: {labels}"
+    assert any("Actual Heading" in l for l in labels)
+
+
+def test_markdown_horizontal_rule_is_not_frontmatter():
+    """A `---` that is not on line 1 is a horizontal rule."""
+    r = _md_extract("# Title\n\n---\n\nsome text\n")
+    page = [n for n in r["nodes"] if n["node_kind"] == "page"][0]
+    assert "frontmatter" not in page
+    assert any("Title" in l for l in _labels(r))
+
+
+def test_markdown_unterminated_frontmatter_fence_is_content():
+    """An opening `---` with no closing fence must not swallow the document."""
+    r = _md_extract("---\ntitle: Dangling\n\n# Still A Heading\n")
+    assert any("Still A Heading" in l for l in _labels(r))
+
+
+def test_markdown_frontmatter_wikilinks_still_produce_edges():
+    """Review workflows keep wikilinks in frontmatter (a `consulted:` list);
+    those are genuine references and must not be dropped."""
+    import tempfile, os
+    d = tempfile.mkdtemp()
+    try:
+        (Path(d) / "target.md").write_text("# Target\n")
+        src = Path(d) / "source.md"
+        src.write_text("---\nconsulted: [[target]]\n---\n\n# Source\n")
+        r = extract_markdown(src)
+        refs = [e for e in r["edges"] if e["relation"] == "references"]
+        assert refs, "wikilink in frontmatter produced no reference edge"
+    finally:
+        import shutil; shutil.rmtree(d)
+
+
+def test_markdown_nested_frontmatter_survives():
+    """Nested blocks (a coherence_check: record) must not be flattened away."""
+    r = _md_extract(
+        "---\n"
+        "title: Nested\n"
+        "coherence_check:\n"
+        "  verdict: extends\n"
+        "---\n"
+        "\n"
+        "# Body\n"
+    )
+    page = [n for n in r["nodes"] if n["node_kind"] == "page"][0]
+    assert page["frontmatter"]["coherence_check"]["verdict"] == "extends"
+
+
+def test_markdown_malformed_frontmatter_does_not_raise():
+    r = _md_extract("---\n: : : not valid yaml : :\n---\n\n# Body\n")
+    assert "error" not in r
+    assert any("Body" in l for l in _labels(r))