diff --git a/crates/zensical/src/config.rs b/crates/zensical/src/config.rs index e907d69..19b4e6e 100644 --- a/crates/zensical/src/config.rs +++ b/crates/zensical/src/config.rs @@ -26,8 +26,10 @@ //! Configuration. use fluent_uri::Uri; -use pyo3::types::PyAnyMethods; +use pyo3::exceptions::PyValueError; +use pyo3::types::{PyAnyMethods, PyDict}; use pyo3::{PyErr, Python}; +use regex::Regex; use std::fs; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::{Path, PathBuf}; @@ -78,6 +80,8 @@ pub struct Config { output_root: OutputRoot, /// Resolved Python Markdown extensions after compatibility shims. markdown_extensions: Arc<[String]>, + /// Compiled detector for Snippets markers and configured include tags. + snippet_pattern: Regex, /// Whether auto-references should be collected as backlinks. record_backlinks: bool, /// Configuration hash. @@ -93,6 +97,7 @@ impl Config { /// /// This method supports `mkdocs.yml`, as well as `zensical.toml` files. /// Right now, parsing is done in Python for compatibility with MkDocs. + #[allow(clippy::too_many_lines)] pub fn new

(path: P) -> Result where P: AsRef, @@ -107,7 +112,7 @@ impl Config { "configuration path must be valid UTF-8", ) })?; - let (project, markdown_extensions, record_backlinks) = + let (project, markdown_extensions, record_backlinks, include_syntax) = Python::attach(|py| { // Reset global data in compatibility modules py.import("zensical.extensions.autorefs")? @@ -138,6 +143,37 @@ impl Config { .extract::()? }; + // The resolved extension settings are used so plugin shims and + // direct extension configuration are handled identically. + let name = "zensical.extensions.include_markdown"; + let include = config + .get_item("mdx_configs")? + .call_method1("get", (name, PyDict::new(py)))?; + let include_syntax = if markdown_extensions + .iter() + .any(|extension| extension == name) + && include + .call_method1("get", ("enabled", true))? + .extract::()? + { + let opening = include + .call_method1("get", ("opening_tag", "{%"))? + .extract::()?; + let directives = include + .call_method1("get", ("directives", PyDict::new(py)))?; + let plain = directives + .call_method1("get", ("include", "include"))? + .extract::()?; + let markdown = directives + .call_method1( + "get", ("include-markdown", "include-markdown"), + )? + .extract::()?; + Some((opening, plain, markdown)) + } else { + None + }; + // Validate raw native plugin configuration before derived project // extraction can replace its precise diagnostic with generic // nested-field context from PyO3. @@ -156,9 +192,22 @@ impl Config { let project = config.extract::()?; // Return configuration and theme directory - Ok::<_, PyErr>((project, markdown_extensions, record_backlinks)) + Ok::<_, PyErr>(( + project, markdown_extensions, record_backlinks, + include_syntax, + )) })?; + let snippet_pattern = snippet_pattern( + include_syntax.as_ref().map(|(opening, plain, markdown)| { + (opening.as_str(), plain.as_str(), markdown.as_str()) + }), + ).map_err(|error| { + PyValueError::new_err(format!( + "invalid include-markdown detector: {error}" + )) + })?; + // Merge theme directories, giving precedence to custom directory over // the main theme directory to allow for overrides. let iter = project.theme_dirs.clone().into_iter(); @@ -191,6 +240,7 @@ impl Config { docs_root, output_root, markdown_extensions: markdown_extensions.into(), + snippet_pattern, record_backlinks, hash, }) @@ -203,6 +253,12 @@ impl Config { .any(|extension| extension == name) } + /// Returns whether Snippets markers or configured include tags are present. + pub fn has_snippets(&self, data: &str) -> bool { + self.snippet_pattern + .is_match(data.strip_prefix('\u{FEFF}').unwrap_or(data)) + } + /// Returns whether resolved auto-references should become backlinks. pub fn records_backlinks(&self) -> bool { self.record_backlinks @@ -284,6 +340,27 @@ where .replace('\\', "/") } +// ---------------------------------------------------------------------------- +// Snippet detection +// ---------------------------------------------------------------------------- + +/// A detector is compiled from the effective include syntax, when enabled. +fn snippet_pattern( + syntax: Option<(&str, &str, &str)>, +) -> std::result::Result { + let include = syntax + .map(|(opening, plain, markdown)| { + format!( + r"|{}\s*(?:{}|{})\s", + regex::escape(opening), + regex::escape(plain), + regex::escape(markdown), + ) + }) + .unwrap_or_default(); + Regex::new(&format!(r"(?m)^[ \t]*(?:-+8<-+{include})")) +} + // ---------------------------------------------------------------------------- // Trait implementations // ---------------------------------------------------------------------------- @@ -298,3 +375,75 @@ impl Hash for Config { state.write_u64(self.hash); } } + +#[cfg(test)] +mod tests { + use super::snippet_pattern; + + #[test] + fn custom_syntax_is_matched_literally() { + let pattern = snippet_pattern(Some(("[(", "read+file", "read.md"))) + .unwrap(); + + for markdown in [ + "[( read+file \"shared.txt\" )]", + "[(\n read.md\n \"shared.md\"\n)]", + "--8<-- \"shared.md\"", + ] { + assert!(pattern.is_match(markdown), "{markdown:?}"); + } + + for markdown in [ + "{% include-markdown \"shared.md\" %}", + "[( readXmd \"shared.md\" )]", + "[( readfile \"shared.txt\" )]", + "[( read.md-more \"shared.md\" )]", + "# Ordinary page", + ] { + assert!(!pattern.is_match(markdown), "{markdown:?}"); + } + } + + #[test] + fn disabled_includes_do_not_disable_caching() { + let pattern = snippet_pattern(None).unwrap(); + + assert!(pattern.is_match("--8<-- \"shared.md\"")); + assert!(!pattern.is_match("{% include \"shared.md\" %}")); + assert!(!pattern.is_match("[( read.md \"shared.md\" )]")); + } + + #[test] + fn detects_snippet_markers_and_default_include_tags() { + let pattern = snippet_pattern(Some(("{%", "include", "include-markdown"))).unwrap(); + + for markdown in [ + "--8<-- \"shared.md\"", + " --8<--\nshared.md\n--8<--", + "{% include \"shared.txt\" %}", + "\t{% include-markdown 'shared.md' %}", + "{%\n include-markdown\n \"shared.md\"\n%}", + "```text\n{% include \"shared.txt\" %}\n```", + "\u{FEFF}{% include-markdown \"shared.md\" %}", + ] { + assert!(pattern.is_match(markdown.strip_prefix('\u{FEFF}').unwrap_or(markdown)), "{markdown:?}"); + } + } + + #[test] + fn keeps_pages_without_include_markers_cacheable() { + let pattern = snippet_pattern(Some(("{%", "include", "include-markdown"))).unwrap(); + + for markdown in [ + "# Ordinary page\n\nContent.", + ";--8<-- \"shared.md\"", + "{% if enabled %}\nContent\n{% endif %}", + "{% include-other \"shared.md\" %}", + "{! include \"shared.md\" !}", + "The include-markdown plugin is enabled.", + ] { + assert!(!pattern.is_match(markdown.strip_prefix('\u{FEFF}').unwrap_or(markdown)), "{markdown:?}"); + } + } + +} diff --git a/crates/zensical/src/lib.rs b/crates/zensical/src/lib.rs index db0123f..6d48aba 100644 --- a/crates/zensical/src/lib.rs +++ b/crates/zensical/src/lib.rs @@ -61,9 +61,7 @@ use config::Config; use path::SourcePath; use server::{create_server, ServeOptions}; use watcher::{Batch, Watcher}; -use workflow::{ - create_workflow, has_snippets, Configuration, Input, SourcesReady, -}; +use workflow::{create_workflow, Configuration, Input, SourcesReady}; // ---------------------------------------------------------------------------- // Enums @@ -312,8 +310,8 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult { })?; let mut dependents = BTreeMap::from_iter(dependents); - // Until snippet dependencies are tracked, rebuild every page - // containing a marker whenever a source changes in serve mode + // Until snippet dependencies are tracked, pages with Snippets + // markers or include tags are rebuilt when a source changes. if serve { dependents.extend(snippets.clone()); for change in &changes { @@ -332,7 +330,7 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult { && config .inclusion .is_included(path.as_str(), serve)? - && has_snippets(&source.read_to_string()?) + && config.has_snippets(&source.read_to_string()?) { snippets.insert(key.clone(), source.clone()); } diff --git a/crates/zensical/src/workflow.rs b/crates/zensical/src/workflow.rs index 6d368aa..1a60cc2 100644 --- a/crates/zensical/src/workflow.rs +++ b/crates/zensical/src/workflow.rs @@ -26,14 +26,13 @@ //! Workflow definitions. use percent_encoding::percent_decode_str; -use regex::Regex; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; use std::fs; use std::hash::{DefaultHasher, Hash, Hasher}; use std::ops::Deref; use std::str::FromStr; -use std::sync::{Arc, LazyLock, OnceLock}; +use std::sync::{Arc, OnceLock}; use zrx::id::matcher::Matcher; use zrx::id::Id; @@ -70,14 +69,6 @@ pub(crate) mod output; use cached::cached; -// ---------------------------------------------------------------------------- -// Constants -// ---------------------------------------------------------------------------- - -/// Regular expression to detect use of snippets -static SNIPPET_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?m)^[ \t]*-+8<-+").expect("invariant")); - // ---------------------------------------------------------------------------- // Structs // ---------------------------------------------------------------------------- @@ -713,11 +704,6 @@ fn read_documents( }) } -/// Returns whether Markdown contains a snippet marker. -pub fn has_snippets(data: &str) -> bool { - SNIPPET_RE.is_match(data.strip_prefix('\u{FEFF}').unwrap_or(data)) -} - /// Create a stream to process routed Markdown files. fn process_markdown( config: &Config, plugins: &plugin::Settings, @@ -737,10 +723,9 @@ fn process_markdown( let document = routed.document.clone(); let properties = routed.properties.clone(); let variables = routed.variables.clone(); - // Don't cache page if it inserts (pymdownx) snippets. - // This is a hack while waiting for CommonMark (AST) and components, - // as well as topic-based authoring functionality. - if has_snippets(&document.body) { + // Pages with Snippets markers or include tags are rendered without + // caching until their file dependencies are tracked. + if config.has_snippets(&document.body) { render_markdown( &config, origin, diff --git a/python/tests/integration/test_include_markdown.py b/python/tests/integration/test_include_markdown.py new file mode 100644 index 0000000..dfc0318 --- /dev/null +++ b/python/tests/integration/test_include_markdown.py @@ -0,0 +1,289 @@ +# Copyright (c) 2025-2026 Zensical and contributors + +# SPDX-License-Identifier: MIT +# All contributions are certified under the DCO + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from __future__ import annotations + +import subprocess +import sys +import time +from typing import TYPE_CHECKING, Any + +import pytest +import yaml +from markdown import Markdown + +import zensical + +if TYPE_CHECKING: + from pathlib import Path + + +def _write_config( + root: Path, options: dict[str, Any], configuration: str, **settings: Any +) -> Path: + """Configure includes through the plugin or the Markdown extension.""" + project = {"site_name": "Test", **settings} + if configuration == "plugin": + project["plugins"] = {"include-markdown": options} + else: + project["markdown_extensions"] = { + "zensical.extensions.include_markdown": options + } + if configuration == "override": + project["plugins"] = { + "include-markdown": { + "opening_tag": "{!", + "closing_tag": "!}", + "directives": {"include": "other"}, + } + } + config = root / "mkdocs.yml" + config.write_text(yaml.safe_dump(project), encoding="utf-8") + return config + + +@pytest.fixture( + params=[ + {}, + { + "opening_tag": "[(", + "closing_tag": ")]", + "directives": { + "include": "read+file", + "include-markdown": "read.md", + }, + }, + ], + ids=["default", "custom"], +) +def syntax(request: pytest.FixtureRequest) -> dict[str, Any]: + return request.param + + +@pytest.mark.parametrize("configuration", ["plugin", "extension", "override"]) +def test_rebuilds_included_content_and_caches_other_pages( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + syntax: dict[str, Any], + configuration: str, +) -> None: + docs = tmp_path / "docs" + docs.mkdir() + opening = syntax.get("opening_tag", "{%") + closing = syntax.get("closing_tag", "%}") + directive = syntax.get("directives", {}).get( + "include-markdown", "include-markdown" + ) + (docs / "index.md").write_text( + f'# Home\n\n{opening} {directive} "../shared.txt" {closing}\n', + encoding="utf-8", + ) + ordinary_markdown = "# Ordinary page\n\nContent without includes." + if syntax: + # The default syntax and similar names are inactive with custom tags. + ordinary_markdown += ( + '\n\n{% include-markdown "unused.md" %}' + '\n\n[( readXmd "unused.md" )]' + ) + (docs / "ordinary.md").write_text(ordinary_markdown, encoding="utf-8") + included = tmp_path / "shared.txt" + included.write_text("First **inclusion**", encoding="utf-8") + config = _write_config(tmp_path, syntax, configuration) + ordinary_renders = [] + convert = Markdown.convert + + def record_render(md: Markdown, source: str) -> str: + if source.strip() == ordinary_markdown: + ordinary_renders.append(source) + return convert(md, source) + + monkeypatch.setattr(Markdown, "convert", record_render) + + zensical.build(str(config), {"clean": False, "strict": False}) + + output = tmp_path / "site" / "index.html" + assert "First inclusion" in output.read_text( + encoding="utf-8" + ) + assert len(ordinary_renders) == 1 + + # Only the included file is changed; the page and configuration are kept. + included.write_text("Updated **inclusion**", encoding="utf-8") + + zensical.build(str(config), {"clean": False, "strict": False}) + + html = output.read_text(encoding="utf-8") + assert "Updated inclusion" in html + assert "First inclusion" not in html + assert len(ordinary_renders) == 1 + + +def test_serve_rebuilds_multiline_include_tags( + tmp_path: Path, syntax: dict[str, Any] +) -> None: + docs = tmp_path / "docs" + docs.mkdir() + included = docs / "shared.txt" + included.write_text("First inclusion", encoding="utf-8") + opening = syntax.get("opening_tag", "{%") + closing = syntax.get("closing_tag", "%}") + directive = syntax.get("directives", {}).get("include", "include") + (docs / "index.md").write_text( + f'{opening}\n {directive}\n "shared.txt"\n{closing}', encoding="utf-8" + ) + config = _write_config(tmp_path, syntax, "plugin", dev_addr="127.0.0.1:0") + output = tmp_path / "site" / "index.html" + + with (tmp_path / "serve.log").open("w+", encoding="utf-8") as log: + process = subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-m", + "zensical", + "serve", + "--config-file", + str(config), + ], + cwd=tmp_path, + stdout=log, + stderr=subprocess.STDOUT, + ) + + def wait_for_content(expected: str) -> None: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline and process.poll() is None: + if output.exists() and expected in output.read_text( + encoding="utf-8" + ): + return + time.sleep(0.02) + log.seek(0) + raise AssertionError( + f"Included content was not rebuilt: {log.read()}" + ) + + try: + wait_for_content("First inclusion") + + # A source change must refresh the retained including page. + included.write_text("Updated inclusion", encoding="utf-8") + + wait_for_content("Updated inclusion") + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +@pytest.mark.parametrize("configuration", ["plugin", "extension"]) +def test_disabled_includes_keep_pages_cached( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, configuration: str +) -> None: + docs = tmp_path / "docs" + docs.mkdir() + source = '{% include-markdown "missing.md" %}' + (docs / "index.md").write_text(source, encoding="utf-8") + config = _write_config(tmp_path, {"enabled": False}, configuration) + rendered = [] + convert = Markdown.convert + + def record_render(md: Markdown, content: str) -> str: + if content.strip() == source: + rendered.append(content) + return convert(md, content) + + monkeypatch.setattr(Markdown, "convert", record_render) + + zensical.build(str(config), {"clean": False, "strict": False}) + zensical.build(str(config), {"clean": False, "strict": False}) + + assert len(rendered) == 1 + assert source in (tmp_path / "site" / "index.html").read_text( + encoding="utf-8" + ) + + +def test_converted_site_matches_plain_snippets( + tmp_path: Path, syntax: dict[str, Any] +) -> None: + docs = tmp_path / "docs" + shared = docs / "shared" + shared.mkdir(parents=True) + opening = syntax.get("opening_tag", "{%") + closing = syntax.get("closing_tag", "%}") + directive = syntax.get("directives", {}).get("include", "include") + index = docs / "index.md" + outer = shared / "outer.txt" + inner = shared / "inner.txt" + index.write_text( + f"# Home\n\n{{{{ 1 + 2 }}}}\n\n {opening} {directive} " + f'"./shared/outer.txt" {closing}', + encoding="utf-8", + ) + outer.write_text( + f' {opening} {directive} "./inner.txt" {closing}', encoding="utf-8" + ) + inner.write_text( + " Included **content**\n\n {{ 2 + 3 }}\n", encoding="utf-8" + ) + options = {**syntax, "dedent": True, "preserve_includer_indent": False} + config = _write_config( + tmp_path, + options, + "plugin", + markdown_extensions=["zensical.extensions.macros"], + ) + + zensical.build(str(config), {"clean": False, "strict": False}) + + output = tmp_path / "site" / "index.html" + included_html = output.read_text(encoding="utf-8") + assert "Included content" in included_html + assert "{{ 2 + 3 }}" in included_html + + # Every tag, nested path, and mapped setting is converted before removal. + index.write_text( + '# Home\n\n{{ 1 + 2 }}\n\n--8<-- "shared/outer.txt:1"', encoding="utf-8" + ) + outer.write_text('--8<-- "shared/inner.txt:1"', encoding="utf-8") + project = { + "site_name": "Test", + "markdown_extensions": { + "zensical.extensions.macros": {}, + "pymdownx.snippets": { + "base_path": [str(docs)], + "restrict_base_path": False, + "check_paths": True, + "url_download": True, + "dedent_subsections": True, + }, + }, + } + config.write_text(yaml.safe_dump(project), encoding="utf-8") + + zensical.build(str(config), {"clean": False, "strict": False}) + + assert output.read_text(encoding="utf-8") == included_html diff --git a/python/tests/unit/extensions/test_include_markdown.py b/python/tests/unit/extensions/test_include_markdown.py new file mode 100644 index 0000000..53483d4 --- /dev/null +++ b/python/tests/unit/extensions/test_include_markdown.py @@ -0,0 +1,637 @@ +# Copyright (c) 2025-2026 Zensical and contributors + +# SPDX-License-Identifier: MIT +# All contributions are certified under the DCO + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest +import yaml +from markdown import Markdown +from pymdownx.snippets import SnippetMissingError, SnippetPreprocessor + +import zensical.config as config_module +from zensical.config import ConfigurationError, parse_config +from zensical.extensions.context import ContextExtension, Page +from zensical.extensions.include_markdown import IncludeMarkdownExtension + +if TYPE_CHECKING: + from pathlib import Path + + +def _markdown( + root: Path, + options: dict[str, Any] | None = None, + snippets: dict[str, Any] | None = None, + *, + page: str = "guides/index.md", +) -> Markdown: + """Configure the plugin and page context for a temporary project.""" + config = { + "root_dir": str(root), + "docs_dir": "docs", + "markdown_extensions": ["pymdownx.snippets", "pymdownx.superfences"], + "mdx_configs": {"pymdownx.snippets": snippets or {}}, + "plugins": {"include-markdown": {"config": options or {}}}, + } + config_module._shim_include_markdown(config) + + return Markdown( + extensions=[ + ContextExtension( + config=config, page=Page(path=page, url="guides/") + ), + *config["markdown_extensions"], + ], + extension_configs=config["mdx_configs"], + ) + + +def _native_markdown( + root: Path, settings: dict[str, Any] | None = None +) -> Markdown: + """Configure plain Snippets without the include-markdown adapter.""" + snippets = { + "base_path": [str(root / "docs")], + "restrict_base_path": False, + "check_paths": True, + "url_download": True, + **(settings or {}), + } + return Markdown( + extensions=["pymdownx.snippets", "pymdownx.superfences"], + extension_configs={"pymdownx.snippets": snippets}, + ) + + +@pytest.mark.parametrize("directive", ["include", "include-markdown"]) +@pytest.mark.parametrize( + "filename", + ["shared.md", "shared file.md", "shared.md:section", "both'\"quotes.md"], +) +@pytest.mark.parametrize( + ("options", "settings", "include_indent", "snippet_indent"), + [ + ({}, {}, "", ""), + ({}, {}, " ", " "), + ({"preserve_includer_indent": False}, {}, " ", ""), + ({"dedent": True}, {"dedent_subsections": True}, " ", " "), + ], +) +def test_include_tags_match_plain_snippets( + tmp_path: Path, + directive: str, + filename: str, + options: dict[str, Any], + settings: dict[str, Any], + include_indent: str, + snippet_indent: str, +) -> None: + # Paths, indentation, and mapped settings must survive + # conversion to Snippets. + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / filename).write_text( + " Included\n content\n\n", encoding="utf-8" + ) + md = _markdown(tmp_path, options) + native = _native_markdown(tmp_path, settings) + escaped = filename.replace('"', '\\"') + include_source = f'{include_indent}{{% {directive} "{escaped}" %}}' + snippet_source = f"--8<--\n{snippet_indent}{filename}:1\n--8<--" + + included_lines = md.preprocessors["snippet"].run(include_source.split("\n")) + snippet_lines = native.preprocessors["snippet"].run( + snippet_source.split("\n") + ) + + # Raw lines are compared so whitespace differences cannot be hidden by HTML. + assert included_lines == snippet_lines + assert md.convert(include_source) == native.convert(snippet_source) + + +@pytest.mark.parametrize( + "options", + [ + 'start="BEGIN" end="END"', + 'start="missing"', + 'end="missing"', + 'start="" end=""', + r'start=""', + "preserve-includer-indent=false dedent=true", + "encoding=ascii trailing-newlines=false recursive=false", + 'exclude="*.md" order="name"', + "rewrite-relative-urls=true comments=true heading-offset=2", + ], +) +def test_unsupported_options_match_plain_whole_file_snippets( + tmp_path: Path, options: str +) -> None: + (tmp_path / "docs").mkdir() + content = "# Heading\n\nbeforeBEGINcaféENDafter\n\n[Link](./target.md)\n\n" + (tmp_path / "docs" / "shared.md").write_text(content, encoding="utf-8") + md = _markdown(tmp_path) + native = _native_markdown(tmp_path) + include_source = f' {{% include "shared.md" {options} %}}' + snippet_source = "--8<--\n shared.md:1\n--8<--" + + included_lines = md.preprocessors["snippet"].run(include_source.split("\n")) + snippet_lines = native.preprocessors["snippet"].run( + snippet_source.split("\n") + ) + + # Unsupported arguments cannot introduce behavior beyond native Snippets. + assert included_lines == snippet_lines + assert md.convert(include_source) == native.convert(snippet_source) + + +@pytest.mark.parametrize("directive", ["include", "include-markdown"]) +@pytest.mark.parametrize("quote", ["'", '"']) +def test_includes_docs_relative_paths( + tmp_path: Path, directive: str, quote: str +) -> None: + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "shared file.md").write_text( + "**Included**", encoding="utf-8" + ) + md = _markdown(tmp_path) + + html = md.convert(f"{{% {directive} {quote}shared file.md{quote} %}}") + + assert html == "

Included

" + + +def test_nested_includes_resolve_from_each_source(tmp_path: Path) -> None: + (tmp_path / "docs" / "guides").mkdir(parents=True) + (tmp_path / "docs" / "shared").mkdir() + (tmp_path / "docs" / "shared" / "outer.md").write_text( + '{% include-markdown "./inner.md" %}', encoding="utf-8" + ) + (tmp_path / "docs" / "shared" / "inner.md").write_text( + "Nested **content**", encoding="utf-8" + ) + md = _markdown(tmp_path) + + html = md.convert('{% include-markdown "../shared/outer.md" %}') + + assert html == "

Nested content

" + + # Nested paths must be converted relative to Snippets' configured base path. + (tmp_path / "docs" / "shared" / "outer.md").write_text( + '--8<-- "shared/inner.md:1"', encoding="utf-8" + ) + native = _native_markdown(tmp_path) + + assert html == native.convert('--8<-- "shared/outer.md:1"') + + +def test_repeated_and_circular_includes(tmp_path: Path) -> None: + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "shared.md").write_text( + 'Content\n\n{% include "./shared.md" %}', encoding="utf-8" + ) + md = _markdown(tmp_path) + + html = md.convert('{% include "shared.md" %}\n\n{% include "shared.md" %}') + + assert html.count("Content") == 2 + assert "{%" not in html + + # The same recursion guard must apply after every tag has been converted. + (tmp_path / "docs" / "shared.md").write_text( + 'Content\n\n--8<-- "shared.md:1"', encoding="utf-8" + ) + native = _native_markdown(tmp_path) + + assert html == native.convert( + '--8<-- "shared.md:1"\n\n--8<-- "shared.md:1"' + ) + + +def test_multiline_tags_and_per_inclusion_options(tmp_path: Path) -> None: + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "shared.md").write_text("# Heading", encoding="utf-8") + md = _markdown(tmp_path) + + html = md.convert( + '{%\n include-markdown "shared.md"' + '\n heading-offset=2\n start="%} ignored"\n%}' + ) + + assert html == "

Heading

" + + +def test_trailing_whitespace_does_not_break_generated_snippets( + tmp_path: Path, +) -> None: + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "shared.md").write_text("Included", encoding="utf-8") + md = _markdown(tmp_path) + + html = md.convert('{% include "shared.md" %} \t\n\nAfter') + + assert html == "

Included

\n

After

" + + +def test_quoted_absolute_paths_and_global_encoding(tmp_path: Path) -> None: + (tmp_path / "docs").mkdir() + included = tmp_path / "both'\"quotes.txt" + included.write_text("café", encoding="latin-1") + md = _markdown(tmp_path, {"encoding": "latin-1"}) + escaped = str(included).replace('"', '\\"') + + html = md.convert(f'{{% include "{escaped}" %}}') + + assert html == "

café

" + + native = _native_markdown(tmp_path, {"encoding": "latin-1"}) + + assert html == native.convert(f"--8<--\n{included}:1\n--8<--") + + +def test_macros_and_file_expansion_match_plain_snippets(tmp_path: Path) -> None: + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "shared.md").write_text( + "{{ 2 + 3 }}", encoding="utf-8" + ) + md = _markdown(tmp_path) + md.registerExtensions(["zensical.extensions.macros"], {}) + native = _native_markdown(tmp_path) + native.registerExtensions(["zensical.extensions.macros"], {}) + + html = md.convert('{{ 1 + 2 }}\n\n{% include "shared.md" %}') + native_html = native.convert('{{ 1 + 2 }}\n\n--8<-- "shared.md:1"') + + # Macros are processed before file expansion, as with native Snippets. + assert html == native_html == "

3

\n

{{ 2 + 3 }}

" + + +@pytest.mark.parametrize("preserve", [True, False]) +def test_global_indentation_and_dedenting( + tmp_path: Path, preserve: bool +) -> None: + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "shared.md").write_text( + " included\n content", encoding="utf-8" + ) + md = _markdown( + tmp_path, {"dedent": True, "preserve_includer_indent": preserve} + ) + + html = md.convert(' {% include "shared.md" %}') + + assert ("
" in html) is preserve
+    assert "included\ncontent" in html
+
+
+def test_explicit_default_delimiters_and_directives(tmp_path: Path) -> None:
+    (tmp_path / "docs").mkdir()
+    (tmp_path / "docs" / "shared.md").write_text("Included", encoding="utf-8")
+    options = {
+        "opening_tag": "{%",
+        "closing_tag": "%}",
+        "directives": {
+            "include": "include",
+            "include-markdown": "include-markdown",
+        },
+    }
+    config = {"extra": {"polyfills": []}, "root_dir": str(tmp_path)}
+    plugin = config_module._convert_plugins(
+        {"include-markdown": options}, config
+    )["include-markdown"]["config"]
+    md = _markdown(tmp_path, plugin)
+
+    html = md.convert('{% include-markdown "shared.md" %}')
+
+    assert html == "

Included

" + + +@pytest.mark.parametrize( + "options", + [ + {"opening_tag": "{!"}, + {"closing_tag": "!}"}, + {"directives": {"include": "replace"}}, + {"directives": {"include-markdown": "include-md"}}, + { + "opening_tag": "[(", + "closing_tag": ")]", + "directives": { + "include": "read+file", + "include-markdown": "read.md", + }, + }, + ], +) +def test_custom_syntax_in_plugin_and_extension( + tmp_path: Path, options: dict[str, Any] +) -> None: + docs = tmp_path / "docs" + docs.mkdir() + included = docs / "shared.md" + included.write_text("Included", encoding="utf-8") + config = {"extra": {"polyfills": []}, "root_dir": str(tmp_path)} + plugin = config_module._convert_plugins( + {"include-markdown": options}, config + )["include-markdown"]["config"] + opening = options.get("opening_tag", "{%") + closing = options.get("closing_tag", "%}") + names = { + "include": "include", + "include-markdown": "include-markdown", + **options.get("directives", {}), + } + + md = _markdown(tmp_path, plugin) + + for directive in names.values(): + assert ( + md.convert(f'{opening}\n {directive} "shared.md"\n{closing}') + == "

Included

" + ) + + # The same options are also accepted without a plugin configuration. + md = Markdown( + extensions=[IncludeMarkdownExtension.name], + extension_configs={IncludeMarkdownExtension.name: options}, + ) + + assert ( + md.convert(f'{opening} {names["include"]} "{included}" {closing}') + == "

Included

" + ) + + +@pytest.mark.parametrize( + "options", + [{"opening_tag": ""}, {"closing_tag": ""}, {"directives": {"include": ""}}], +) +def test_rejects_empty_syntax_in_direct_extension_configuration( + tmp_path: Path, options: dict[str, Any] +) -> None: + (tmp_path / "docs").mkdir() + config_file = tmp_path / "mkdocs.yml" + config_file.write_text( + yaml.safe_dump( + { + "site_name": "Test", + "markdown_extensions": {IncludeMarkdownExtension.name: options}, + } + ), + encoding="utf-8", + ) + + with pytest.raises(ConfigurationError, match="include-markdown"): + parse_config(str(config_file)) + + +def test_includes_inside_fenced_code(tmp_path: Path) -> None: + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "example.txt").write_text("", encoding="utf-8") + md = _markdown(tmp_path) + + html = md.convert('```text\n{% include "example.txt" %}\n```') + + assert "<sample>" in html + assert "{%" not in html + + +def test_native_snippets_and_auto_append_run_once(tmp_path: Path) -> None: + docs = tmp_path / "docs" + docs.mkdir() + (docs / "shared.md").write_text("Included", encoding="utf-8") + (docs / "footer.md").write_text("Footer", encoding="utf-8") + md = _markdown(tmp_path, snippets={"auto_append": ["footer.md"]}) + + html = md.convert('--8<-- "shared.md"\n\n;--8<-- "escaped.md"') + + assert html.count("Included") == 1 + assert html.count("Footer") == 1 + assert '"escaped.md"' in html + + native = _native_markdown(tmp_path, {"auto_append": ["footer.md"]}) + + assert html == native.convert('--8<-- "shared.md"\n\n;--8<-- "escaped.md"') + + +def test_remote_nested_includes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + downloads = [] + outer = '{% include "./inner.md" %}' + + def download(_self: SnippetPreprocessor, url: str) -> list[str]: + downloads.append(url) + return [outer] if url.endswith("outer.md") else ["Remote content"] + + # Network access is replaced after Snippets initializes its download cache. + md = _markdown(tmp_path) + native = _native_markdown(tmp_path) + monkeypatch.setattr(SnippetPreprocessor, "download", download) + + html = md.convert('{% include "https://example.com/docs/outer.md" %}') + + assert html == "

Remote content

" + assert downloads == [ + "https://example.com/docs/outer.md", + "https://example.com/docs/inner.md", + ] + + # Native Snippets requires complete URLs in downloaded snippets. + outer = '--8<-- "https://example.com/docs/inner.md:1"' + downloads.clear() + + native_html = native.convert('--8<-- "https://example.com/docs/outer.md:1"') + + assert html == native_html + assert downloads == [ + "https://example.com/docs/outer.md", + "https://example.com/docs/inner.md", + ] + + +def test_explicit_snippets_dedenting_survives_adapter_removal( + tmp_path: Path, +) -> None: + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "shared.md").write_text( + " Included\n content", encoding="utf-8" + ) + md = _markdown( + tmp_path, {"dedent": True}, snippets={"dedent_subsections": False} + ) + native = _native_markdown(tmp_path, {"dedent_subsections": False}) + + html = md.convert('{% include "shared.md" %}') + native_html = native.convert('--8<-- "shared.md:1"') + + # Explicit native settings take precedence over mapped plugin defaults. + assert html == native_html + assert "
" in html
+
+
+@pytest.mark.parametrize("selection", ["2:3", "named"])
+def test_native_line_ranges_and_sections_remain_unchanged(
+    tmp_path: Path, selection: str
+) -> None:
+    (tmp_path / "docs").mkdir()
+    (tmp_path / "docs" / "shared.md").write_text(
+        "Before\n--8<-- [start:named]\nSelected\n--8<-- [end:named]\nAfter",
+        encoding="utf-8",
+    )
+    md = _markdown(tmp_path)
+    native = _native_markdown(tmp_path)
+    source = f'--8<-- "shared.md:{selection}"'
+
+    html = md.convert(source)
+    native_html = native.convert(source)
+
+    assert html == native_html == "

Selected

" + + +def test_missing_files_fail(tmp_path: Path) -> None: + md = _markdown(tmp_path) + + with pytest.raises(SnippetMissingError): + md.convert('{% include "missing.md" %}') + + +def test_remote_includes_can_be_disabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + md = _markdown(tmp_path, {"include_from_url": False}) + + def download(_self: SnippetPreprocessor, _url: str) -> list[str]: + pytest.fail("Downloads must remain disabled") + + monkeypatch.setattr(SnippetPreprocessor, "download", download) + + with pytest.raises(SnippetMissingError): + md.convert('{% include "https://example.com/file.md" %}') + + +@pytest.mark.parametrize( + "tag", + [ + "{% include filename %}", + '{% include "file.md" unknown=true %}', + 'Before {% include "file.md" %} after', + ], +) +def test_invalid_or_inline_tags_fail(tmp_path: Path, tag: str) -> None: + md = _markdown(tmp_path) + + with pytest.raises(ValueError, match="include-markdown"): + md.convert(tag) + + +def test_explicit_snippets_settings_take_precedence(tmp_path: Path) -> None: + (tmp_path / "docs").mkdir() + (tmp_path / "outside.md").write_text("Outside", encoding="utf-8") + md = _markdown(tmp_path, snippets={"restrict_base_path": True}) + + with pytest.raises(SnippetMissingError): + md.convert('{% include "../../outside.md" %}') + + +@pytest.mark.parametrize("config_format", ["toml", "yaml"]) +def test_plugin_shim_from_project_configuration( + tmp_path: Path, config_format: str +) -> None: + (tmp_path / "docs").mkdir() + if config_format == "toml": + config_file = tmp_path / "zensical.toml" + config_file.write_text( + '[project]\nsite_name = "Test"\n' + '[project.plugins.include-markdown]\nencoding = "ascii"\n', + encoding="utf-8", + ) + else: + config_file = tmp_path / "mkdocs.yml" + config_file.write_text( + "site_name: Test\nplugins:\n - include-markdown:" + "\n encoding: ascii\n", + encoding="utf-8", + ) + + config = parse_config(str(config_file)) + + assert IncludeMarkdownExtension.name in config["markdown_extensions"] + assert config["mdx_configs"]["pymdownx.snippets"]["encoding"] == "ascii" + + +def test_disabled_plugin_leaves_tags_untouched(tmp_path: Path) -> None: + md = _markdown(tmp_path, {"enabled": False}) + + html = md.convert('{% include "missing.md" %}') + + assert '{% include "missing.md" %}' in html + + +@pytest.mark.parametrize( + ("option", "value"), + [ + ("enabled", "yes"), + ("dedent", 1), + ("include_from_url", "true"), + ("preserve_includer_indent", None), + ("encoding", 42), + ("opening_tag", ""), + ("closing_tag", []), + ("directives", []), + ("directives", {"unknown": "name"}), + ("directives", {"include": ""}), + ("directives", {"include": "two words"}), + ], +) +def test_rejects_invalid_configuration(option: str, value: Any) -> None: + with pytest.raises(ConfigurationError, match="include-markdown"): + config_module._convert_plugins( + {"include-markdown": {option: value}}, + {"extra": {"polyfills": []}, "root_dir": "."}, + ) + + +@pytest.mark.parametrize( + "option", + [ + "trailing_newlines", + "comments", + "rewrite_relative_urls", + "heading_offset", + "order", + "start", + "end", + "exclude", + "cache", + "cache_dir", + "recursive", + ], +) +def test_unsupported_global_options_are_discarded(option: str) -> None: + config = {"extra": {"polyfills": []}, "root_dir": "."} + + plugins = config_module._convert_plugins( + {"include-markdown": {option: object()}}, config + ) + + assert plugins["include-markdown"]["config"] == {} diff --git a/python/tests/unit/test_plugin_config.py b/python/tests/unit/test_plugin_config.py index c668c0c..1582905 100644 --- a/python/tests/unit/test_plugin_config.py +++ b/python/tests/unit/test_plugin_config.py @@ -52,6 +52,7 @@ PYTHON_PLUGINS = ( "macros", "markdownextradata", "table-reader", + "include-markdown", ) SHIM_PLUGINS = ( @@ -64,6 +65,7 @@ SHIM_PLUGINS = ( "macros", "markdownextradata", "table-reader", + "include-markdown", ) diff --git a/python/zensical/config.py b/python/zensical/config.py index fdbe6e3..b2be8e4 100644 --- a/python/zensical/config.py +++ b/python/zensical/config.py @@ -49,6 +49,10 @@ from zensical.compat.mkdocstrings import _python_handler_has_backlinks from zensical.extensions.autorefs import AutorefsExtension from zensical.extensions.emoji import to_svg, twemoji from zensical.extensions.glightbox import GlightboxExtension +from zensical.extensions.include_markdown import ( + IncludeMarkdownConfig, + IncludeMarkdownExtension, +) from zensical.extensions.macros import MacrosExtension from zensical.extensions.markdownextradata import ( MarkdownExtraDataExtension, @@ -145,6 +149,19 @@ _PLUGIN_UNSUPPORTED_OPTIONS = { "background", "shadow", ), + "include-markdown": ( + "trailing_newlines", + "comments", + "rewrite_relative_urls", + "heading_offset", + "order", + "start", + "end", + "exclude", + "cache", + "cache_dir", + "recursive", + ), "literate-nav": (), "llmstxt": ("preprocess",), "macros": (), @@ -833,6 +850,7 @@ def _apply_defaults(config: dict, path: str) -> dict: _shim_macros(config) _shim_markdownextradata(config) _shim_table_reader(config) + _shim_include_markdown(config) # List files along with their hashes, so we can rebuild when they change watched_files = ( @@ -1207,6 +1225,49 @@ def _shim_table_reader(config: dict[str, Any]) -> None: # ---------------------------------------------------------------------------- +def _shim_include_markdown(config: dict[str, Any]) -> None: + """Map include-markdown 7.3.0 configuration to Snippets.""" + name = IncludeMarkdownExtension.name + extensions = config["markdown_extensions"] + if name not in extensions: + plugin = config["plugins"].get("include-markdown", {}).get("config") + if plugin is None or not plugin.get("enabled", True): + return + extensions.append(name) + config["mdx_configs"][name] = plugin + + options = config["mdx_configs"].get(name, {}) + if not options.get("enabled", True): + return + + # Direct extension options are validated before the Rust detector is built. + try: + IncludeMarkdownConfig( + **{key: value for key, value in options.items() if key != "enabled"} + ) + except (TypeError, ValueError) as error: + raise ConfigurationError(str(error)) from error + + snippets = "pymdownx.snippets" + if snippets not in extensions: + extensions.append(snippets) + settings = config["mdx_configs"].setdefault(snippets, {}) + defaults = { + "base_path": [str(Path(config["root_dir"]) / config["docs_dir"])], + "restrict_base_path": False, + "check_paths": True, + "encoding": options.get("encoding", "utf-8"), + "url_download": options.get("include_from_url", True), + "dedent_subsections": options.get("dedent", False), + } + for key, value in defaults.items(): + settings.setdefault(key, value) + + # The adapter is registered after Snippets so its configuration is reused. + extensions.remove(name) + extensions.append(name) + + def _ignore_directory(dirpath: str) -> bool: """Determine whether to ignore a folder based on its name.""" return dirpath == "__pycache__" or dirpath.startswith(".venv") @@ -2712,6 +2773,64 @@ def _convert_plugins(value: Any, config: dict) -> dict: "markdownextradata jinja_options must be a mapping" ) + if "include-markdown" in plugins: + include_markdown = plugins["include-markdown"] + _reject_unknown_options( + "include-markdown", + include_markdown, + { + "enabled", + "opening_tag", + "closing_tag", + "encoding", + "preserve_includer_indent", + "dedent", + "directives", + "include_from_url", + }, + ) + _validate_boolean_options( + "include-markdown", + include_markdown, + ( + "enabled", + "preserve_includer_indent", + "dedent", + "include_from_url", + ), + ) + _validate_string_options( + "include-markdown", + include_markdown, + ("opening_tag", "closing_tag", "encoding"), + ) + for option in ("opening_tag", "closing_tag", "encoding"): + if option in include_markdown and not include_markdown[option]: + raise ConfigurationError( + f"include-markdown {option} must not be empty" + ) + if "directives" in include_markdown: + directives = include_markdown["directives"] + if not isinstance(directives, dict): + raise ConfigurationError( + "include-markdown directives must be a mapping" + ) + _reject_unknown_options( + "include-markdown directives", + directives, + {"include", "include-markdown"}, + ) + if any( + not isinstance(value, str) + or not value + or any(character.isspace() for character in value) + for value in directives.values() + ): + raise ConfigurationError( + "include-markdown directives must contain " + "nonempty names without whitespace" + ) + if "table-reader" in plugins: table_reader = plugins["table-reader"] _reject_unknown_options( diff --git a/python/zensical/extensions/include_markdown.py b/python/zensical/extensions/include_markdown.py new file mode 100644 index 0000000..9f6118e --- /dev/null +++ b/python/zensical/extensions/include_markdown.py @@ -0,0 +1,290 @@ +# Copyright (c) 2025-2026 Zensical and contributors + +# SPDX-License-Identifier: MIT +# All contributions are certified under the DCO + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import urljoin + +from markdown import Extension +from markdown.preprocessors import Preprocessor +from pymdownx.snippets import SnippetExtension, SnippetPreprocessor + +from zensical.extensions.context import ContextPreprocessor + +if TYPE_CHECKING: + from markdown import Markdown + + +@dataclass +class IncludeMarkdownConfig: + """Configure the include-markdown tag adapter.""" + + opening_tag: str = "{%" + closing_tag: str = "%}" + encoding: str = "utf-8" + preserve_includer_indent: bool = True + dedent: bool = False + include_from_url: bool = True + directives: dict[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + for option in ("opening_tag", "closing_tag"): + value = getattr(self, option) + if not isinstance(value, str) or not value: + raise ValueError( + f"include-markdown {option} must be a nonempty string" + ) + if not isinstance(self.directives, dict): + raise TypeError("include-markdown directives must be a mapping") + for name, value in self.directives.items(): + if name not in {"include", "include-markdown"}: + raise ValueError(f"unknown include-markdown directive: {name}") + if ( + not isinstance(value, str) + or not value + or any(character.isspace() for character in value) + ): + raise ValueError( + "include-markdown directives must contain " + "nonempty names without whitespace" + ) + + +_QUOTED = r"""(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')""" +_OPTIONS = re.compile(rf"\s+(?P[a-z][a-z-]*)\s*=\s*(?:{_QUOTED}|[^\s]+)") +_OPTION_NAMES = { + "start", + "end", + "preserve-includer-indent", + "dedent", + "exclude", + "trailing-newlines", + "recursive", + "order", + "encoding", + "rewrite-relative-urls", + "comments", + "heading-offset", +} + + +class IncludeMarkdownPreprocessor(SnippetPreprocessor): + """Translate include tags before Snippets reads each document.""" + + def __init__( + self, + md: Markdown, + config: IncludeMarkdownConfig, + snippets: dict[str, Any], + ) -> None: + super().__init__(snippets, md) + self.md = md + self.config = config + names = { + "include": "include", + "include-markdown": "include-markdown", + **config.directives, + } + directives = "|".join( + re.escape(name) for name in sorted(set(names.values())) + ) + self.tags = re.compile( + rf"{re.escape(config.opening_tag)}\s*(?:{directives})(?=\s)" + rf"(?P(?:{_QUOTED}|(?!{re.escape(config.closing_tag)})[^\"'])*?)" + rf"{re.escape(config.closing_tag)}[ \t]*", + re.DOTALL, + ) + self.arguments = re.compile( + rf"\s*(?P{_QUOTED})(?P.*)", re.DOTALL + ) + + def translate_tags( + self, + lines: list[str], + file_name: str | None = None, + is_url: bool = False, # noqa: FBT002 + ) -> list[str]: + """Translate include tags into standalone native Snippets directives. + + Leave file reading and content processing to Snippets so converted + documents produce the same output after users remove this adapter. + """ + context = ContextPreprocessor.from_markdown(self.md) + project = context.config if context else {} + docs_dir = Path(project.get("root_dir", ".")) / project.get( + "docs_dir", "docs" + ) + page_path = context.page.path if context else "index.md" + source = file_name or str(docs_dir / page_path) + text = "\n".join(lines) + + def replace(match: re.Match[str]) -> str: + arguments = self.arguments.fullmatch(match["body"]) + if arguments is None: + raise ValueError("include-markdown requires a quoted file path") + + # Only known literal options are accepted; their values are ignored. + options = arguments["options"].rstrip() + position = 0 + while position < len(options): + option = _OPTIONS.match(options, position) + if option is None or option["name"] not in _OPTION_NAMES: + raise ValueError( + f"invalid include-markdown options: {options}" + ) + position = option.end() + + quoted = arguments["path"] + path = ( + quoted[1:-1] + .replace("\\" + quoted[0], quoted[0]) + .replace("\\\\", "\\") + ) + if not path or "\n" in path or "\r" in path: + raise ValueError( + "include-markdown requires a nonempty, " + "single-line file path" + ) + if is_url: + path = urljoin(source, path) + elif not path.lower().startswith(("https://", "http://")): + base = ( + Path(source).parent + if path.startswith(("./", "../")) + else docs_dir + ) + path = str((base / path).resolve()) + + # Each directive must be placed on its own line for Snippets. + line_start = text.rfind("\n", 0, match.start()) + 1 + line_end = text.find("\n", match.end()) + if line_end < 0: + line_end = len(text) + indent = text[line_start : match.start()] + if indent.strip() or text[match.end() : line_end].strip(): + raise ValueError( + "include-markdown tags must be on their own lines" + ) + if not self.config.preserve_includer_indent: + indent = "" + + # Quotes in paths are preserved by block notation. Dedenting is + # applied to the whole file, and colon suffixes are kept literal. + return f"--8<--\n{indent}{path}:1\n--8<--" + + return self.tags.sub(replace, text).split("\n") + + def parse_snippets( + self, + lines: list[str], + file_name: str | None = None, + is_url: bool = False, # noqa: FBT002 + is_section: bool = False, # noqa: FBT002 + ) -> list[str]: + """Translate nested tags and delegate file inclusion to Snippets.""" + return super().parse_snippets( + self.translate_tags(lines, file_name, is_url), + file_name, + is_url, + is_section, + ) + + +class IncludeMarkdownTagPreprocessor(Preprocessor): + """Translate page tags before macros without expanding included files.""" + + def __init__( + self, md: Markdown, snippets: IncludeMarkdownPreprocessor + ) -> None: + super().__init__(md) + self.snippets = snippets + + def run(self, lines: list[str]) -> list[str]: + """Translate page tags and leave file expansion to native Snippets.""" + return self.snippets.translate_tags(lines) + + +class IncludeMarkdownExtension(Extension): + """Adapt include-markdown 7.3.0 syntax to PyMdown Snippets.""" + + name = "zensical.extensions.include_markdown" + + def __init__(self, **kwargs: Any) -> None: + self._enabled = kwargs.pop("enabled", True) + self._config = IncludeMarkdownConfig(**kwargs) + super().__init__() + + def extendMarkdown(self, md: Markdown) -> None: + """Register a Snippets preprocessor with include-tag translation.""" + if not self._enabled: + return + snippets = next( + ( + extension + for extension in md.registeredExtensions + if isinstance(extension, SnippetExtension) + ), + None, + ) + if snippets is None: + snippets = SnippetExtension( + base_path=["."], + restrict_base_path=False, + check_paths=True, + encoding=self._config.encoding, + url_download=self._config.include_from_url, + dedent_subsections=self._config.dedent, + ) + self._processor = IncludeMarkdownPreprocessor( + md, self._config, snippets.getConfigs() + ) + md.registerExtension(self) + # PyMdown's "snippet" preprocessor is replaced with this subclass. + # Native snippet processing is retained, and include tags are translated + # during recursive inclusion. A second processing pass and duplicate + # auto_append content are avoided by reusing the name. Registration must + # follow Snippets. + # + # Page tags are translated before macros so include tags are not parsed + # as Jinja statements. File expansion is kept at Snippets' native + # priority (32), after macros, so conversion to native notation + # preserves the order of Markdown processing. + md.preprocessors.register( + IncludeMarkdownTagPreprocessor(md, self._processor), + "include_markdown", + 36, + ) + md.preprocessors.register(self._processor, "snippet", 32) + + def reset(self) -> None: + """Clear downloaded snippets between Markdown conversions.""" + self._processor.download.cache_clear() + + +def makeExtension(**kwargs: Any) -> IncludeMarkdownExtension: + """Create the include-markdown compatibility extension.""" + return IncludeMarkdownExtension(**kwargs)