From 8dc4a80a4e9b1c41c2bc4e886a7121bc9ff529fa Mon Sep 17 00:00:00 2001 From: squidfunk Date: Tue, 1 Sep 2026 18:19:41 +0200 Subject: [PATCH] feature: add `meta` MkDocs plugin replacement Signed-off-by: squidfunk --- Cargo.lock | 87 ++++ Cargo.toml | 1 + crates/zensical/Cargo.toml | 1 + crates/zensical/src/compat/mkdocs/plugin.rs | 1 + .../zensical/src/compat/mkdocs/plugin/meta.rs | 467 ++++++++++++++++++ .../src/compat/mkdocs/plugin/meta/parser.rs | 224 +++++++++ .../src/compat/mkdocs/plugin/search.rs | 80 +-- crates/zensical/src/config/plugins.rs | 22 + crates/zensical/src/lib.rs | 188 ++++++- crates/zensical/src/structure/dynamic.rs | 73 ++- crates/zensical/src/structure/markdown.rs | 11 +- crates/zensical/src/structure/page.rs | 8 +- crates/zensical/src/workflow.rs | 137 +++-- python/tests/unit/test_config.py | 10 + python/zensical/config.py | 12 + python/zensical/markdown/render.py | 39 +- 16 files changed, 1231 insertions(+), 130 deletions(-) create mode 100644 crates/zensical/src/compat/mkdocs/plugin/meta.rs create mode 100644 crates/zensical/src/compat/mkdocs/plugin/meta/parser.rs diff --git a/Cargo.lock b/Cargo.lock index 24acb6b..4a153c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,6 +40,18 @@ dependencies = [ "yansi", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "base64" version = "0.22.1" @@ -173,6 +185,15 @@ dependencies = [ "syn", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "errno" version = "0.3.14" @@ -209,6 +230,12 @@ dependencies = [ "serde", ] +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -287,6 +314,24 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown", +] + [[package]] name = "heck" version = "0.5.0" @@ -644,6 +689,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "objc2" version = "0.6.4" @@ -698,6 +752,15 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "ordered-float" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c860fd3227ca4ac3cc032e2cd20cba3f02ccdf4b610538f8ee6584d56bb62e96" +dependencies = [ + "num-traits", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -936,6 +999,29 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "saphyr" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3767dfe8889ebb55a21409df2b6f36e66abfbe1eb92d64ff76ae799d3f91016" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink", + "ordered-float", + "saphyr-parser", +] + +[[package]] +name = "saphyr-parser" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb771b59f6b1985d1406325ec28f97cfb14256abcec4fdfb37b36a1766d6af7" +dependencies = [ + "arraydeque", + "hashlink", +] + [[package]] name = "semver" version = "1.0.28" @@ -1487,6 +1573,7 @@ dependencies = [ "percent-encoding", "pyo3", "regex", + "saphyr", "serde", "serde_json", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 832ad0f..b7bfd99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ minijinja-contrib = "2.19.0" notify = "8.2" percent-encoding = "2.3" regex = "1.12.3" +saphyr = "0.0.6" sha1_smol = "1.0" slab = "0.4.12" serde = "1.0" diff --git a/crates/zensical/Cargo.toml b/crates/zensical/Cargo.toml index ee45754..d7115a2 100644 --- a/crates/zensical/Cargo.toml +++ b/crates/zensical/Cargo.toml @@ -59,6 +59,7 @@ mio = { workspace = true, features = ["net", "os-poll"] } percent-encoding.workspace = true pyo3.workspace = true regex.workspace = true +saphyr.workspace = true serde = { workspace = true, features = ["derive", "rc"] } serde_json.workspace = true thiserror.workspace = true diff --git a/crates/zensical/src/compat/mkdocs/plugin.rs b/crates/zensical/src/compat/mkdocs/plugin.rs index b6b0003..7a0d333 100644 --- a/crates/zensical/src/compat/mkdocs/plugin.rs +++ b/crates/zensical/src/compat/mkdocs/plugin.rs @@ -13,6 +13,7 @@ use crate::config::Config; use crate::structure::markdown::Markdown; pub mod autorefs; +pub mod meta; pub mod mkdocstrings; pub mod search; diff --git a/crates/zensical/src/compat/mkdocs/plugin/meta.rs b/crates/zensical/src/compat/mkdocs/plugin/meta.rs new file mode 100644 index 0000000..0b2264f --- /dev/null +++ b/crates/zensical/src/compat/mkdocs/plugin/meta.rs @@ -0,0 +1,467 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +//! MkDocs Material metadata inheritance. + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::hash::Hash; +use std::ops::Range; +use std::path::Path; + +use crate::config::Config; +use crate::structure::dynamic::Dynamic; + +mod parser; + +// ---------------------------------------------------------------------------- +// Structs +// ---------------------------------------------------------------------------- + +/// Metadata plugin settings used by the workflow. +#[derive(Clone, Debug)] +pub(crate) struct Settings { + /// Whether inheritance is enabled. + pub enabled: bool, + /// Exact basename of metadata files. + pub meta_file: String, +} + +/// A source range expressed as UTF-8 byte offsets. +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct SourceSpan { + /// Source identifier. + pub source: String, + /// Half-open byte range within the complete source. + pub range: Range, +} + +/// Origin of one metadata value. +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) enum Origin { + /// Value read from a source document. + Source(SourceSpan), + /// Value created or changed during rendering. + Runtime, +} + +/// A source-aware metadata value. +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Node { + /// Origin of this node. + origin: Origin, + /// Value and source-aware children. + value: Value, +} + +/// Recursive metadata value. +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +enum Value { + /// Scalar value. + Scalar(Dynamic), + /// Sequence value. + List(Vec), + /// Mapping value. + Map(BTreeMap), +} + +/// One parsed YAML metadata document. +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Document { + /// Source-relative path. + path: String, + /// Source-aware root mapping. + root: Node, +} + +/// Metadata resolved for one Markdown page. +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Resolved { + /// Source-aware root mapping. + root: Node, +} + +/// Immutable metadata documents available to one workflow revision. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Index { + /// Parsed metadata files, shared by every page in the revision. + documents: Vec, +} + +// ---------------------------------------------------------------------------- +// Implementations +// ---------------------------------------------------------------------------- + +impl Settings { + /// Extracts native meta settings from resolved configuration. + pub(crate) fn new(config: &Config) -> Self { + let config = &config.project.plugins.meta.config; + Self { + enabled: config.enabled, + meta_file: config.meta_file.clone(), + } + } +} + +impl Node { + /// Creates a runtime-owned tree from a plain dynamic value. + fn runtime(value: Dynamic) -> Self { + let value = match value { + Dynamic::List(values) => { + Value::List(values.into_iter().map(Self::runtime).collect()) + } + Dynamic::Map(values) => Value::Map( + values + .into_iter() + .map(|(key, value)| (key, Self::runtime(value))) + .collect(), + ), + value => Value::Scalar(value), + }; + Self { origin: Origin::Runtime, value } + } + + /// Projects a source-aware tree into template-visible metadata. + fn dynamic(&self) -> Dynamic { + match &self.value { + Value::Scalar(value) => value.clone(), + Value::List(values) => { + Dynamic::List(values.iter().map(Self::dynamic).collect()) + } + Value::Map(values) => Dynamic::Map( + values + .iter() + .map(|(key, value)| (key.clone(), value.dynamic())) + .collect(), + ), + } + } + + /// Retains origins for values unchanged by Python extensions. + fn reconcile(&self, value: Dynamic) -> Self { + if self.dynamic() == value { + return self.clone(); + } + match (&self.value, value) { + (Value::Map(previous), Dynamic::Map(current)) => { + let values = current + .into_iter() + .map(|(key, value)| { + let value = if let Some(previous) = previous.get(&key) { + previous.reconcile(value) + } else { + Self::runtime(value) + }; + (key, value) + }) + .collect(); + Self { + origin: Origin::Runtime, + value: Value::Map(values), + } + } + (Value::List(previous), Dynamic::List(current)) => { + let values = current + .into_iter() + .enumerate() + .map(|(index, value)| { + if let Some(previous) = previous.get(index) { + previous.reconcile(value) + } else { + Self::runtime(value) + } + }) + .collect(); + Self { + origin: Origin::Runtime, + value: Value::List(values), + } + } + (_, value) => Self::runtime(value), + } + } +} + +impl Resolved { + /// Returns plain values for the Python Markdown boundary. + pub(crate) fn values(&self) -> BTreeMap { + let Value::Map(values) = &self.root.value else { + unreachable!("metadata root is always a mapping") + }; + values + .iter() + .map(|(key, value)| (key.clone(), value.dynamic())) + .collect() + } + + /// Reconciles source origins with metadata returned from Python. + pub(crate) fn reconcile(&self, values: BTreeMap) -> Self { + let root = self.root.reconcile(Dynamic::Map(values)); + Self { root } + } +} + +impl Index { + /// Loads and parses every configured metadata file exactly once. + pub(crate) fn load(docs: &Path, settings: &Settings) -> Result { + if !settings.enabled { + return Ok(Self::default()); + } + let mut documents = Vec::new(); + collect(docs, docs, settings, &mut documents)?; + Ok(Self { documents }) + } + + /// Resolves the metadata chain applicable to one page. + pub(crate) fn resolve( + &self, page: &str, front_matter: Option, + ) -> Result { + resolve( + self.documents + .iter() + .filter(|document| applies(&document.path, page)) + .cloned(), + front_matter, + ) + } +} + +// ---------------------------------------------------------------------------- +// Functions +// ---------------------------------------------------------------------------- + +/// Returns whether a source is claimed as a metadata file. +pub(crate) fn claims(path: &str, settings: &Settings) -> bool { + settings.enabled + && path.rsplit('/').next() == Some(settings.meta_file.as_str()) +} + +/// Returns whether a metadata file applies to a Markdown page. +pub(crate) fn applies(meta: &str, page: &str) -> bool { + let parent = meta.rsplit_once('/').map_or("", |(parent, _)| parent); + parent.is_empty() + || page + .strip_prefix(parent) + .is_some_and(|suffix| suffix.starts_with('/')) +} + +/// Parses one standalone metadata file. +pub(crate) fn parse(path: &str, source: &str) -> Result { + parser::parse(path, source, 0) + .with_context(|| format!("error reading meta file '{path}'")) +} + +/// Extracts and parses YAML front matter from a Markdown source. +pub(crate) fn front_matter( + path: &str, source: &str, +) -> Result<(String, Option)> { + parser::front_matter(path, source) + .with_context(|| format!("error reading page metadata '{path}'")) +} + +/// Recursively loads metadata documents from the docs tree. +fn collect( + root: &Path, directory: &Path, settings: &Settings, + documents: &mut Vec, +) -> Result<()> { + for entry in std::fs::read_dir(directory)? { + let path = entry?.path(); + if path.is_dir() { + collect(root, &path, settings, documents)?; + } else if path + .file_name() + .is_some_and(|name| name == settings.meta_file.as_str()) + { + let relative = path.strip_prefix(root)?; + let location = relative.to_string_lossy().replace('\\', "/"); + let source = std::fs::read_to_string(&path)?; + documents.push(parse(&location, &source)?); + } + } + Ok(()) +} + +/// Resolves applicable meta files and page front matter. +pub(crate) fn resolve( + documents: impl IntoIterator, page: Option, +) -> Result { + let mut documents = documents.into_iter().collect::>(); + documents.sort_by(|left, right| { + let left_depth = left.path.matches('/').count(); + let right_depth = right.path.matches('/').count(); + left_depth + .cmp(&right_depth) + .then(left.path.cmp(&right.path)) + }); + + let mut root = Node { + origin: Origin::Runtime, + value: Value::Map(BTreeMap::new()), + }; + for document in documents { + merge(&mut root, document.root.clone()).with_context(|| { + format!("error merging meta file '{}'", document.path) + })?; + } + if let Some(page) = page { + merge(&mut root, page.root).context("error merging page metadata")?; + } + Ok(Resolved { root }) +} + +/// Applies Material's typesafe-additive merge strategy. +fn merge(target: &mut Node, incoming: Node) -> Result<()> { + let incoming_origin = incoming.origin.clone(); + match (&mut target.value, incoming.value) { + (Value::Map(target), Value::Map(incoming)) => { + for (key, value) in incoming { + if let Some(current) = target.get_mut(&key) { + merge(current, value)?; + } else { + target.insert(key, value); + } + } + Ok(()) + } + (Value::List(target), Value::List(mut incoming)) => { + target.append(&mut incoming); + Ok(()) + } + (Value::Scalar(target_value), Value::Scalar(incoming)) + if scalar_kind(target_value) == scalar_kind(&incoming) => + { + *target_value = incoming; + target.origin = incoming_origin; + Ok(()) + } + _ => bail!( + "metadata types do not match ({} conflicts with {})", + origin_label(&target.origin), + origin_label(&incoming_origin) + ), + } +} + +/// Formats an origin for a concise merge diagnostic. +fn origin_label(origin: &Origin) -> String { + match origin { + Origin::Source(span) => { + format!("{}:{}..{}", span.source, span.range.start, span.range.end) + } + Origin::Runtime => "runtime metadata".into(), + } +} + +/// Returns an exact scalar kind for typesafe replacement. +fn scalar_kind(value: &Dynamic) -> u8 { + match value { + Dynamic::Null => 0, + Dynamic::String(_) => 1, + Dynamic::Bool(_) => 2, + Dynamic::Integer(_) => 3, + Dynamic::Float(_) => 4, + Dynamic::List(_) | Dynamic::Map(_) => unreachable!("nested value"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn values(source: &str) -> BTreeMap { + resolve([], Some(parse("docs/page.md", source).unwrap())) + .unwrap() + .values() + } + + #[test] + fn matches_path_components() { + assert!(applies("docs/guide/.meta.yml", "docs/guide/page.md")); + assert!(!applies("docs/guide/.meta.yml", "docs/guidelines/page.md")); + } + + #[test] + fn parses_null_as_null() { + assert_eq!(values("value: null\n")["value"], Dynamic::Null); + } + + #[test] + fn merges_maps_and_appends_lists() { + let root = parse("docs/.meta.yml", "x:\n a: 1\nitems: [a]\n").unwrap(); + let nested = + parse("docs/guide/.meta.yml", "x:\n b: 2\nitems: [b]\n").unwrap(); + let values = resolve([nested, root], None).unwrap().values(); + assert_eq!( + values["items"], + Dynamic::List(vec![ + Dynamic::String("a".into()), + Dynamic::String("b".into()) + ]) + ); + assert_eq!( + values["x"], + Dynamic::Map(BTreeMap::from([ + ("a".into(), Dynamic::Integer(1)), + ("b".into(), Dynamic::Integer(2)), + ])) + ); + } + + #[test] + fn rejects_type_mismatch() { + let root = parse("docs/.meta.yml", "value: text\n").unwrap(); + let page = parse("docs/page.md", "value: [text]\n").unwrap(); + let error = format!("{:#}", resolve([root], Some(page)).unwrap_err()); + assert!(error.contains(".meta.yml")); + assert!(error.contains("page.md")); + } + + #[test] + fn scalar_override_keeps_winning_source() { + let root = parse(".meta.yml", "title: Default\n").unwrap(); + let page = parse("page.md", "title: Page\n").unwrap(); + let resolved = resolve([root], Some(page)).unwrap(); + let Value::Map(values) = &resolved.root.value else { + panic!("mapping") + }; + let Origin::Source(span) = &values["title"].origin else { + panic!("source") + }; + assert_eq!(span.source, "page.md"); + } + + #[test] + fn loads_only_component_ancestors() { + let directory = tempfile::tempdir().unwrap(); + let docs = directory.path(); + std::fs::create_dir_all(docs.join("guide")).unwrap(); + std::fs::create_dir_all(docs.join("guidelines")).unwrap(); + std::fs::write(docs.join(".meta.yml"), "items: [root]\n").unwrap(); + std::fs::write(docs.join("guide/.meta.yml"), "items: [guide]\n") + .unwrap(); + std::fs::write( + docs.join("guidelines/.meta.yml"), + "items: [guidelines]\n", + ) + .unwrap(); + let settings = Settings { + enabled: true, + meta_file: ".meta.yml".into(), + }; + let values = Index::load(docs, &settings) + .unwrap() + .resolve("guide/page.md", None) + .unwrap() + .values(); + assert_eq!( + values["items"], + Dynamic::List(vec![ + Dynamic::String("root".into()), + Dynamic::String("guide".into()), + ]) + ); + } +} diff --git a/crates/zensical/src/compat/mkdocs/plugin/meta/parser.rs b/crates/zensical/src/compat/mkdocs/plugin/meta/parser.rs new file mode 100644 index 0000000..4197f37 --- /dev/null +++ b/crates/zensical/src/compat/mkdocs/plugin/meta/parser.rs @@ -0,0 +1,224 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +//! Source-aware YAML parsing for metadata. + +use anyhow::{bail, Result}; +use saphyr::{LoadableYamlNode, MarkedYaml, Scalar, YamlData}; +use std::collections::BTreeMap; + +use super::{Document, Node, Origin, SourceSpan, Value}; +use crate::structure::dynamic::Dynamic; + +/// Parses one YAML mapping and retains source ranges on every value. +pub(super) fn parse( + path: &str, source: &str, offset: usize, +) -> Result { + let (source, offset) = source + .strip_prefix('\u{FEFF}') + .map_or((source, offset), |source| (source, offset + 3)); + let documents = MarkedYaml::load_from_str(source)?; + if documents.len() != 1 { + bail!("metadata must contain exactly one YAML document") + } + let root = convert(path, source, offset, &documents[0])?; + match root.value { + Value::Map(_) => Ok(Document { path: path.into(), root }), + Value::Scalar(Dynamic::Null) if source.trim().is_empty() => { + Ok(Document { + path: path.into(), + root: Node { + origin: root.origin, + value: Value::Map(BTreeMap::new()), + }, + }) + } + _ => bail!("metadata root must be a mapping"), + } +} + +/// Extracts front matter using the same delimiters as Python Markdown. +pub(super) fn front_matter( + path: &str, source: &str, +) -> Result<(String, Option)> { + let (source, source_offset) = source + .strip_prefix('\u{FEFF}') + .map_or((source, 0), |source| (source, 3)); + let mut lines = source.split_inclusive('\n'); + let Some(first) = lines.next() else { + return Ok((source.into(), None)); + }; + if !is_delimiter(first, "---") { + return Ok((source.into(), None)); + } + + let yaml_start = first.len(); + let mut cursor = yaml_start; + for line in lines { + if is_delimiter(line, "---") || is_delimiter(line, "...") { + let yaml = &source[yaml_start..cursor]; + let body = source[cursor + line.len()..] + .trim_start_matches('\n') + .to_owned(); + return parse(path, yaml, yaml_start + source_offset) + .map(|document| (body, Some(document))); + } + cursor += line.len(); + } + Ok((source.into(), None)) +} + +/// Returns whether a complete line is a front-matter delimiter. +fn is_delimiter(line: &str, delimiter: &str) -> bool { + let line = line.strip_suffix('\n').unwrap_or(line); + let line = line.strip_suffix('\r').unwrap_or(line); + line.strip_prefix(delimiter) + .is_some_and(|suffix| suffix.chars().all(|ch| matches!(ch, ' ' | '\t'))) +} + +/// Converts one Saphyr node into an owned source-aware node. +fn convert( + path: &str, source: &str, offset: usize, node: &MarkedYaml<'_>, +) -> Result { + let origin = Origin::Source(SourceSpan { + source: path.into(), + range: marker_to_byte(source, node.span.start.index()) + offset + ..marker_to_byte(source, node.span.end.index()) + offset, + }); + let value = match &node.data { + YamlData::Value(value) => Value::Scalar(convert_scalar(value)), + YamlData::Sequence(values) => Value::List( + values + .iter() + .map(|value| convert(path, source, offset, value)) + .collect::>()?, + ), + YamlData::Mapping(values) => { + let mut explicit = BTreeMap::new(); + let mut inherited = BTreeMap::new(); + for (key, value) in values { + let key = string_key(key)?; + let value = convert(path, source, offset, value)?; + if key == "<<" { + merge_key(&mut inherited, value)?; + } else { + explicit.insert(key, value); + } + } + inherited.extend(explicit); + Value::Map(inherited) + } + YamlData::Tagged(_, _) => bail!("custom YAML tags are not supported"), + YamlData::Alias(_) => bail!("unresolved YAML alias"), + YamlData::BadValue => bail!("invalid YAML value"), + YamlData::Representation(_, _, _) => { + unreachable!("Saphyr performs early scalar parsing") + } + }; + Ok(Node { origin, value }) +} + +/// Converts a Saphyr scalar to the common metadata representation. +fn convert_scalar(value: &Scalar<'_>) -> Dynamic { + match value { + Scalar::Null => Dynamic::Null, + Scalar::Boolean(value) => Dynamic::Bool(*value), + Scalar::Integer(value) => Dynamic::Integer(*value), + Scalar::FloatingPoint(value) => Dynamic::from_float(value.into_inner()), + Scalar::String(value) => Dynamic::String(value.to_string()), + } +} + +/// Requires mapping keys to be strings. +fn string_key(node: &MarkedYaml<'_>) -> Result { + match &node.data { + YamlData::Value(Scalar::String(value)) => Ok(value.to_string()), + _ => bail!("metadata mapping keys must be strings"), + } +} + +/// Expands the YAML merge key while retaining the referenced node origins. +fn merge_key(target: &mut BTreeMap, node: Node) -> Result<()> { + match node.value { + Value::Map(values) => { + for (key, value) in values { + target.entry(key).or_insert(value); + } + Ok(()) + } + Value::List(values) => { + for value in values { + let Value::Map(values) = value.value else { + bail!("YAML merge sequence entries must be mappings") + }; + for (key, value) in values { + target.entry(key).or_insert(value); + } + } + Ok(()) + } + Value::Scalar(_) => { + bail!("YAML merge value must be a mapping or list of mappings") + } + } +} + +/// Converts Saphyr's character index to a UTF-8 byte offset. +fn marker_to_byte(source: &str, index: usize) -> usize { + if index == source.chars().count() { + source.len() + } else { + source + .char_indices() + .nth(index) + .map_or(source.len(), |(index, _)| index) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retains_unicode_byte_ranges() { + let document = parse("docs/.meta.yml", "title: Héllo\n", 0).unwrap(); + let Value::Map(values) = document.root.value else { + panic!("mapping") + }; + let Origin::Source(span) = &values["title"].origin else { + panic!("source") + }; + assert_eq!(&"title: Héllo\n"[span.range.clone()], "Héllo"); + } + + #[test] + fn extracts_front_matter_and_offsets_ranges() { + let source = "---\ntitle: Home\n---\n\n# Home\n"; + let (body, document) = front_matter("docs/index.md", source).unwrap(); + assert_eq!(body, "# Home\n"); + let document = document.unwrap(); + let Value::Map(values) = document.root.value else { + panic!("mapping") + }; + let Origin::Source(span) = &values["title"].origin else { + panic!("source") + }; + assert_eq!(&source[span.range.clone()], "Home"); + } + + #[test] + fn expands_alias_merge_keys() { + let source = "base: &base\n one: 1\nvalue:\n <<: *base\n two: 2\n"; + let document = parse("docs/.meta.yml", source, 0).unwrap(); + let Value::Map(root) = document.root.value else { + panic!("mapping") + }; + let Value::Map(value) = &root["value"].value else { + panic!("mapping") + }; + assert!(value.contains_key("one")); + assert!(value.contains_key("two")); + } +} diff --git a/crates/zensical/src/compat/mkdocs/plugin/search.rs b/crates/zensical/src/compat/mkdocs/plugin/search.rs index f6d1545..35f6c41 100644 --- a/crates/zensical/src/compat/mkdocs/plugin/search.rs +++ b/crates/zensical/src/compat/mkdocs/plugin/search.rs @@ -32,8 +32,7 @@ use std::io::{BufWriter, Write}; use std::sync::Arc; use zrx::id::Id; use zrx::scheduler::Value; -use zrx::stream::function::Collection; -use zrx::stream::{Key, Signal, Stream}; +use zrx::stream::{Key, Signal}; use crate::config::plugins::SearchPluginConfig; use crate::config::Config; @@ -89,6 +88,15 @@ pub(crate) struct Document { facts: Arc, } +/// Revision-aligned search inputs from the site settlement boundary. +#[derive(Clone, Debug)] +pub(crate) struct Snapshot { + /// Compact page documents. + documents: Arc, Document)>>, + /// Navigation from the same page revision. + nav: Navigation, +} + // ---------------------------------------------------------------------------- // Implementations // ---------------------------------------------------------------------------- @@ -119,6 +127,20 @@ impl Document { // ---------------------------------------------------------------------------- +impl Snapshot { + /// Creates a search snapshot without another site-wide reduction. + pub(crate) fn new( + documents: Vec<(Key, Document)>, nav: Navigation, + ) -> Self { + Self { + documents: Arc::new(documents), + nav, + } + } +} + +// ---------------------------------------------------------------------------- + impl Facts { /// Returns whether this page contributes anything to the search index. pub(crate) fn is_empty(&self) -> bool { @@ -190,51 +212,29 @@ impl SearchIndex { // ---------------------------------------------------------------------------- impl Value for Document {} +impl Value for Snapshot {} // ---------------------------------------------------------------------------- // Functions // ---------------------------------------------------------------------------- /// Attach MkDocs-compatible search artifact generation to the build graph. -pub(crate) fn attach( - config: &Config, documents: &Stream, - nav: &Signal, -) { - if !config.project.plugins.search.config.enabled { - let config = config.clone(); - let _ = nav.map(move |nav: &Navigation| { - let search = SearchIndex::new( - Vec::new(), - nav, - config.project.plugins.search.config.clone(), - &config.project.theme.language, - ); - write(&config, &search) - }); - return; - } - - let documents = - documents.reduce(|documents: &dyn Collection, Document>| { - Some( - documents - .iter() - .map(|(key, document)| (key.clone(), document.clone())) - .collect::>(), - ) - }); +pub(crate) fn attach(config: &Config, snapshot: &Signal) { let config = config.clone(); - let _ = documents.product(nav).map( - move |documents: &Vec<(Key, Document)>, nav: &Navigation| { - let search = SearchIndex::new( - documents.clone(), - nav, - config.project.plugins.search.config.clone(), - &config.project.theme.language, - ); - write(&config, &search) - }, - ); + let _ = snapshot.map(move |snapshot: &Snapshot| { + let documents = if config.project.plugins.search.config.enabled { + snapshot.documents.as_ref().clone() + } else { + Vec::new() + }; + let search = SearchIndex::new( + documents, + &snapshot.nav, + config.project.plugins.search.config.clone(), + &config.project.theme.language, + ); + write(&config, &search) + }); } /// Creates the page-local search visitor. diff --git a/crates/zensical/src/config/plugins.rs b/crates/zensical/src/config/plugins.rs index 22bce6b..9406b91 100644 --- a/crates/zensical/src/config/plugins.rs +++ b/crates/zensical/src/config/plugins.rs @@ -46,12 +46,34 @@ use serde::Serialize; pub struct Plugins { /// Search plugin. pub search: SearchPlugin, + /// Material meta plugin. + pub meta: MetaPlugin, /// Offline plugin. pub offline: OfflinePlugin, } // ---------------------------------------------------------------------------- +/// Material meta plugin. +#[derive(Clone, Debug, Hash, FromPyObject, Serialize)] +#[pyo3(from_item_all)] +pub struct MetaPlugin { + /// Plugin configuration. + pub config: MetaPluginConfig, +} + +/// Material meta plugin configuration. +#[derive(Clone, Debug, Hash, FromPyObject, Serialize)] +#[pyo3(from_item_all)] +pub struct MetaPluginConfig { + /// Whether metadata inheritance is enabled. + pub enabled: bool, + /// Name of metadata files inside the documentation tree. + pub meta_file: String, +} + +// ---------------------------------------------------------------------------- + /// Search plugin. #[derive(Clone, Debug, Hash, FromPyObject, Serialize)] #[pyo3(from_item_all)] diff --git a/crates/zensical/src/lib.rs b/crates/zensical/src/lib.rs index 859fd0d..f73d1fc 100644 --- a/crates/zensical/src/lib.rs +++ b/crates/zensical/src/lib.rs @@ -34,11 +34,14 @@ use crossbeam::channel::{unbounded, RecvTimeoutError}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; use pyo3::Python; +use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::process; +use std::sync::Arc; use std::time::{Duration, Instant}; use std::{fs, io, thread}; use zrx::id::Id; +use zrx::stream::{Change, Key}; mod compat; mod config; @@ -49,10 +52,11 @@ mod template; mod watcher; mod workflow; +use compat::mkdocs::plugin::meta; use config::Config; use server::{create_server, ServeOptions}; -use watcher::Watcher; -use workflow::create_workflow; +use watcher::{Source, Watcher}; +use workflow::{create_workflow, Input}; // ---------------------------------------------------------------------------- // Enums @@ -198,8 +202,9 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult { .runner() .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; let mut input = runner - .input::() + .input::() .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + let meta_settings = meta::Settings::new(&config); // Create channel for reload notifications let (sender, receiver) = unbounded(); @@ -237,13 +242,40 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult { loop { match watcher.receive(Duration::from_millis(100)) { Ok(changes) => { + let metadata = Arc::new( + meta::Index::load(&config.get_docs_dir(), &meta_settings) + .map_err(|error| { + PyRuntimeError::new_err(format!("{error:#}")) + })?, + ); + let dependents = metadata_dependents( + &changes, + &config.get_docs_dir(), + &meta_settings, + )?; let mut revision = input .begin() .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + for (key, source) in dependents { + revision + .insert(key, Input::new(source, metadata.clone())) + .map_err(|err| { + PyRuntimeError::new_err(err.to_string()) + })?; + } for change in changes { - revision.emit(change).map_err(|err| { - PyRuntimeError::new_err(err.to_string()) - })?; + match change { + Change::Insert(key, source) => revision + .insert(key, Input::new(source, metadata.clone())) + .map_err(|err| { + PyRuntimeError::new_err(err.to_string()) + })?, + Change::Remove(key) => { + revision.remove(key).map_err(|err| { + PyRuntimeError::new_err(err.to_string()) + })?; + } + } } input = revision .seal() @@ -282,6 +314,75 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult { Ok(false) } +/// Expands metadata-file changes into descendant Markdown updates. +fn metadata_dependents( + changes: &[Change], docs: &Path, settings: &meta::Settings, +) -> PyResult, Source)>> { + if !settings.enabled { + return Ok(Vec::new()); + } + let mut dependents = BTreeMap::new(); + for change in changes { + let key = match change { + Change::Insert(key, _) | Change::Remove(key) => key, + }; + let location = key[0].location(); + if !meta::claims(&location, settings) { + continue; + } + let parent = Path::new(location.as_ref()) + .parent() + .unwrap_or_else(|| Path::new("")); + let mut paths = Vec::new(); + collect_markdown(&docs.join(parent), &mut paths) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + for path in paths { + let relative = path + .strip_prefix(docs) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let location = relative.to_string_lossy().replace('\\', "/"); + let id = key[0] + .to_builder() + .location(location) + .build() + .expect("invariant"); + dependents.insert( + Key::from(id), + Source::from(path.to_string_lossy().into_owned()), + ); + } + } + + // A provider update for the page itself is authoritative. In particular, + // the initial snapshot contains both metadata files and every Markdown + // page, so retaining synthesized inserts here would admit each page twice. + for change in changes { + let key = match change { + Change::Insert(key, _) | Change::Remove(key) => key, + }; + dependents.remove(key); + } + Ok(dependents.into_iter().collect()) +} + +/// Recursively collects Markdown files below one metadata directory. +fn collect_markdown( + directory: &Path, paths: &mut Vec, +) -> io::Result<()> { + let Ok(entries) = fs::read_dir(directory) else { + return Ok(()); + }; + for entry in entries { + let path = entry?.path(); + if path.is_dir() { + collect_markdown(&path, paths)?; + } else if path.extension().is_some_and(|extension| extension == "md") { + paths.push(path); + } + } + Ok(()) +} + /// Returns the first action failure reported by one settled run. fn report_failures(run: &zrx::stream::Run) -> PyResult<()> { for invocation in run.report().invocations() { @@ -441,4 +542,79 @@ mod tests { assert!(dir.path().exists()); } + + #[test] + fn metadata_change_selects_only_descendant_markdown() { + let dir = tempdir().unwrap(); + let docs = dir.path(); + fs::create_dir_all(docs.join("guide/nested")).unwrap(); + fs::create_dir_all(docs.join("guidelines")).unwrap(); + fs::write(docs.join("guide/page.md"), "# Page").unwrap(); + fs::write(docs.join("guide/nested/page.md"), "# Nested").unwrap(); + fs::write(docs.join("guidelines/page.md"), "# Other").unwrap(); + + let id = Id::builder() + .provider("file") + .context("docs") + .location("guide/.meta.yml") + .build() + .unwrap(); + let changes = vec![Change::Insert( + Key::from(id), + Source::from(docs.join("guide/.meta.yml").display().to_string()), + )]; + let settings = meta::Settings { + enabled: true, + meta_file: ".meta.yml".into(), + }; + let dependents = + metadata_dependents(&changes, docs, &settings).unwrap(); + let locations = dependents + .iter() + .map(|(key, _)| key[0].location().into_owned()) + .collect::>(); + assert_eq!(locations, vec!["guide/nested/page.md", "guide/page.md"]); + } + + #[test] + fn provider_page_change_supersedes_metadata_dependent() { + let dir = tempdir().unwrap(); + let docs = dir.path(); + fs::create_dir_all(docs.join("guide")).unwrap(); + let page = docs.join("guide/page.md"); + fs::write(&page, "# Page").unwrap(); + + let meta_id = Id::builder() + .provider("file") + .context("docs") + .location("guide/.meta.yml") + .build() + .unwrap(); + let page_id = Id::builder() + .provider("file") + .context("docs") + .location("guide/page.md") + .build() + .unwrap(); + let changes = vec![ + Change::Insert( + Key::from(meta_id), + Source::from( + docs.join("guide/.meta.yml").display().to_string(), + ), + ), + Change::Insert( + Key::from(page_id), + Source::from(page.display().to_string()), + ), + ]; + let settings = meta::Settings { + enabled: true, + meta_file: ".meta.yml".into(), + }; + + assert!(metadata_dependents(&changes, docs, &settings) + .unwrap() + .is_empty()); + } } diff --git a/crates/zensical/src/structure/dynamic.rs b/crates/zensical/src/structure/dynamic.rs index 79f7412..7404c6a 100644 --- a/crates/zensical/src/structure/dynamic.rs +++ b/crates/zensical/src/structure/dynamic.rs @@ -25,7 +25,11 @@ //! Dynamic value. -use pyo3::FromPyObject; +use pyo3::exceptions::PyTypeError; +use pyo3::types::{ + PyAny, PyAnyMethods, PyBool, PyDict, PyFloat, PyInt, PyList, PyString, +}; +use pyo3::{Borrowed, FromPyObject, PyErr}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fmt; @@ -42,19 +46,15 @@ use float::Float; /// /// This data type represents any valid value that can be used as part of the /// metadata of a page and the extra data of configuration, supporting strings, -/// booleans, integers, floating point numbers, lists, and maps, so basically +/// nulls, booleans, integers, floating point numbers, lists, and maps, so +/// basically /// everything supported in YAML and TOML. /// -/// Null value are not supported, and currently represented as empty strings. -/// We're aiming to provide a type safe way to define custom namespaces in the -/// configuration, so we'll definitely revisit this as part of our efforts to -/// make configuration much more flexible. -#[derive( - Clone, Debug, FromPyObject, Hash, PartialEq, Eq, Serialize, Deserialize, -)] +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] #[serde(untagged)] -#[pyo3(from_item_all)] pub enum Dynamic { + /// Null value. + Null, /// String value. String(String), /// Boolean value. @@ -77,6 +77,7 @@ impl fmt::Display for Dynamic { /// Formats the dynamic value for display. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Dynamic::Null => write!(f, "null"), Dynamic::String(value) => write!(f, "{value}"), Dynamic::Bool(value) => write!(f, "{value}"), Dynamic::Integer(value) => write!(f, "{value}"), @@ -94,3 +95,55 @@ impl fmt::Display for Dynamic { } } } + +// ---------------------------------------------------------------------------- + +impl<'a, 'py> FromPyObject<'a, 'py> for Dynamic { + type Error = PyErr; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { + if obj.is_none() { + Ok(Self::Null) + } else if obj.is_instance_of::() { + obj.extract().map(Self::Bool) + } else if obj.is_instance_of::() { + obj.extract().map(Self::Integer) + } else if obj.is_instance_of::() { + obj.extract().map(|value| Self::Float(Float(value))) + } else if obj.is_instance_of::() { + obj.extract().map(Self::String) + } else if obj.is_instance_of::() { + obj.extract().map(Self::List) + } else if obj.is_instance_of::() { + obj.extract().map(Self::Map) + } else { + Err(PyTypeError::new_err("unsupported dynamic value")) + } + } +} + +// ---------------------------------------------------------------------------- + +impl Dynamic { + /// Creates a dynamic floating-point value. + pub(crate) fn from_float(value: f64) -> Self { + Self::Float(Float(value)) + } +} + +// ---------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn null_round_trips_through_json() { + let data = serde_json::to_string(&Dynamic::Null).unwrap(); + assert_eq!(data, "null"); + assert_eq!( + serde_json::from_str::(&data).unwrap(), + Dynamic::Null + ); + } +} diff --git a/crates/zensical/src/structure/markdown.rs b/crates/zensical/src/structure/markdown.rs index 38c2c81..e971c3f 100644 --- a/crates/zensical/src/structure/markdown.rs +++ b/crates/zensical/src/structure/markdown.rs @@ -89,12 +89,15 @@ struct RenderedMarkdown { impl Markdown { /// Renders Markdown using Python Markdown. #[cfg_attr(feature = "tracing", tracing::instrument(skip_all))] - pub fn new(id: &Id, url: String, content: String) -> Result { + pub fn new( + id: &Id, url: String, content: String, meta: BTreeMap, + ) -> Result { let id = id.clone(); + let meta = serde_json::to_string(&meta)?; let res = Python::attach(|py| { let module = py.import("zensical.markdown.render")?; module - .call_method1("render", (content, id.location(), url))? + .call_method1("render", (content, id.location(), url, meta))? .extract::() }) .map_err(|err| { @@ -168,7 +171,9 @@ impl Eq for Markdown {} /// We'll fix this in our modular navigation proposal that will make title /// handling much more flexible in the near future. fn extract_title(id: &Id, markdown: &MarkdownData) -> String { - if let Some(value) = markdown.meta.get("title") { + if let Some(value) = markdown.meta.get("title") + && !matches!(value, Dynamic::Null) + { return value.to_string(); } diff --git a/crates/zensical/src/structure/page.rs b/crates/zensical/src/structure/page.rs index 1ed2ba6..15e1bac 100644 --- a/crates/zensical/src/structure/page.rs +++ b/crates/zensical/src/structure/page.rs @@ -199,8 +199,10 @@ impl Page { pub fn render_template( &mut self, template: &Template, config: &Config, nav: Navigation, ) -> Result { - let name = self.meta.get("template").map(ToString::to_string); - let name = name.as_deref().unwrap_or("main.html"); + let name = match self.meta.get("template") { + Some(Dynamic::String(value)) => value.clone(), + _ => "main.html".into(), + }; // Compute page relations from the immutable navigation. self.ancestors = nav.ancestors(self); @@ -210,7 +212,7 @@ impl Page { // Add the page-local active overlay without cloning the navigation tree. let nav = NavigationView::new(nav, Some(&self.url)); let output = template.render_with_context( - name, + &name, context! { generator => GENERATOR, nav => TemplateValue::from_object(nav), diff --git a/crates/zensical/src/workflow.rs b/crates/zensical/src/workflow.rs index 0007da2..0966e8e 100644 --- a/crates/zensical/src/workflow.rs +++ b/crates/zensical/src/workflow.rs @@ -28,6 +28,7 @@ use regex::Regex; use serde::{Deserialize, Serialize}; use std::hash::{DefaultHasher, Hash, Hasher}; +use std::ops::Deref; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::{Arc, LazyLock, OnceLock}; @@ -40,7 +41,9 @@ use zrx::stream::{ concurrent, Key, Signal, Stream, StreamTupleExt, Value, Workflow, }; -use super::compat::mkdocs::plugin::{self, autorefs, mkdocstrings, search}; +use super::compat::mkdocs::plugin::{ + self, autorefs, meta, mkdocstrings, search, +}; use super::config::Config; use super::structure::markdown::Markdown; use super::structure::nav::Navigation; @@ -86,6 +89,32 @@ pub struct Main { strict: bool, } +/// File input enriched with immutable facts for the current revision. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Input { + /// Source supplied by the file provider. + source: Source, + /// Metadata files parsed once and shared by every page in the revision. + metadata: Arc, +} + +impl Value for Input {} + +impl Input { + /// Enriches one provider source with revision-local metadata facts. + pub(crate) fn new(source: Source, metadata: Arc) -> Self { + Self { source, metadata } + } +} + +impl Deref for Input { + type Target = Source; + + fn deref(&self) -> &Self::Target { + &self.source + } +} + /// Revision-settled site batch derived from the current page relation. #[derive(Clone, Debug)] struct Site { @@ -95,6 +124,8 @@ struct Site { nav: Navigation, /// Autoref registry derived from the same settled page snapshot. autorefs: autorefs::Registry, + /// Search inputs derived from the same settled page snapshot. + search: search::Snapshot, } impl Value for Site {} @@ -123,6 +154,8 @@ struct RenderedMarkdown { registrations: Arc, /// Facts extracted by the shared MkDocs-compatible HTML pass. html: plugin::HtmlFacts, + /// Resolved metadata and source mappings for later compatibility modules. + pub(crate) meta: Arc, } impl Value for RenderedMarkdown {} @@ -138,22 +171,26 @@ struct RenderedPage { registrations: Arc, /// HTML compatibility facts revision-aligned with the page. html: plugin::HtmlFacts, + /// Resolved metadata and source mappings for later compatibility modules. + pub(crate) meta: Arc, } impl Value for RenderedPage {} // ---------------------------------------------------------------------------- + // Implementations // ---------------------------------------------------------------------------- impl Main { /// Initializes the module. fn setup(&self, ctx: &mut Builder) { - let files = ctx.input::(); + let files = ctx.input::(); + let meta = meta::Settings::new(&self.config); // Set up workflow to process static assets and Markdown files. process_theme_assets(&self.config, &files); - process_assets(&self.config, &files); + process_assets(&self.config, &files, &meta); let rendered = process_markdown(&self.config, &files); // Cross the one global settlement boundary, derive all site-wide @@ -161,17 +198,10 @@ impl Main { let rendered_page = generate_page(&self.config, &rendered); let page = rendered_page.map(|rendered: &RenderedPage| rendered.page.clone()); - let document = rendered_page - .filter(|rendered: &RenderedPage| !rendered.html.search.is_empty()) - .map(|rendered: &RenderedPage| { - search::Document::new( - &rendered.page, - rendered.html.search.clone(), - ) - }); let site = generate_site(&self.config, &rendered_page); let nav = generate_nav(&site); - search::attach(&self.config, &document, &nav); + let search = site.map(|site: &Site| site.search.clone()); + search::attach(&self.config, &search); mkdocstrings::attach(&self.config, &nav); let _ = render_templates(&self.config, &files, &nav); let unresolved = render_pages(&self.config, &site); @@ -208,7 +238,7 @@ pub fn wait_for_markdown(config: &Config) -> (Key, Barrier) { /// Create a stream to collect references from all Markdown files. pub fn collect_references( - config: &Config, files: &Stream, + config: &Config, files: &Stream, ) -> Stream { let matcher = Arc::new( Matcher::from_str(&format!( @@ -221,7 +251,7 @@ pub fn collect_references( // Create pipeline to collect references files .filter(move |id: &Id| matcher.is_match(id).expect("invariant")) - .map(|source: &Source| { + .map(|source: &Input| { let references: References = fs::read_to_string(&*source.path)?.parse()?; Ok::<_, anyhow::Error>(SharedReferences::from(references)) @@ -230,7 +260,7 @@ pub fn collect_references( /// Validate references and autorefs after every current page has rendered. fn validate( - config: &Config, strict: bool, files: &Stream, + config: &Config, strict: bool, files: &Stream, pages: &Stream, unresolved: &Stream, ) { let validation = config.project.validation.clone(); @@ -272,7 +302,9 @@ fn page_hash(page: &Page, autorefs: &autorefs::References) -> u64 { } /// Create a stream to process static assets. -pub fn process_assets(config: &Config, files: &Stream) { +pub fn process_assets( + config: &Config, files: &Stream, meta: &meta::Settings, +) { let extra_templates = config.project.extra_templates.clone(); let docs_dir = config.project.docs_dir.clone(); let matcher = Arc::new( @@ -282,7 +314,8 @@ pub fn process_assets(config: &Config, files: &Stream) { // Create pipeline to copy static assets let site_dir = config.project.site_dir.clone(); let root_dir = config.get_root_dir(); - let _ = files.map(move |id: &Id, from: &Source| { + let meta = meta.clone(); + let _ = files.map(move |id: &Id, from: &Input| { if !matcher.is_match(id).expect("invariant") { return Ok(()); } @@ -292,6 +325,11 @@ pub fn process_assets(config: &Config, files: &Stream) { return Ok(()); } + // Metadata files are inputs, not site assets. + if meta::claims(&id.location(), &meta) { + return Ok(()); + } + // Don't copy template files that we render later if extra_templates.contains(&id.location().into_owned()) { return Ok(()); @@ -305,20 +343,20 @@ pub fn process_assets(config: &Config, files: &Stream) { // Compute parent path, create intermediate directories and copy files let to = root_dir.join(id.to_path()); fs::create_dir_all(to.parent().expect("invariant"))?; - copy_file(&**from, to)?; + copy_file(&from.path, to)?; Ok::<(), anyhow::Error>(()) }); } /// Create a stream to process static assets in theme. -pub fn process_theme_assets(config: &Config, files: &Stream) { +pub fn process_theme_assets(config: &Config, files: &Stream) { let matcher = Arc::new(Matcher::from_str("zrs::::templates/*::").expect("invariant")); // Create pipeline to copy static assets let site_dir = config.project.site_dir.clone(); let root_dir = config.get_root_dir(); - let _ = files.map(move |id: &Id, from: &Source| { + let _ = files.map(move |id: &Id, from: &Input| { if !matcher.is_match(id).expect("invariant") { return Ok(()); } @@ -336,7 +374,7 @@ pub fn process_theme_assets(config: &Config, files: &Stream) { // Compute parent path, create intermediate directories and copy files let to = root_dir.join(id.to_path()); fs::create_dir_all(to.parent().expect("invariant"))?; - copy_file(&**from, to)?; + copy_file(&from.path, to)?; Ok::<_, anyhow::Error>(()) }); } @@ -352,7 +390,7 @@ fn copy_file( /// Create a stream to process Markdown files. fn process_markdown( - config: &Config, files: &Stream, + config: &Config, files: &Stream, ) -> Stream { let matcher = Arc::new( Matcher::from_str(&format!( @@ -371,12 +409,12 @@ fn process_markdown( // disposal. Otherwise, just return that if the content did not change. // Note that we need to limit concurrency here, or we'll overwhelm the // Python interpreter with all tasks competing for the GIL. - .map(concurrent(1, move |id: &Id, path: &Source| { - let data = fs::read_to_string(&**path)?; + .map(concurrent(1, move |id: &Id, source: &Input| { + let location = id.location().into_owned(); + let data = fs::read_to_string(&*source.path)?; - // Remove Byte-Order-Mark (BOM) - let data = data.strip_prefix('\u{FEFF}').unwrap_or(&data); - let data = data.to_owned(); + let (data, page_meta) = meta::front_matter(&location, &data)?; + let resolved = source.metadata.resolve(&location, page_meta)?; // Compute URL using same logic as Page::new() let site_dir = config.project.site_dir.clone(); @@ -417,13 +455,21 @@ fn process_markdown( // This is a hack while waiting for CommonMark (AST) and components, // as well as topic-based authoring functionality. if SNIPPET_RE.is_match(&data) { - render_markdown(id, url, data, plugins) + render_markdown(id, url, data, plugins, resolved) } else { cached( &config, id.as_str(), - (config.hash, data.clone(), url.clone()), - |(_, data, url)| render_markdown(id, url, data, plugins), + ( + 1_u8, + config.hash, + data.clone(), + url.clone(), + resolved.clone(), + ), + |(_, _, data, url, resolved)| { + render_markdown(id, url, data, plugins, resolved) + }, ) } })) @@ -432,15 +478,22 @@ fn process_markdown( /// Render Markdown and collect the page-local facts produced alongside it. fn render_markdown( id: &Id, url: String, content: String, plugins: plugin::Settings, + meta: meta::Resolved, ) -> anyhow::Result { - let mut markdown = Markdown::new(id, url.clone(), content)?; + let mut markdown = Markdown::new(id, url.clone(), content, meta.values())?; let html = plugin::prepare(&mut markdown, plugins); let registrations = if plugins.autorefs { autorefs::take_page(&url) } else { Arc::default() }; - Ok(RenderedMarkdown { markdown, registrations, html }) + let meta = Arc::new(meta.reconcile(markdown.meta.clone())); + Ok(RenderedMarkdown { + markdown, + registrations, + html, + meta, + }) } /// Generate pages from Markdown files. @@ -452,6 +505,7 @@ fn generate_page( page: Page::new(&config, id, markdown.markdown.clone()), registrations: markdown.registrations.clone(), html: markdown.html.clone(), + meta: markdown.meta.clone(), }) } @@ -464,6 +518,7 @@ fn generate_site( let mut nav_pages = Vec::new(); let mut site_pages = Vec::new(); let mut facts = Vec::new(); + let mut documents = Vec::new(); for (key, rendered) in pages.iter() { nav_pages.push((key.clone(), rendered.page.clone())); site_pages.push(( @@ -474,14 +529,25 @@ fn generate_site( }, )); facts.push((key.clone(), rendered.registrations.clone())); + if !rendered.html.search.is_empty() { + documents.push(( + key.clone(), + search::Document::new( + &rendered.page, + rendered.html.search.clone(), + ), + )); + } } let nav = Navigation::new(config.project.nav.clone(), nav_pages); let autorefs = autorefs::assemble(&config, facts); + let search = search::Snapshot::new(documents, nav.clone()); Some(Site { pages: Arc::new(site_pages), nav, autorefs, + search, }) }) } @@ -493,7 +559,7 @@ fn generate_nav(site: &Signal) -> Signal { /// Render static and extra templates. pub fn render_templates( - config: &Config, files: &Stream, nav: &Signal, + config: &Config, files: &Stream, nav: &Signal, ) -> Stream { let docs_dir = config.project.docs_dir.clone(); @@ -527,8 +593,9 @@ pub fn render_templates( let config = config.clone(); templates .product(nav) - .map(move |template: &Source, nav: &Navigation| { - let name = Path::new(&**template).file_name().expect("invariant"); + .map(move |template: &Input, nav: &Navigation| { + let name = + Path::new(&template.path).file_name().expect("invariant"); let site_dir = config.get_site_dir(); // Render template and write to disk diff --git a/python/tests/unit/test_config.py b/python/tests/unit/test_config.py index a0fc4ad..7e501a4 100644 --- a/python/tests/unit/test_config.py +++ b/python/tests/unit/test_config.py @@ -171,6 +171,16 @@ class TestPluginShimming: config = self._parse_yaml(tmp_path, plugins={"glightbox": {}}) assert GlightboxExtension.name in config["markdown_extensions"] + def test_material_meta_plugin_is_normalized(self, tmp_path: Path) -> None: + config = self._parse_yaml( + tmp_path, + plugins={"material/meta": {"meta_file": "defaults.yml"}}, + ) + assert config["plugins"]["meta"]["config"] == { + "enabled": True, + "meta_file": "defaults.yml", + } + def test_mike_plugin_defaults_with_versioned_build( self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/python/zensical/config.py b/python/zensical/config.py index cf68b4c..65a6d68 100644 --- a/python/zensical/config.py +++ b/python/zensical/config.py @@ -1270,6 +1270,18 @@ def _convert_plugins(value: Any, config: dict) -> dict: search, "separator", '[\\s\\-_,:!=\\[\\]()\\\\"`/]+|\\.(?!\\d)', str ) + # Normalize Material's meta plugin to an identifier that can be extracted + # into the typed Rust configuration. Keep the original entry intact for + # compatibility with consumers of the MkDocs plugin mapping. + material_meta = plugins.get("material/meta") + if material_meta is None: + meta = {"enabled": False, "meta_file": ".meta.yml"} + else: + meta = dict(material_meta or {}) + set_default(meta, "enabled", True, bool) + set_default(meta, "meta_file", ".meta.yml", str) + plugins["meta"] = meta + # Define defaults for offline plugin offline = set_default(plugins, "offline", {"enabled": False}, dict) set_default(offline, "enabled", True, bool) diff --git a/python/zensical/markdown/render.py b/python/zensical/markdown/render.py index 97869a9..fe43c5d 100644 --- a/python/zensical/markdown/render.py +++ b/python/zensical/markdown/render.py @@ -23,39 +23,24 @@ from __future__ import annotations +import json import re from datetime import date, datetime from typing import Any -import yaml from markdown import Markdown -from yaml import SafeLoader from zensical.config import get_config from zensical.extensions.autorefs import set_autorefs_page from zensical.extensions.context import ContextExtension, Page from zensical.extensions.links import LinksExtension -# ---------------------------------------------------------------------------- -# Constants -# ---------------------------------------------------------------------------- - - -FRONT_MATTER_RE = re.compile( - r"^-{3}[ \r\t]*?\n(.*?\r?\n)(?:\.{3}|-{3})[ \r\t]*\n", - re.UNICODE | re.DOTALL, -) -""" -Regex pattern to extract front matter. -""" - - # ---------------------------------------------------------------------------- # Functions # ---------------------------------------------------------------------------- -def render(content: str, path: str, url: str) -> dict: +def render(content: str, path: str, url: str, metadata: str = "{}") -> dict: """Render Markdown and return HTML. This function returns rendered HTML as well as the table of contents and @@ -63,19 +48,10 @@ def render(content: str, path: str, url: str) -> dict: in order to support the specific syntax of Python Markdown. We're working on moving the entire rendering chain to Rust. """ - # First, extract metadata - the Python Markdown parser brings a metadata - # extension, but the implementation is broken, as it does not support full - # YAML syntax, e.g. lists. Thus, we just parse the metadata with YAML. - meta: dict = {} - if match := FRONT_MATTER_RE.match(content): - try: - meta = yaml.load(match.group(1), SafeLoader) - if isinstance(meta, dict): - content = content[match.end() :].lstrip("\n") - else: - meta = {} - except Exception: # noqa: BLE001 - pass + # Metadata inheritance and front matter are resolved in Rust before this + # boundary. JSON keeps the call explicit and avoids reconstructing Python + # objects one value at a time through the FFI. + meta: dict = json.loads(metadata) # Create page context and set it for autorefs. # We can stop setting the page if/when we vendor mkdocstrings. @@ -143,9 +119,6 @@ def render(content: str, path: str, url: str) -> dict: def _sanitize(value: Any) -> Any: - # We currently don't have a null value for metadata in the Rust runtime - if value is None: - return "" if isinstance(value, (date, datetime)): return value.isoformat() if isinstance(value, dict):