lee/zensical
1
0
Fork 0
mirror of https://github.com/zensical/zensical.git synced 2026-09-24 23:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feature: add meta MkDocs plugin replacement

Browse Source 
Signed-off-by: squidfunk <martin.donath@squidfunk.com>
This commit is contained in:
squidfunk
2026-09-01 18:36:00 +02:00
parent 83b0fd7541
commit 8dc4a80a4e
16 changed files with 1231 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+87
View File
@@ -40,6 +40,18 @@ dependencies = [
"yansi",
]
[[package]]
name = "arraydeque"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236"
[[package]]
name = "autocfg"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "base64"
version = "0.22.1"
@@ -173,6 +185,15 @@ dependencies = [
"syn",
]
[[package]]
name = "encoding_rs"
version = "0.8.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
dependencies = [
"cfg-if",
]
[[package]]
name = "errno"
version = "0.3.14"
@@ -209,6 +230,12 @@ dependencies = [
"serde",
]
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "form_urlencoded"
version = "1.2.2"
@@ -287,6 +314,24 @@ dependencies = [
"regex-syntax",
]
[[package]]
name = "hashbrown"
version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"foldhash",
]
[[package]]
name = "hashlink"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
dependencies = [
"hashbrown",
]
[[package]]
name = "heck"
version = "0.5.0"
@@ -644,6 +689,15 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "num-traits"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
]
[[package]]
name = "objc2"
version = "0.6.4"
@@ -698,6 +752,15 @@ version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "ordered-float"
version = "5.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c860fd3227ca4ac3cc032e2cd20cba3f02ccdf4b610538f8ee6584d56bb62e96"
dependencies = [
"num-traits",
]
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -936,6 +999,29 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "saphyr"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3767dfe8889ebb55a21409df2b6f36e66abfbe1eb92d64ff76ae799d3f91016"
dependencies = [
"arraydeque",
"encoding_rs",
"hashlink",
"ordered-float",
"saphyr-parser",
]
[[package]]
name = "saphyr-parser"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fb771b59f6b1985d1406325ec28f97cfb14256abcec4fdfb37b36a1766d6af7"
dependencies = [
"arraydeque",
"hashlink",
]
[[package]]
name = "semver"
version = "1.0.28"
@@ -1487,6 +1573,7 @@ dependencies = [
"percent-encoding",
"pyo3",
"regex",
"saphyr",
"serde",
"serde_json",
"tempfile",
Cargo.toml
+1
View File
@@ -61,6 +61,7 @@ minijinja-contrib = "2.19.0"
notify = "8.2"
percent-encoding = "2.3"
regex = "1.12.3"
saphyr = "0.0.6"
sha1_smol = "1.0"
slab = "0.4.12"
serde = "1.0"
crates/zensical/Cargo.toml
+1
View File
@@ -59,6 +59,7 @@ mio = { workspace = true, features = ["net", "os-poll"] }
percent-encoding.workspace = true
pyo3.workspace = true
regex.workspace = true
saphyr.workspace = true
serde = { workspace = true, features = ["derive", "rc"] }
serde_json.workspace = true
thiserror.workspace = true
crates/zensical/src/compat/mkdocs/plugin.rs
+1
View File
@@ -13,6 +13,7 @@ use crate::config::Config;
use crate::structure::markdown::Markdown;
pub mod autorefs;
pub mod meta;
pub mod mkdocstrings;
pub mod search;
crates/zensical/src/compat/mkdocs/plugin/meta.rs
+467
View File
@@ -0,0 +1,467 @@
// Copyright (c) 2025-2026 Zensical and contributors
// SPDX-License-Identifier: MIT
// All contributions are certified under the DCO
//! MkDocs Material metadata inheritance.
use anyhow::{bail, Context, Result};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::hash::Hash;
use std::ops::Range;
use std::path::Path;
use crate::config::Config;
use crate::structure::dynamic::Dynamic;
mod parser;
// ----------------------------------------------------------------------------
// Structs
// ----------------------------------------------------------------------------
/// Metadata plugin settings used by the workflow.
#[derive(Clone, Debug)]
pub(crate) struct Settings {
/// Whether inheritance is enabled.
pub enabled: bool,
/// Exact basename of metadata files.
pub meta_file: String,
}
/// A source range expressed as UTF-8 byte offsets.
#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct SourceSpan {
/// Source identifier.
pub source: String,
/// Half-open byte range within the complete source.
pub range: Range<usize>,
}
/// Origin of one metadata value.
#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) enum Origin {
/// Value read from a source document.
Source(SourceSpan),
/// Value created or changed during rendering.
Runtime,
}
/// A source-aware metadata value.
#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct Node {
/// Origin of this node.
origin: Origin,
/// Value and source-aware children.
value: Value,
}
/// Recursive metadata value.
#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
enum Value {
/// Scalar value.
Scalar(Dynamic),
/// Sequence value.
List(Vec<Node>),
/// Mapping value.
Map(BTreeMap<String, Node>),
}
/// One parsed YAML metadata document.
#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct Document {
/// Source-relative path.
path: String,
/// Source-aware root mapping.
root: Node,
}
/// Metadata resolved for one Markdown page.
#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct Resolved {
/// Source-aware root mapping.
root: Node,
}
/// Immutable metadata documents available to one workflow revision.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub(crate) struct Index {
/// Parsed metadata files, shared by every page in the revision.
documents: Vec<Document>,
}
// ----------------------------------------------------------------------------
// Implementations
// ----------------------------------------------------------------------------
impl Settings {
/// Extracts native meta settings from resolved configuration.
pub(crate) fn new(config: &Config) -> Self {
let config = &config.project.plugins.meta.config;
Self {
enabled: config.enabled,
meta_file: config.meta_file.clone(),
}
}
}
impl Node {
/// Creates a runtime-owned tree from a plain dynamic value.
fn runtime(value: Dynamic) -> Self {
let value = match value {
Dynamic::List(values) => {
Value::List(values.into_iter().map(Self::runtime).collect())
}
Dynamic::Map(values) => Value::Map(
values
.into_iter()
.map(|(key, value)| (key, Self::runtime(value)))
.collect(),
),
value => Value::Scalar(value),
};
Self { origin: Origin::Runtime, value }
}
/// Projects a source-aware tree into template-visible metadata.
fn dynamic(&self) -> Dynamic {
match &self.value {
Value::Scalar(value) => value.clone(),
Value::List(values) => {
Dynamic::List(values.iter().map(Self::dynamic).collect())
}
Value::Map(values) => Dynamic::Map(
values
.iter()
.map(|(key, value)| (key.clone(), value.dynamic()))
.collect(),
),
}
}
/// Retains origins for values unchanged by Python extensions.
fn reconcile(&self, value: Dynamic) -> Self {
if self.dynamic() == value {
return self.clone();
}
match (&self.value, value) {
(Value::Map(previous), Dynamic::Map(current)) => {
let values = current
.into_iter()
.map(|(key, value)| {
let value = if let Some(previous) = previous.get(&key) {
previous.reconcile(value)
} else {
Self::runtime(value)
};
(key, value)
})
.collect();
Self {
origin: Origin::Runtime,
value: Value::Map(values),
}
}
(Value::List(previous), Dynamic::List(current)) => {
let values = current
.into_iter()
.enumerate()
.map(|(index, value)| {
if let Some(previous) = previous.get(index) {
previous.reconcile(value)
} else {
Self::runtime(value)
}
})
.collect();
Self {
origin: Origin::Runtime,
value: Value::List(values),
}
}
(_, value) => Self::runtime(value),
}
}
}
impl Resolved {
/// Returns plain values for the Python Markdown boundary.
pub(crate) fn values(&self) -> BTreeMap<String, Dynamic> {
let Value::Map(values) = &self.root.value else {
unreachable!("metadata root is always a mapping")
};
values
.iter()
.map(|(key, value)| (key.clone(), value.dynamic()))
.collect()
}
/// Reconciles source origins with metadata returned from Python.
pub(crate) fn reconcile(&self, values: BTreeMap<String, Dynamic>) -> Self {
let root = self.root.reconcile(Dynamic::Map(values));
Self { root }
}
}
impl Index {
/// Loads and parses every configured metadata file exactly once.
pub(crate) fn load(docs: &Path, settings: &Settings) -> Result<Self> {
if !settings.enabled {
return Ok(Self::default());
}
let mut documents = Vec::new();
collect(docs, docs, settings, &mut documents)?;
Ok(Self { documents })
}
/// Resolves the metadata chain applicable to one page.
pub(crate) fn resolve(
&self, page: &str, front_matter: Option<Document>,
) -> Result<Resolved> {
resolve(
self.documents
.iter()
.filter(|document| applies(&document.path, page))
.cloned(),
front_matter,
)
}
}
// ----------------------------------------------------------------------------
// Functions
// ----------------------------------------------------------------------------
/// Returns whether a source is claimed as a metadata file.
pub(crate) fn claims(path: &str, settings: &Settings) -> bool {
settings.enabled
&& path.rsplit('/').next() == Some(settings.meta_file.as_str())
}
/// Returns whether a metadata file applies to a Markdown page.
pub(crate) fn applies(meta: &str, page: &str) -> bool {
let parent = meta.rsplit_once('/').map_or("", |(parent, _)| parent);
parent.is_empty()
|| page
.strip_prefix(parent)
.is_some_and(|suffix| suffix.starts_with('/'))
}
/// Parses one standalone metadata file.
pub(crate) fn parse(path: &str, source: &str) -> Result<Document> {
parser::parse(path, source, 0)
.with_context(|| format!("error reading meta file '{path}'"))
}
/// Extracts and parses YAML front matter from a Markdown source.
pub(crate) fn front_matter(
path: &str, source: &str,
) -> Result<(String, Option<Document>)> {
parser::front_matter(path, source)
.with_context(|| format!("error reading page metadata '{path}'"))
}
/// Recursively loads metadata documents from the docs tree.
fn collect(
root: &Path, directory: &Path, settings: &Settings,
documents: &mut Vec<Document>,
) -> Result<()> {
for entry in std::fs::read_dir(directory)? {
let path = entry?.path();
if path.is_dir() {
collect(root, &path, settings, documents)?;
} else if path
.file_name()
.is_some_and(|name| name == settings.meta_file.as_str())
{
let relative = path.strip_prefix(root)?;
let location = relative.to_string_lossy().replace('\\', "/");
let source = std::fs::read_to_string(&path)?;
documents.push(parse(&location, &source)?);
}
}
Ok(())
}
/// Resolves applicable meta files and page front matter.
pub(crate) fn resolve(
documents: impl IntoIterator<Item = Document>, page: Option<Document>,
) -> Result<Resolved> {
let mut documents = documents.into_iter().collect::<Vec<_>>();
documents.sort_by(|left, right| {
let left_depth = left.path.matches('/').count();
let right_depth = right.path.matches('/').count();
left_depth
.cmp(&right_depth)
.then(left.path.cmp(&right.path))
});
let mut root = Node {
origin: Origin::Runtime,
value: Value::Map(BTreeMap::new()),
};
for document in documents {
merge(&mut root, document.root.clone()).with_context(|| {
format!("error merging meta file '{}'", document.path)
})?;
}
if let Some(page) = page {
merge(&mut root, page.root).context("error merging page metadata")?;
}
Ok(Resolved { root })
}
/// Applies Material's typesafe-additive merge strategy.
fn merge(target: &mut Node, incoming: Node) -> Result<()> {
let incoming_origin = incoming.origin.clone();
match (&mut target.value, incoming.value) {
(Value::Map(target), Value::Map(incoming)) => {
for (key, value) in incoming {
if let Some(current) = target.get_mut(&key) {
merge(current, value)?;
} else {
target.insert(key, value);
}
}
Ok(())
}
(Value::List(target), Value::List(mut incoming)) => {
target.append(&mut incoming);
Ok(())
}
(Value::Scalar(target_value), Value::Scalar(incoming))
if scalar_kind(target_value) == scalar_kind(&incoming) =>
{
*target_value = incoming;
target.origin = incoming_origin;
Ok(())
}
_ => bail!(
"metadata types do not match ({} conflicts with {})",
origin_label(&target.origin),
origin_label(&incoming_origin)
),
}
}
/// Formats an origin for a concise merge diagnostic.
fn origin_label(origin: &Origin) -> String {
match origin {
Origin::Source(span) => {
format!("{}:{}..{}", span.source, span.range.start, span.range.end)
}
Origin::Runtime => "runtime metadata".into(),
}
}
/// Returns an exact scalar kind for typesafe replacement.
fn scalar_kind(value: &Dynamic) -> u8 {
match value {
Dynamic::Null => 0,
Dynamic::String(_) => 1,
Dynamic::Bool(_) => 2,
Dynamic::Integer(_) => 3,
Dynamic::Float(_) => 4,
Dynamic::List(_) | Dynamic::Map(_) => unreachable!("nested value"),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn values(source: &str) -> BTreeMap<String, Dynamic> {
resolve([], Some(parse("docs/page.md", source).unwrap()))
.unwrap()
.values()
}
#[test]
fn matches_path_components() {
assert!(applies("docs/guide/.meta.yml", "docs/guide/page.md"));
assert!(!applies("docs/guide/.meta.yml", "docs/guidelines/page.md"));
}
#[test]
fn parses_null_as_null() {
assert_eq!(values("value: null\n")["value"], Dynamic::Null);
}
#[test]
fn merges_maps_and_appends_lists() {
let root = parse("docs/.meta.yml", "x:\n a: 1\nitems: [a]\n").unwrap();
let nested =
parse("docs/guide/.meta.yml", "x:\n b: 2\nitems: [b]\n").unwrap();
let values = resolve([nested, root], None).unwrap().values();
assert_eq!(
values["items"],
Dynamic::List(vec![
Dynamic::String("a".into()),
Dynamic::String("b".into())
])
);
assert_eq!(
values["x"],
Dynamic::Map(BTreeMap::from([
("a".into(), Dynamic::Integer(1)),
("b".into(), Dynamic::Integer(2)),
]))
);
}
#[test]
fn rejects_type_mismatch() {
let root = parse("docs/.meta.yml", "value: text\n").unwrap();
let page = parse("docs/page.md", "value: [text]\n").unwrap();
let error = format!("{:#}", resolve([root], Some(page)).unwrap_err());
assert!(error.contains(".meta.yml"));
assert!(error.contains("page.md"));
}
#[test]
fn scalar_override_keeps_winning_source() {
let root = parse(".meta.yml", "title: Default\n").unwrap();
let page = parse("page.md", "title: Page\n").unwrap();
let resolved = resolve([root], Some(page)).unwrap();
let Value::Map(values) = &resolved.root.value else {
panic!("mapping")
};
let Origin::Source(span) = &values["title"].origin else {
panic!("source")
};
assert_eq!(span.source, "page.md");
}
#[test]
fn loads_only_component_ancestors() {
let directory = tempfile::tempdir().unwrap();
let docs = directory.path();
std::fs::create_dir_all(docs.join("guide")).unwrap();
std::fs::create_dir_all(docs.join("guidelines")).unwrap();
std::fs::write(docs.join(".meta.yml"), "items: [root]\n").unwrap();
std::fs::write(docs.join("guide/.meta.yml"), "items: [guide]\n")
.unwrap();
std::fs::write(
docs.join("guidelines/.meta.yml"),
"items: [guidelines]\n",
)
.unwrap();
let settings = Settings {
enabled: true,
meta_file: ".meta.yml".into(),
};
let values = Index::load(docs, &settings)
.unwrap()
.resolve("guide/page.md", None)
.unwrap()
.values();
assert_eq!(
values["items"],
Dynamic::List(vec![
Dynamic::String("root".into()),
Dynamic::String("guide".into()),
])
);
}
}
crates/zensical/src/compat/mkdocs/plugin/meta/parser.rs
+224
View File
@@ -0,0 +1,224 @@
// Copyright (c) 2025-2026 Zensical and contributors
// SPDX-License-Identifier: MIT
// All contributions are certified under the DCO
//! Source-aware YAML parsing for metadata.
use anyhow::{bail, Result};
use saphyr::{LoadableYamlNode, MarkedYaml, Scalar, YamlData};
use std::collections::BTreeMap;
use super::{Document, Node, Origin, SourceSpan, Value};
use crate::structure::dynamic::Dynamic;
/// Parses one YAML mapping and retains source ranges on every value.
pub(super) fn parse(
path: &str, source: &str, offset: usize,
) -> Result<Document> {
let (source, offset) = source
.strip_prefix('\u{FEFF}')
.map_or((source, offset), |source| (source, offset + 3));
let documents = MarkedYaml::load_from_str(source)?;
if documents.len() != 1 {
bail!("metadata must contain exactly one YAML document")
}
let root = convert(path, source, offset, &documents[0])?;
match root.value {
Value::Map(_) => Ok(Document { path: path.into(), root }),
Value::Scalar(Dynamic::Null) if source.trim().is_empty() => {
Ok(Document {
path: path.into(),
root: Node {
origin: root.origin,
value: Value::Map(BTreeMap::new()),
},
})
}
_ => bail!("metadata root must be a mapping"),
}
}
/// Extracts front matter using the same delimiters as Python Markdown.
pub(super) fn front_matter(
path: &str, source: &str,
) -> Result<(String, Option<Document>)> {
let (source, source_offset) = source
.strip_prefix('\u{FEFF}')
.map_or((source, 0), |source| (source, 3));
let mut lines = source.split_inclusive('\n');
let Some(first) = lines.next() else {
return Ok((source.into(), None));
};
if !is_delimiter(first, "---") {
return Ok((source.into(), None));
}
let yaml_start = first.len();
let mut cursor = yaml_start;
for line in lines {
if is_delimiter(line, "---") || is_delimiter(line, "...") {
let yaml = &source[yaml_start..cursor];
let body = source[cursor + line.len()..]
.trim_start_matches('\n')
.to_owned();
return parse(path, yaml, yaml_start + source_offset)
.map(|document| (body, Some(document)));
}
cursor += line.len();
}
Ok((source.into(), None))
}
/// Returns whether a complete line is a front-matter delimiter.
fn is_delimiter(line: &str, delimiter: &str) -> bool {
let line = line.strip_suffix('\n').unwrap_or(line);
let line = line.strip_suffix('\r').unwrap_or(line);
line.strip_prefix(delimiter)
.is_some_and(|suffix| suffix.chars().all(|ch| matches!(ch, ' ' | '\t')))
}
/// Converts one Saphyr node into an owned source-aware node.
fn convert(
path: &str, source: &str, offset: usize, node: &MarkedYaml<'_>,
) -> Result<Node> {
let origin = Origin::Source(SourceSpan {
source: path.into(),
range: marker_to_byte(source, node.span.start.index()) + offset
..marker_to_byte(source, node.span.end.index()) + offset,
});
let value = match &node.data {
YamlData::Value(value) => Value::Scalar(convert_scalar(value)),
YamlData::Sequence(values) => Value::List(
values
.iter()
.map(|value| convert(path, source, offset, value))
.collect::<Result<_>>()?,
),
YamlData::Mapping(values) => {
let mut explicit = BTreeMap::new();
let mut inherited = BTreeMap::new();
for (key, value) in values {
let key = string_key(key)?;
let value = convert(path, source, offset, value)?;
if key == "<<" {
merge_key(&mut inherited, value)?;
} else {
explicit.insert(key, value);
}
}
inherited.extend(explicit);
Value::Map(inherited)
}
YamlData::Tagged(_, _) => bail!("custom YAML tags are not supported"),
YamlData::Alias(_) => bail!("unresolved YAML alias"),
YamlData::BadValue => bail!("invalid YAML value"),
YamlData::Representation(_, _, _) => {
unreachable!("Saphyr performs early scalar parsing")
}
};
Ok(Node { origin, value })
}
/// Converts a Saphyr scalar to the common metadata representation.
fn convert_scalar(value: &Scalar<'_>) -> Dynamic {
match value {
Scalar::Null => Dynamic::Null,
Scalar::Boolean(value) => Dynamic::Bool(*value),
Scalar::Integer(value) => Dynamic::Integer(*value),
Scalar::FloatingPoint(value) => Dynamic::from_float(value.into_inner()),
Scalar::String(value) => Dynamic::String(value.to_string()),
}
}
/// Requires mapping keys to be strings.
fn string_key(node: &MarkedYaml<'_>) -> Result<String> {
match &node.data {
YamlData::Value(Scalar::String(value)) => Ok(value.to_string()),
_ => bail!("metadata mapping keys must be strings"),
}
}
/// Expands the YAML merge key while retaining the referenced node origins.
fn merge_key(target: &mut BTreeMap<String, Node>, node: Node) -> Result<()> {
match node.value {
Value::Map(values) => {
for (key, value) in values {
target.entry(key).or_insert(value);
}
Ok(())
}
Value::List(values) => {
for value in values {
let Value::Map(values) = value.value else {
bail!("YAML merge sequence entries must be mappings")
};
for (key, value) in values {
target.entry(key).or_insert(value);
}
}
Ok(())
}
Value::Scalar(_) => {
bail!("YAML merge value must be a mapping or list of mappings")
}
}
}
/// Converts Saphyr's character index to a UTF-8 byte offset.
fn marker_to_byte(source: &str, index: usize) -> usize {
if index == source.chars().count() {
source.len()
} else {
source
.char_indices()
.nth(index)
.map_or(source.len(), |(index, _)| index)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn retains_unicode_byte_ranges() {
let document = parse("docs/.meta.yml", "title: Héllo\n", 0).unwrap();
let Value::Map(values) = document.root.value else {
panic!("mapping")
};
let Origin::Source(span) = &values["title"].origin else {
panic!("source")
};
assert_eq!(&"title: Héllo\n"[span.range.clone()], "Héllo");
}
#[test]
fn extracts_front_matter_and_offsets_ranges() {
let source = "---\ntitle: Home\n---\n\n# Home\n";
let (body, document) = front_matter("docs/index.md", source).unwrap();
assert_eq!(body, "# Home\n");
let document = document.unwrap();
let Value::Map(values) = document.root.value else {
panic!("mapping")
};
let Origin::Source(span) = &values["title"].origin else {
panic!("source")
};
assert_eq!(&source[span.range.clone()], "Home");
}
#[test]
fn expands_alias_merge_keys() {
let source = "base: &base\n one: 1\nvalue:\n <<: *base\n two: 2\n";
let document = parse("docs/.meta.yml", source, 0).unwrap();
let Value::Map(root) = document.root.value else {
panic!("mapping")
};
let Value::Map(value) = &root["value"].value else {
panic!("mapping")
};
assert!(value.contains_key("one"));
assert!(value.contains_key("two"));
}
}
crates/zensical/src/compat/mkdocs/plugin/search.rs
+40 -40
View File
@@ -32,8 +32,7 @@ use std::io::{BufWriter, Write};
use std::sync::Arc;
use zrx::id::Id;
use zrx::scheduler::Value;
use zrx::stream::function::Collection;
use zrx::stream::{Key, Signal, Stream};
use zrx::stream::{Key, Signal};
use crate::config::plugins::SearchPluginConfig;
use crate::config::Config;
@@ -89,6 +88,15 @@ pub(crate) struct Document {
facts: Arc<Facts>,
}
/// Revision-aligned search inputs from the site settlement boundary.
#[derive(Clone, Debug)]
pub(crate) struct Snapshot {
/// Compact page documents.
documents: Arc<Vec<(Key<Id>, Document)>>,
/// Navigation from the same page revision.
nav: Navigation,
}
// ----------------------------------------------------------------------------
// Implementations
// ----------------------------------------------------------------------------
@@ -119,6 +127,20 @@ impl Document {
// ----------------------------------------------------------------------------
impl Snapshot {
/// Creates a search snapshot without another site-wide reduction.
pub(crate) fn new(
documents: Vec<(Key<Id>, Document)>, nav: Navigation,
) -> Self {
Self {
documents: Arc::new(documents),
nav,
}
}
}
// ----------------------------------------------------------------------------
impl Facts {
/// Returns whether this page contributes anything to the search index.
pub(crate) fn is_empty(&self) -> bool {
@@ -190,51 +212,29 @@ impl SearchIndex {
// ----------------------------------------------------------------------------
impl Value for Document {}
impl Value for Snapshot {}
// ----------------------------------------------------------------------------
// Functions
// ----------------------------------------------------------------------------
/// Attach MkDocs-compatible search artifact generation to the build graph.
pub(crate) fn attach(
config: &Config, documents: &Stream<Id, Document>,
nav: &Signal<Id, Navigation>,
) {
if !config.project.plugins.search.config.enabled {
let config = config.clone();
let _ = nav.map(move |nav: &Navigation| {
let search = SearchIndex::new(
Vec::new(),
nav,
config.project.plugins.search.config.clone(),
&config.project.theme.language,
);
write(&config, &search)
});
return;
}
let documents =
documents.reduce(|documents: &dyn Collection<Key<Id>, Document>| {
Some(
documents
.iter()
.map(|(key, document)| (key.clone(), document.clone()))
.collect::<Vec<_>>(),
)
});
pub(crate) fn attach(config: &Config, snapshot: &Signal<Id, Snapshot>) {
let config = config.clone();
let _ = documents.product(nav).map(
move |documents: &Vec<(Key<Id>, Document)>, nav: &Navigation| {
let search = SearchIndex::new(
documents.clone(),
nav,
config.project.plugins.search.config.clone(),
&config.project.theme.language,
);
write(&config, &search)
},
);
let _ = snapshot.map(move |snapshot: &Snapshot| {
let documents = if config.project.plugins.search.config.enabled {
snapshot.documents.as_ref().clone()
} else {
Vec::new()
};
let search = SearchIndex::new(
documents,
&snapshot.nav,
config.project.plugins.search.config.clone(),
&config.project.theme.language,
);
write(&config, &search)
});
}
/// Creates the page-local search visitor.
crates/zensical/src/config/plugins.rs
+22
View File
@@ -46,12 +46,34 @@ use serde::Serialize;
pub struct Plugins {
/// Search plugin.
pub search: SearchPlugin,
/// Material meta plugin.
pub meta: MetaPlugin,
/// Offline plugin.
pub offline: OfflinePlugin,
}
// ----------------------------------------------------------------------------
/// Material meta plugin.
#[derive(Clone, Debug, Hash, FromPyObject, Serialize)]
#[pyo3(from_item_all)]
pub struct MetaPlugin {
/// Plugin configuration.
pub config: MetaPluginConfig,
}
/// Material meta plugin configuration.
#[derive(Clone, Debug, Hash, FromPyObject, Serialize)]
#[pyo3(from_item_all)]
pub struct MetaPluginConfig {
/// Whether metadata inheritance is enabled.
pub enabled: bool,
/// Name of metadata files inside the documentation tree.
pub meta_file: String,
}
// ----------------------------------------------------------------------------
/// Search plugin.
#[derive(Clone, Debug, Hash, FromPyObject, Serialize)]
#[pyo3(from_item_all)]
crates/zensical/src/lib.rs
+182 -6
View File
@@ -34,11 +34,14 @@ use crossbeam::channel::{unbounded, RecvTimeoutError};
use pyo3::exceptions::PyRuntimeError;
use pyo3::prelude::*;
use pyo3::Python;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::Arc;
use std::time::{Duration, Instant};
use std::{fs, io, thread};
use zrx::id::Id;
use zrx::stream::{Change, Key};
mod compat;
mod config;
@@ -49,10 +52,11 @@ mod template;
mod watcher;
mod workflow;
use compat::mkdocs::plugin::meta;
use config::Config;
use server::{create_server, ServeOptions};
use watcher::Watcher;
use workflow::create_workflow;
use watcher::{Source, Watcher};
use workflow::{create_workflow, Input};
// ----------------------------------------------------------------------------
// Enums
@@ -198,8 +202,9 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult<bool> {
.runner()
.map_err(|err| PyRuntimeError::new_err(err.to_string()))?;
let mut input = runner
.input::<watcher::Source>()
.input::<Input>()
.map_err(|err| PyRuntimeError::new_err(err.to_string()))?;
let meta_settings = meta::Settings::new(&config);
// Create channel for reload notifications
let (sender, receiver) = unbounded();
@@ -237,13 +242,40 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult<bool> {
loop {
match watcher.receive(Duration::from_millis(100)) {
Ok(changes) => {
let metadata = Arc::new(
meta::Index::load(&config.get_docs_dir(), &meta_settings)
.map_err(|error| {
PyRuntimeError::new_err(format!("{error:#}"))
})?,
);
let dependents = metadata_dependents(
&changes,
&config.get_docs_dir(),
&meta_settings,
)?;
let mut revision = input
.begin()
.map_err(|err| PyRuntimeError::new_err(err.to_string()))?;
for (key, source) in dependents {
revision
.insert(key, Input::new(source, metadata.clone()))
.map_err(|err| {
PyRuntimeError::new_err(err.to_string())
})?;
}
for change in changes {
revision.emit(change).map_err(|err| {
PyRuntimeError::new_err(err.to_string())
})?;
match change {
Change::Insert(key, source) => revision
.insert(key, Input::new(source, metadata.clone()))
.map_err(|err| {
PyRuntimeError::new_err(err.to_string())
})?,
Change::Remove(key) => {
revision.remove(key).map_err(|err| {
PyRuntimeError::new_err(err.to_string())
})?;
}
}
}
input = revision
.seal()
@@ -282,6 +314,75 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult<bool> {
Ok(false)
}
/// Expands metadata-file changes into descendant Markdown updates.
fn metadata_dependents(
changes: &[Change<Id, Source>], docs: &Path, settings: &meta::Settings,
) -> PyResult<Vec<(Key<Id>, Source)>> {
if !settings.enabled {
return Ok(Vec::new());
}
let mut dependents = BTreeMap::new();
for change in changes {
let key = match change {
Change::Insert(key, _) | Change::Remove(key) => key,
};
let location = key[0].location();
if !meta::claims(&location, settings) {
continue;
}
let parent = Path::new(location.as_ref())
.parent()
.unwrap_or_else(|| Path::new(""));
let mut paths = Vec::new();
collect_markdown(&docs.join(parent), &mut paths)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
for path in paths {
let relative = path
.strip_prefix(docs)
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
let location = relative.to_string_lossy().replace('\\', "/");
let id = key[0]
.to_builder()
.location(location)
.build()
.expect("invariant");
dependents.insert(
Key::from(id),
Source::from(path.to_string_lossy().into_owned()),
);
}
}
// A provider update for the page itself is authoritative. In particular,
// the initial snapshot contains both metadata files and every Markdown
// page, so retaining synthesized inserts here would admit each page twice.
for change in changes {
let key = match change {
Change::Insert(key, _) | Change::Remove(key) => key,
};
dependents.remove(key);
}
Ok(dependents.into_iter().collect())
}
/// Recursively collects Markdown files below one metadata directory.
fn collect_markdown(
directory: &Path, paths: &mut Vec<PathBuf>,
) -> io::Result<()> {
let Ok(entries) = fs::read_dir(directory) else {
return Ok(());
};
for entry in entries {
let path = entry?.path();
if path.is_dir() {
collect_markdown(&path, paths)?;
} else if path.extension().is_some_and(|extension| extension == "md") {
paths.push(path);
}
}
Ok(())
}
/// Returns the first action failure reported by one settled run.
fn report_failures(run: &zrx::stream::Run<Id>) -> PyResult<()> {
for invocation in run.report().invocations() {
@@ -441,4 +542,79 @@ mod tests {
assert!(dir.path().exists());
}
#[test]
fn metadata_change_selects_only_descendant_markdown() {
let dir = tempdir().unwrap();
let docs = dir.path();
fs::create_dir_all(docs.join("guide/nested")).unwrap();
fs::create_dir_all(docs.join("guidelines")).unwrap();
fs::write(docs.join("guide/page.md"), "# Page").unwrap();
fs::write(docs.join("guide/nested/page.md"), "# Nested").unwrap();
fs::write(docs.join("guidelines/page.md"), "# Other").unwrap();
let id = Id::builder()
.provider("file")
.context("docs")
.location("guide/.meta.yml")
.build()
.unwrap();
let changes = vec![Change::Insert(
Key::from(id),
Source::from(docs.join("guide/.meta.yml").display().to_string()),
)];
let settings = meta::Settings {
enabled: true,
meta_file: ".meta.yml".into(),
};
let dependents =
metadata_dependents(&changes, docs, &settings).unwrap();
let locations = dependents
.iter()
.map(|(key, _)| key[0].location().into_owned())
.collect::<Vec<_>>();
assert_eq!(locations, vec!["guide/nested/page.md", "guide/page.md"]);
}
#[test]
fn provider_page_change_supersedes_metadata_dependent() {
let dir = tempdir().unwrap();
let docs = dir.path();
fs::create_dir_all(docs.join("guide")).unwrap();
let page = docs.join("guide/page.md");
fs::write(&page, "# Page").unwrap();
let meta_id = Id::builder()
.provider("file")
.context("docs")
.location("guide/.meta.yml")
.build()
.unwrap();
let page_id = Id::builder()
.provider("file")
.context("docs")
.location("guide/page.md")
.build()
.unwrap();
let changes = vec![
Change::Insert(
Key::from(meta_id),
Source::from(
docs.join("guide/.meta.yml").display().to_string(),
),
),
Change::Insert(
Key::from(page_id),
Source::from(page.display().to_string()),
),
];
let settings = meta::Settings {
enabled: true,
meta_file: ".meta.yml".into(),
};
assert!(metadata_dependents(&changes, docs, &settings)
.unwrap()
.is_empty());
}
}
crates/zensical/src/structure/dynamic.rs
+63 -10
View File
@@ -25,7 +25,11 @@
//! Dynamic value.
use pyo3::FromPyObject;
use pyo3::exceptions::PyTypeError;
use pyo3::types::{
PyAny, PyAnyMethods, PyBool, PyDict, PyFloat, PyInt, PyList, PyString,
};
use pyo3::{Borrowed, FromPyObject, PyErr};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::fmt;
@@ -42,19 +46,15 @@ use float::Float;
///
/// This data type represents any valid value that can be used as part of the
/// metadata of a page and the extra data of configuration, supporting strings,
/// booleans, integers, floating point numbers, lists, and maps, so basically
/// nulls, booleans, integers, floating point numbers, lists, and maps, so
/// basically
/// everything supported in YAML and TOML.
///
/// Null value are not supported, and currently represented as empty strings.
/// We're aiming to provide a type safe way to define custom namespaces in the
/// configuration, so we'll definitely revisit this as part of our efforts to
/// make configuration much more flexible.
#[derive(
Clone, Debug, FromPyObject, Hash, PartialEq, Eq, Serialize, Deserialize,
)]
#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
#[pyo3(from_item_all)]
pub enum Dynamic {
/// Null value.
Null,
/// String value.
String(String),
/// Boolean value.
@@ -77,6 +77,7 @@ impl fmt::Display for Dynamic {
/// Formats the dynamic value for display.
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Dynamic::Null => write!(f, "null"),
Dynamic::String(value) => write!(f, "{value}"),
Dynamic::Bool(value) => write!(f, "{value}"),
Dynamic::Integer(value) => write!(f, "{value}"),
@@ -94,3 +95,55 @@ impl fmt::Display for Dynamic {
}
}
}
// ----------------------------------------------------------------------------
impl<'a, 'py> FromPyObject<'a, 'py> for Dynamic {
type Error = PyErr;
fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result<Self, Self::Error> {
if obj.is_none() {
Ok(Self::Null)
} else if obj.is_instance_of::<PyBool>() {
obj.extract().map(Self::Bool)
} else if obj.is_instance_of::<PyInt>() {
obj.extract().map(Self::Integer)
} else if obj.is_instance_of::<PyFloat>() {
obj.extract().map(|value| Self::Float(Float(value)))
} else if obj.is_instance_of::<PyString>() {
obj.extract().map(Self::String)
} else if obj.is_instance_of::<PyList>() {
obj.extract().map(Self::List)
} else if obj.is_instance_of::<PyDict>() {
obj.extract().map(Self::Map)
} else {
Err(PyTypeError::new_err("unsupported dynamic value"))
}
}
}
// ----------------------------------------------------------------------------
impl Dynamic {
/// Creates a dynamic floating-point value.
pub(crate) fn from_float(value: f64) -> Self {
Self::Float(Float(value))
}
}
// ----------------------------------------------------------------------------
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn null_round_trips_through_json() {
let data = serde_json::to_string(&Dynamic::Null).unwrap();
assert_eq!(data, "null");
assert_eq!(
serde_json::from_str::<Dynamic>(&data).unwrap(),
Dynamic::Null
);
}
}
crates/zensical/src/structure/markdown.rs
+8 -3
View File
@@ -89,12 +89,15 @@ struct RenderedMarkdown {
impl Markdown {
/// Renders Markdown using Python Markdown.
#[cfg_attr(feature = "tracing", tracing::instrument(skip_all))]
pub fn new(id: &Id, url: String, content: String) -> Result<Markdown> {
pub fn new(
id: &Id, url: String, content: String, meta: BTreeMap<String, Dynamic>,
) -> Result<Markdown> {
let id = id.clone();
let meta = serde_json::to_string(&meta)?;
let res = Python::attach(|py| {
let module = py.import("zensical.markdown.render")?;
module
.call_method1("render", (content, id.location(), url))?
.call_method1("render", (content, id.location(), url, meta))?
.extract::<RenderedMarkdown>()
})
.map_err(|err| {
@@ -168,7 +171,9 @@ impl Eq for Markdown {}
/// We'll fix this in our modular navigation proposal that will make title
/// handling much more flexible in the near future.
fn extract_title(id: &Id, markdown: &MarkdownData) -> String {
if let Some(value) = markdown.meta.get("title") {
if let Some(value) = markdown.meta.get("title")
&& !matches!(value, Dynamic::Null)
{
return value.to_string();
}
crates/zensical/src/structure/page.rs
+5 -3
View File
@@ -199,8 +199,10 @@ impl Page {
pub fn render_template(
&mut self, template: &Template, config: &Config, nav: Navigation,
) -> Result<Output, Error> {
let name = self.meta.get("template").map(ToString::to_string);
let name = name.as_deref().unwrap_or("main.html");
let name = match self.meta.get("template") {
Some(Dynamic::String(value)) => value.clone(),
_ => "main.html".into(),
};
// Compute page relations from the immutable navigation.
self.ancestors = nav.ancestors(self);
@@ -210,7 +212,7 @@ impl Page {
// Add the page-local active overlay without cloning the navigation tree.
let nav = NavigationView::new(nav, Some(&self.url));
let output = template.render_with_context(
name,
&name,
context! {
generator => GENERATOR,
nav => TemplateValue::from_object(nav),
crates/zensical/src/workflow.rs
+102 -35
View File
@@ -28,6 +28,7 @@
use regex::Regex;
use serde::{Deserialize, Serialize};
use std::hash::{DefaultHasher, Hash, Hasher};
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::{Arc, LazyLock, OnceLock};
@@ -40,7 +41,9 @@ use zrx::stream::{
concurrent, Key, Signal, Stream, StreamTupleExt, Value, Workflow,
};
use super::compat::mkdocs::plugin::{self, autorefs, mkdocstrings, search};
use super::compat::mkdocs::plugin::{
self, autorefs, meta, mkdocstrings, search,
};
use super::config::Config;
use super::structure::markdown::Markdown;
use super::structure::nav::Navigation;
@@ -86,6 +89,32 @@ pub struct Main {
strict: bool,
}
/// File input enriched with immutable facts for the current revision.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Input {
/// Source supplied by the file provider.
source: Source,
/// Metadata files parsed once and shared by every page in the revision.
metadata: Arc<meta::Index>,
}
impl Value for Input {}
impl Input {
/// Enriches one provider source with revision-local metadata facts.
pub(crate) fn new(source: Source, metadata: Arc<meta::Index>) -> Self {
Self { source, metadata }
}
}
impl Deref for Input {
type Target = Source;
fn deref(&self) -> &Self::Target {
&self.source
}
}
/// Revision-settled site batch derived from the current page relation.
#[derive(Clone, Debug)]
struct Site {
@@ -95,6 +124,8 @@ struct Site {
nav: Navigation,
/// Autoref registry derived from the same settled page snapshot.
autorefs: autorefs::Registry,
/// Search inputs derived from the same settled page snapshot.
search: search::Snapshot,
}
impl Value for Site {}
@@ -123,6 +154,8 @@ struct RenderedMarkdown {
registrations: Arc<autorefs::Facts>,
/// Facts extracted by the shared MkDocs-compatible HTML pass.
html: plugin::HtmlFacts,
/// Resolved metadata and source mappings for later compatibility modules.
pub(crate) meta: Arc<meta::Resolved>,
}
impl Value for RenderedMarkdown {}
@@ -138,22 +171,26 @@ struct RenderedPage {
registrations: Arc<autorefs::Facts>,
/// HTML compatibility facts revision-aligned with the page.
html: plugin::HtmlFacts,
/// Resolved metadata and source mappings for later compatibility modules.
pub(crate) meta: Arc<meta::Resolved>,
}
impl Value for RenderedPage {}
// ----------------------------------------------------------------------------
// Implementations
// ----------------------------------------------------------------------------
impl Main {
/// Initializes the module.
fn setup(&self, ctx: &mut Builder<Id>) {
let files = ctx.input::<Source>();
let files = ctx.input::<Input>();
let meta = meta::Settings::new(&self.config);
// Set up workflow to process static assets and Markdown files.
process_theme_assets(&self.config, &files);
process_assets(&self.config, &files);
process_assets(&self.config, &files, &meta);
let rendered = process_markdown(&self.config, &files);
// Cross the one global settlement boundary, derive all site-wide
@@ -161,17 +198,10 @@ impl Main {
let rendered_page = generate_page(&self.config, &rendered);
let page =
rendered_page.map(|rendered: &RenderedPage| rendered.page.clone());
let document = rendered_page
.filter(|rendered: &RenderedPage| !rendered.html.search.is_empty())
.map(|rendered: &RenderedPage| {
search::Document::new(
&rendered.page,
rendered.html.search.clone(),
)
});
let site = generate_site(&self.config, &rendered_page);
let nav = generate_nav(&site);
search::attach(&self.config, &document, &nav);
let search = site.map(|site: &Site| site.search.clone());
search::attach(&self.config, &search);
mkdocstrings::attach(&self.config, &nav);
let _ = render_templates(&self.config, &files, &nav);
let unresolved = render_pages(&self.config, &site);
@@ -208,7 +238,7 @@ pub fn wait_for_markdown(config: &Config) -> (Key<Id>, Barrier<Id>) {
/// Create a stream to collect references from all Markdown files.
pub fn collect_references(
config: &Config, files: &Stream<Id, Source>,
config: &Config, files: &Stream<Id, Input>,
) -> Stream<Id, SharedReferences> {
let matcher = Arc::new(
Matcher::from_str(&format!(
@@ -221,7 +251,7 @@ pub fn collect_references(
// Create pipeline to collect references
files
.filter(move |id: &Id| matcher.is_match(id).expect("invariant"))
.map(|source: &Source| {
.map(|source: &Input| {
let references: References =
fs::read_to_string(&*source.path)?.parse()?;
Ok::<_, anyhow::Error>(SharedReferences::from(references))
@@ -230,7 +260,7 @@ pub fn collect_references(
/// Validate references and autorefs after every current page has rendered.
fn validate(
config: &Config, strict: bool, files: &Stream<Id, Source>,
config: &Config, strict: bool, files: &Stream<Id, Input>,
pages: &Stream<Id, Page>, unresolved: &Stream<Id, UnresolvedAutorefs>,
) {
let validation = config.project.validation.clone();
@@ -272,7 +302,9 @@ fn page_hash(page: &Page, autorefs: &autorefs::References) -> u64 {
}
/// Create a stream to process static assets.
pub fn process_assets(config: &Config, files: &Stream<Id, Source>) {
pub fn process_assets(
config: &Config, files: &Stream<Id, Input>, meta: &meta::Settings,
) {
let extra_templates = config.project.extra_templates.clone();
let docs_dir = config.project.docs_dir.clone();
let matcher = Arc::new(
@@ -282,7 +314,8 @@ pub fn process_assets(config: &Config, files: &Stream<Id, Source>) {
// Create pipeline to copy static assets
let site_dir = config.project.site_dir.clone();
let root_dir = config.get_root_dir();
let _ = files.map(move |id: &Id, from: &Source| {
let meta = meta.clone();
let _ = files.map(move |id: &Id, from: &Input| {
if !matcher.is_match(id).expect("invariant") {
return Ok(());
}
@@ -292,6 +325,11 @@ pub fn process_assets(config: &Config, files: &Stream<Id, Source>) {
return Ok(());
}
// Metadata files are inputs, not site assets.
if meta::claims(&id.location(), &meta) {
return Ok(());
}
// Don't copy template files that we render later
if extra_templates.contains(&id.location().into_owned()) {
return Ok(());
@@ -305,20 +343,20 @@ pub fn process_assets(config: &Config, files: &Stream<Id, Source>) {
// Compute parent path, create intermediate directories and copy files
let to = root_dir.join(id.to_path());
fs::create_dir_all(to.parent().expect("invariant"))?;
copy_file(&**from, to)?;
copy_file(&from.path, to)?;
Ok::<(), anyhow::Error>(())
});
}
/// Create a stream to process static assets in theme.
pub fn process_theme_assets(config: &Config, files: &Stream<Id, Source>) {
pub fn process_theme_assets(config: &Config, files: &Stream<Id, Input>) {
let matcher =
Arc::new(Matcher::from_str("zrs::::templates/*::").expect("invariant"));
// Create pipeline to copy static assets
let site_dir = config.project.site_dir.clone();
let root_dir = config.get_root_dir();
let _ = files.map(move |id: &Id, from: &Source| {
let _ = files.map(move |id: &Id, from: &Input| {
if !matcher.is_match(id).expect("invariant") {
return Ok(());
}
@@ -336,7 +374,7 @@ pub fn process_theme_assets(config: &Config, files: &Stream<Id, Source>) {
// Compute parent path, create intermediate directories and copy files
let to = root_dir.join(id.to_path());
fs::create_dir_all(to.parent().expect("invariant"))?;
copy_file(&**from, to)?;
copy_file(&from.path, to)?;
Ok::<_, anyhow::Error>(())
});
}
@@ -352,7 +390,7 @@ fn copy_file(
/// Create a stream to process Markdown files.
fn process_markdown(
config: &Config, files: &Stream<Id, Source>,
config: &Config, files: &Stream<Id, Input>,
) -> Stream<Id, RenderedMarkdown> {
let matcher = Arc::new(
Matcher::from_str(&format!(
@@ -371,12 +409,12 @@ fn process_markdown(
// disposal. Otherwise, just return that if the content did not change.
// Note that we need to limit concurrency here, or we'll overwhelm the
// Python interpreter with all tasks competing for the GIL.
.map(concurrent(1, move |id: &Id, path: &Source| {
let data = fs::read_to_string(&**path)?;
.map(concurrent(1, move |id: &Id, source: &Input| {
let location = id.location().into_owned();
let data = fs::read_to_string(&*source.path)?;
// Remove Byte-Order-Mark (BOM)
let data = data.strip_prefix('\u{FEFF}').unwrap_or(&data);
let data = data.to_owned();
let (data, page_meta) = meta::front_matter(&location, &data)?;
let resolved = source.metadata.resolve(&location, page_meta)?;
// Compute URL using same logic as Page::new()
let site_dir = config.project.site_dir.clone();
@@ -417,13 +455,21 @@ fn process_markdown(
// This is a hack while waiting for CommonMark (AST) and components,
// as well as topic-based authoring functionality.
if SNIPPET_RE.is_match(&data) {
render_markdown(id, url, data, plugins)
render_markdown(id, url, data, plugins, resolved)
} else {
cached(
&config,
id.as_str(),
(config.hash, data.clone(), url.clone()),
|(_, data, url)| render_markdown(id, url, data, plugins),
(
1_u8,
config.hash,
data.clone(),
url.clone(),
resolved.clone(),
),
|(_, _, data, url, resolved)| {
render_markdown(id, url, data, plugins, resolved)
},
)
}
}))
@@ -432,15 +478,22 @@ fn process_markdown(
/// Render Markdown and collect the page-local facts produced alongside it.
fn render_markdown(
id: &Id, url: String, content: String, plugins: plugin::Settings,
meta: meta::Resolved,
) -> anyhow::Result<RenderedMarkdown> {
let mut markdown = Markdown::new(id, url.clone(), content)?;
let mut markdown = Markdown::new(id, url.clone(), content, meta.values())?;
let html = plugin::prepare(&mut markdown, plugins);
let registrations = if plugins.autorefs {
autorefs::take_page(&url)
} else {
Arc::default()
};
Ok(RenderedMarkdown { markdown, registrations, html })
let meta = Arc::new(meta.reconcile(markdown.meta.clone()));
Ok(RenderedMarkdown {
markdown,
registrations,
html,
meta,
})
}
/// Generate pages from Markdown files.
@@ -452,6 +505,7 @@ fn generate_page(
page: Page::new(&config, id, markdown.markdown.clone()),
registrations: markdown.registrations.clone(),
html: markdown.html.clone(),
meta: markdown.meta.clone(),
})
}
@@ -464,6 +518,7 @@ fn generate_site(
let mut nav_pages = Vec::new();
let mut site_pages = Vec::new();
let mut facts = Vec::new();
let mut documents = Vec::new();
for (key, rendered) in pages.iter() {
nav_pages.push((key.clone(), rendered.page.clone()));
site_pages.push((
@@ -474,14 +529,25 @@ fn generate_site(
},
));
facts.push((key.clone(), rendered.registrations.clone()));
if !rendered.html.search.is_empty() {
documents.push((
key.clone(),
search::Document::new(
&rendered.page,
rendered.html.search.clone(),
),
));
}
}
let nav = Navigation::new(config.project.nav.clone(), nav_pages);
let autorefs = autorefs::assemble(&config, facts);
let search = search::Snapshot::new(documents, nav.clone());
Some(Site {
pages: Arc::new(site_pages),
nav,
autorefs,
search,
})
})
}
@@ -493,7 +559,7 @@ fn generate_nav(site: &Signal<Id, Site>) -> Signal<Id, Navigation> {
/// Render static and extra templates.
pub fn render_templates(
config: &Config, files: &Stream<Id, Source>, nav: &Signal<Id, Navigation>,
config: &Config, files: &Stream<Id, Input>, nav: &Signal<Id, Navigation>,
) -> Stream<Id, ()> {
let docs_dir = config.project.docs_dir.clone();
@@ -527,8 +593,9 @@ pub fn render_templates(
let config = config.clone();
templates
.product(nav)
.map(move |template: &Source, nav: &Navigation| {
let name = Path::new(&**template).file_name().expect("invariant");
.map(move |template: &Input, nav: &Navigation| {
let name =
Path::new(&template.path).file_name().expect("invariant");
let site_dir = config.get_site_dir();
// Render template and write to disk
python/tests/unit/test_config.py
Vendored
+10
View File
@@ -171,6 +171,16 @@ class TestPluginShimming:
config = self._parse_yaml(tmp_path, plugins={"glightbox": {}})
assert GlightboxExtension.name in config["markdown_extensions"]
def test_material_meta_plugin_is_normalized(self, tmp_path: Path) -> None:
config = self._parse_yaml(
tmp_path,
plugins={"material/meta": {"meta_file": "defaults.yml"}},
)
assert config["plugins"]["meta"]["config"] == {
"enabled": True,
"meta_file": "defaults.yml",
}
def test_mike_plugin_defaults_with_versioned_build(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
python/zensical/config.py
+12
View File
@@ -1270,6 +1270,18 @@ def _convert_plugins(value: Any, config: dict) -> dict:
search, "separator", '[\\s\\-_,:!=\\[\\]()\\\\"`/]+|\\.(?!\\d)', str
)
# Normalize Material's meta plugin to an identifier that can be extracted
# into the typed Rust configuration. Keep the original entry intact for
# compatibility with consumers of the MkDocs plugin mapping.
material_meta = plugins.get("material/meta")
if material_meta is None:
meta = {"enabled": False, "meta_file": ".meta.yml"}
else:
meta = dict(material_meta or {})
set_default(meta, "enabled", True, bool)
set_default(meta, "meta_file", ".meta.yml", str)
plugins["meta"] = meta
# Define defaults for offline plugin
offline = set_default(plugins, "offline", {"enabled": False}, dict)
set_default(offline, "enabled", True, bool)
python/zensical/markdown/render.py
+6 -33
View File
@@ -23,39 +23,24 @@
from __future__ import annotations
import json
import re
from datetime import date, datetime
from typing import Any
import yaml
from markdown import Markdown
from yaml import SafeLoader
from zensical.config import get_config
from zensical.extensions.autorefs import set_autorefs_page
from zensical.extensions.context import ContextExtension, Page
from zensical.extensions.links import LinksExtension
# ----------------------------------------------------------------------------
# Constants
# ----------------------------------------------------------------------------
FRONT_MATTER_RE = re.compile(
r"^-{3}[ \r\t]*?\n(.*?\r?\n)(?:\.{3}|-{3})[ \r\t]*\n",
re.UNICODE | re.DOTALL,
)
"""
Regex pattern to extract front matter.
"""
# ----------------------------------------------------------------------------
# Functions
# ----------------------------------------------------------------------------
def render(content: str, path: str, url: str) -> dict:
def render(content: str, path: str, url: str, metadata: str = "{}") -> dict:
"""Render Markdown and return HTML.
This function returns rendered HTML as well as the table of contents and
@@ -63,19 +48,10 @@ def render(content: str, path: str, url: str) -> dict:
in order to support the specific syntax of Python Markdown. We're working
on moving the entire rendering chain to Rust.
"""
# First, extract metadata - the Python Markdown parser brings a metadata
# extension, but the implementation is broken, as it does not support full
# YAML syntax, e.g. lists. Thus, we just parse the metadata with YAML.
meta: dict = {}
if match := FRONT_MATTER_RE.match(content):
try:
meta = yaml.load(match.group(1), SafeLoader)
if isinstance(meta, dict):
content = content[match.end() :].lstrip("\n")
else:
meta = {}
except Exception: # noqa: BLE001
pass
# Metadata inheritance and front matter are resolved in Rust before this
# boundary. JSON keeps the call explicit and avoids reconstructing Python
# objects one value at a time through the FFI.
meta: dict = json.loads(metadata)
# Create page context and set it for autorefs.
# We can stop setting the page if/when we vendor mkdocstrings.
@@ -143,9 +119,6 @@ def render(content: str, path: str, url: str) -> dict:
def _sanitize(value: Any) -> Any:
# We currently don't have a null value for metadata in the Rust runtime
if value is None:
return ""
if isinstance(value, (date, datetime)):
return value.isoformat()
if isinstance(value, dict):