diff --git a/crates/zensical/src/compat/mkdocs.rs b/crates/zensical/src/compat/mkdocs.rs index 491df5e..1f2c012 100644 --- a/crates/zensical/src/compat/mkdocs.rs +++ b/crates/zensical/src/compat/mkdocs.rs @@ -5,5 +5,5 @@ //! MkDocs compatibility modules. -pub mod mkdocstrings; -pub mod search; +pub(crate) mod html; +pub mod plugin; diff --git a/crates/zensical/src/compat/mkdocs/html.rs b/crates/zensical/src/compat/mkdocs/html.rs new file mode 100644 index 0000000..697fe62 --- /dev/null +++ b/crates/zensical/src/compat/mkdocs/html.rs @@ -0,0 +1,311 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +//! Shared HTML processing for MkDocs-compatible plugins. + +use html5gum::emitters::callback::{CallbackEmitter, CallbackEvent}; +use html5gum::{Span, Tokenizer}; +use std::convert::Infallible; +use std::ops::Range; + +// ---------------------------------------------------------------------------- +// Traits +// ---------------------------------------------------------------------------- + +/// Page-local observer participating in the shared HTML pass. +pub(crate) trait Visitor { + /// Observes one tokenizer event and optionally records an output edit. + fn visit( + &mut self, event: &CallbackEvent<'_>, span: Span, + editor: &mut Editor<'_>, + ); +} + +// ---------------------------------------------------------------------------- +// Structs +// ---------------------------------------------------------------------------- + +/// Deferred edits to the HTML currently being scanned. +pub(crate) struct Editor<'a> { + /// Original HTML input. + input: &'a str, + /// Edits recorded by visitors. + edits: Vec, +} + +/// One replacement in the original HTML input. +#[derive(Debug, PartialEq, Eq)] +struct Edit { + /// Byte range replaced by this edit. + range: Range, + /// Replacement HTML. + replacement: Box, +} + +// ---------------------------------------------------------------------------- +// Implementations +// ---------------------------------------------------------------------------- + +impl<'a> Editor<'a> { + /// Creates an editor for an HTML input. + fn new(input: &'a str) -> Self { + Self { input, edits: Vec::new() } + } + + /// Returns original HTML covered by a tokenizer span. + pub(crate) fn text(&self, range: Range) -> &str { + &self.input[range] + } + + /// Replaces a byte range after all visitors have observed the input. + pub(crate) fn replace( + &mut self, range: Range, replacement: impl Into>, + ) { + assert!(range.start <= range.end && range.end <= self.input.len()); + self.edits.push(Edit { + range, + replacement: replacement.into(), + }); + } + + /// Removes the complete attribute whose name occupies `span`. + pub(crate) fn remove_attribute(&mut self, name: &[u8], span: Span) { + let bytes = self.input.as_bytes(); + assert!(span.start <= span.end && span.end <= bytes.len()); + + // Attribute-name spans exclude the whitespace preceding the name. + // Consume it so removing an attribute doesn't leave malformed or + // needlessly expanded start tags behind. + let mut start = span.start; + while start > 0 && is_whitespace(bytes[start - 1]) { + start -= 1; + } + + // Attribute-value spans exclude whitespace, the equals sign, and + // quotes. Recover that syntax directly from the original input so + // boolean, quoted, and unquoted attributes share the same operation. + // html5gum's attribute-name end can point at the byte that caused the + // tokenizer to flush the name. The decoded name length gives us the + // exact boundary for the ASCII compatibility attributes we remove. + let mut end = span.start + name.len(); + let mut equals = end; + skip_whitespace(bytes, &mut equals); + if bytes.get(equals) == Some(&b'=') { + end = equals + 1; + skip_whitespace(bytes, &mut end); + match bytes.get(end).copied() { + Some(quote @ (b'\'' | b'"')) => { + end += 1; + while end < bytes.len() && bytes[end] != quote { + end += 1; + } + if end < bytes.len() { + end += 1; + } + } + Some(_) => { + while end < bytes.len() + && !is_whitespace(bytes[end]) + && bytes[end] != b'>' + { + end += 1; + } + } + None => {} + } + } + + self.replace(start..end, Box::default()); + } + + /// Applies all deferred edits in one linear output pass. + fn finish(mut self) -> Option { + if self.edits.is_empty() { + return None; + } + + // Outer edits sort before edits they contain. An outer replacement + // owns its complete input span, while partially overlapping edits are + // always a programming error between visitors. + self.edits.sort_by(|left, right| { + left.range + .start + .cmp(&right.range.start) + .then_with(|| right.range.end.cmp(&left.range.end)) + }); + + let mut edits: Vec = Vec::with_capacity(self.edits.len()); + for edit in self.edits { + if let Some(previous) = edits.last() { + if edit.range.start < previous.range.end { + assert!( + edit.range.end <= previous.range.end, + "HTML edits partially overlap" + ); + if edit.range == previous.range { + assert_eq!( + edit.replacement, previous.replacement, + "HTML edits disagree on the same span" + ); + } + continue; + } + } + edits.push(edit); + } + + let removed = edits.iter().map(|edit| edit.range.len()).sum::(); + let inserted = edits + .iter() + .map(|edit| edit.replacement.len()) + .sum::(); + let mut output = String::with_capacity( + self.input.len().saturating_sub(removed) + inserted, + ); + let mut cursor = 0; + for edit in edits { + assert!(self.input.is_char_boundary(edit.range.start)); + assert!(self.input.is_char_boundary(edit.range.end)); + output.push_str(&self.input[cursor..edit.range.start]); + output.push_str(&edit.replacement); + cursor = edit.range.end; + } + output.push_str(&self.input[cursor..]); + Some(output) + } +} + +// ---------------------------------------------------------------------------- +// Functions +// ---------------------------------------------------------------------------- + +/// Scans HTML once with all page-local visitors. +/// +/// Returns modified HTML only when a visitor recorded an edit, allowing the +/// caller to retain the original allocation for observational passes. +pub(crate) fn scan( + input: &str, visitors: &mut [&mut dyn Visitor], +) -> Option { + let mut editor = Editor::new(input); + { + let mut emitter = CallbackEmitter::new( + |event: CallbackEvent<'_>, span: Span| { + for visitor in &mut *visitors { + visitor.visit(&event, span, &mut editor); + } + None:: + }, + ); + emitter.naively_switch_states(true); + + Tokenizer::new_with_emitter(input, emitter) + .finish() + .expect("string input is infallible"); + } + editor.finish() +} + +/// Returns whether a byte is HTML whitespace. +fn is_whitespace(byte: u8) -> bool { + matches!(byte, b'\t' | b'\n' | 0x0c | b'\r' | b' ') +} + +/// Advances an offset past HTML whitespace. +fn skip_whitespace(bytes: &[u8], offset: &mut usize) { + while bytes.get(*offset).is_some_and(|byte| is_whitespace(*byte)) { + *offset += 1; + } +} + +// ---------------------------------------------------------------------------- +// Tests +// ---------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Default)] + struct RemoveDataAttribute; + + #[derive(Default)] + struct ReplaceElement { + start: Option, + } + + impl Visitor for RemoveDataAttribute { + fn visit( + &mut self, event: &CallbackEvent<'_>, span: Span, + editor: &mut Editor<'_>, + ) { + if let CallbackEvent::AttributeName { name } = event + && *name == b"data-remove" + { + editor.remove_attribute(name, span); + } + } + } + + impl Visitor for ReplaceElement { + fn visit( + &mut self, event: &CallbackEvent<'_>, span: Span, + editor: &mut Editor<'_>, + ) { + match event { + CallbackEvent::OpenStartTag { name } if *name == b"replace" => { + self.start = Some(span.start); + } + CallbackEvent::EndTag { name } if *name == b"replace" => { + let start = self.start.take().expect("start tag"); + editor.replace(start..span.end, "slot"); + } + _ => {} + } + } + } + + fn remove(input: &str) -> Option { + let mut visitor = RemoveDataAttribute; + scan(input, &mut [&mut visitor]) + } + + #[test] + fn retains_the_original_allocation_without_edits() { + assert_eq!(remove("

Text

"), None); + } + + #[test] + fn removes_boolean_and_quoted_attributes() { + let input = concat!( + r#"
A
"#, + r#"
B
"#, + ); + assert_eq!( + remove(input).as_deref(), + Some(r#"
A
B
"#) + ); + } + + #[test] + fn removes_multiline_and_unquoted_attributes() { + let input = "Text"; + assert_eq!( + remove(input).as_deref(), + Some("Text") + ); + } + + #[test] + fn outer_replacements_own_contained_attribute_edits() { + let input = "Text"; + let mut remove = RemoveDataAttribute; + let mut replace = ReplaceElement::default(); + + assert_eq!( + scan(input, &mut [&mut remove, &mut replace]).as_deref(), + Some("slot") + ); + } +} diff --git a/crates/zensical/src/compat/mkdocs/plugin.rs b/crates/zensical/src/compat/mkdocs/plugin.rs new file mode 100644 index 0000000..b6b0003 --- /dev/null +++ b/crates/zensical/src/compat/mkdocs/plugin.rs @@ -0,0 +1,94 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +//! MkDocs-compatible plugins. + +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +use super::html::{self, Visitor}; +use crate::config::Config; +use crate::structure::markdown::Markdown; + +pub mod autorefs; +pub mod mkdocstrings; +pub mod search; + +// ---------------------------------------------------------------------------- +// Structs +// ---------------------------------------------------------------------------- + +/// Cached facts produced by the shared Markdown HTML pass. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +pub(crate) struct HtmlFacts { + /// Page-local autoref placeholders replaced with stable slots. + pub autorefs: Arc, + /// Page-local search sections. + pub search: Arc, +} + +// ---------------------------------------------------------------------------- + +/// Enabled MkDocs-compatible participants in the shared Markdown HTML pass. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Settings { + /// Whether autorefs extraction and settlement are active. + pub autorefs: bool, + /// Whether search extraction is active. + search: bool, +} + +// ---------------------------------------------------------------------------- +// Implementations +// ---------------------------------------------------------------------------- + +impl Settings { + /// Derives active compatibility participants from resolved configuration. + pub(crate) fn new(config: &Config) -> Self { + Self { + autorefs: autorefs::is_enabled(config), + search: config.project.plugins.search.config.enabled, + } + } +} + +// ---------------------------------------------------------------------------- +// Functions +// ---------------------------------------------------------------------------- + +/// Runs enabled MkDocs-compatible visitors in one page-local HTML pass. +pub(crate) fn prepare( + markdown: &mut Markdown, settings: Settings, +) -> HtmlFacts { + let mut autorefs = autorefs::Parser::default(); + let mut search = search::parser(&markdown.meta); + + let content = match (settings.search, settings.autorefs) { + (true, true) => { + let mut visitors: [&mut dyn Visitor; 2] = + [&mut search, &mut autorefs]; + html::scan(&markdown.content, &mut visitors) + } + (true, false) => html::scan(&markdown.content, &mut [&mut search]), + (false, true) => html::scan(&markdown.content, &mut [&mut autorefs]), + (false, false) => None, + }; + if let Some(content) = content { + markdown.replace_content(content); + } + + HtmlFacts { + autorefs: if settings.autorefs { + Arc::new(autorefs.finish()) + } else { + Arc::default() + }, + search: if settings.search { + search::finish(search) + } else { + Arc::default() + }, + } +} diff --git a/crates/zensical/src/structure/markdown/autorefs.rs b/crates/zensical/src/compat/mkdocs/plugin/autorefs.rs similarity index 57% rename from crates/zensical/src/structure/markdown/autorefs.rs rename to crates/zensical/src/compat/mkdocs/plugin/autorefs.rs index 13bb2f5..e4fa2ce 100644 --- a/crates/zensical/src/structure/markdown/autorefs.rs +++ b/crates/zensical/src/compat/mkdocs/plugin/autorefs.rs @@ -23,27 +23,33 @@ // ---------------------------------------------------------------------------- -//! Autorefs (mkdocstrings). +//! MkDocs-compatible autorefs plugin. -use ahash::{HashMap, HashSet}; -use pyo3::FromPyObject; -use regex::{Captures, Regex}; +use ahash::HashMap; +use pyo3::types::PyAnyMethods; +use pyo3::{FromPyObject, Python}; use serde::{Deserialize, Serialize}; +use std::fs; use std::path::Path; use std::string::ToString; -use std::sync::LazyLock; +use std::sync::Arc; +use zrx::id::Id; use zrx::path::PathExt; -use zrx::stream::Value; +use zrx::stream::{Key, Value}; + +use crate::compat::mkdocs::html; +use crate::config::Config; +use crate::structure::nav::file_sort_key; + +mod parser; + +pub(crate) use parser::{Parser, References}; +use parser::{Reference, SLOT_PREFIX, SLOT_SUFFIX}; // ---------------------------------------------------------------------------- // Constants // ---------------------------------------------------------------------------- -/// Autoref regex. -static AUTOREF_RE: LazyLock = LazyLock::new(|| { - Regex::new(r".*?)>(?P.*?)</autoref>").unwrap() -}); - /// Handled autoref attributes that should not be passed through to the output link. const HANDLED_ATTRS: &[&str] = &[ "identifier", @@ -60,6 +66,9 @@ const HANDLED_ATTRS: &[&str] = &[ "backlink-anchor", ]; +/// Python Markdown extension that produces autorefs compatibility facts. +const EXTENSION_NAME: &str = "zensical.extensions.autorefs"; + // ---------------------------------------------------------------------------- // Helper Functions // ---------------------------------------------------------------------------- @@ -211,6 +220,37 @@ pub struct UnresolvedAutorefs { // ---------------------------------------------------------------------------- +/// Shared immutable registry used to resolve page-local autorefs. +#[derive(Clone, Debug)] +pub(crate) struct Registry(Option<Arc<Autorefs>>); + +// ---------------------------------------------------------------------------- + +/// Autoref registrations produced while rendering one Markdown page. +#[derive( + Clone, Debug, Default, FromPyObject, Serialize, Deserialize, PartialEq, Eq, +)] +#[pyo3(from_item_all)] +pub(crate) struct Facts { + /// Primary page-local URLs. + primary: HashMap<String, Vec<String>>, + /// Secondary page-local URLs. + secondary: HashMap<String, Vec<String>>, + /// Titles for page-local URLs. + titles: HashMap<String, String>, +} + +// ---------------------------------------------------------------------------- + +/// Cached global inventory URLs supplied by mkdocstrings handlers. +#[derive(Debug, Default, Serialize, Deserialize)] +struct InventoryCache { + /// Absolute inventory URLs. + inventory: HashMap<String, String>, +} + +// ---------------------------------------------------------------------------- + /// Autorefs (mkdocstrings). /// /// We use three URL maps, one for "primary" URLs, one for "secondary" URLs, @@ -246,22 +286,16 @@ pub struct UnresolvedAutorefs { /// - Multiple secondary URLs mapped to an identifier? Use the first one, or closest one if configured as such. /// - No secondary URL mapped to an identifier? Try using absolute URLs /// (typically registered by loading inventories in mkdocstrings). -#[derive( - Clone, Debug, Default, FromPyObject, Serialize, Deserialize, PartialEq, Eq, -)] -#[pyo3(from_item_all)] -pub struct Autorefs { +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct Autorefs { // Primary URLs. - pub primary: HashMap<String, Vec<String>>, + primary: HashMap<String, Vec<String>>, // Secondary URLs. - pub secondary: HashMap<String, Vec<String>>, + secondary: HashMap<String, Vec<String>>, // Inventory URLs. - pub inventory: HashMap<String, String>, + inventory: HashMap<String, String>, // Titles. - pub titles: HashMap<String, String>, - // Pages reprocessed since autorefs data was last collected. - #[serde(skip)] - pub updated_pages: Vec<String>, + titles: HashMap<String, String>, } // ---------------------------------------------------------------------------- @@ -274,124 +308,11 @@ impl Autorefs { Self::default() } - /// Merges another `Autorefs` into this one. - /// - /// Entries from `other` take precedence over existing entries. Used to - /// merge stale cached autorefs with fresh data from the Python process, - /// so that pages not re-processed in the current build retain their - /// previously registered identifiers while newly processed pages override - /// any stale entries. - pub fn merge(&mut self, other: Autorefs) { - self.primary.extend(other.primary); - self.secondary.extend(other.secondary); - self.inventory.extend(other.inventory); - self.titles.extend(other.titles); - } - - /// Removes registrations owned by any of the given pages. - pub(crate) fn remove_pages(&mut self, pages: &HashSet<String>) { - self.retain_page_urls(|page| !pages.contains(page)); - } - - /// Retains registrations owned by one of the given pages. - pub(crate) fn retain_pages(&mut self, pages: &HashSet<String>) { - self.retain_page_urls(|page| pages.contains(page)); - } - - /// Retains registrations whose page URL matches the predicate. - fn retain_page_urls(&mut self, predicate: impl Fn(&str) -> bool) { - retain_url_map(&mut self.primary, &predicate); - retain_url_map(&mut self.secondary, &predicate); - self.titles - .retain(|url, _| predicate(page_url_from_autoref(url))); - } - - /// Parses HTML attributes string into a HashMap. - /// - /// @todo Document that this is not the most resilient HTML parser - /// but since we control the autorefs elements, it's fine for now - fn parse_attributes(attrs_str: &str) -> HashMap<String, String> { - let mut attrs = HashMap::default(); - let mut chars = attrs_str.chars().peekable(); - - while let Some(ch) = chars.peek() { - // Skip whitespace - if ch.is_whitespace() { - chars.next(); - continue; - } - - // Parse attribute name - let mut name = String::new(); - while let Some(&ch) = chars.peek() { - if ch.is_whitespace() || ch == '=' { - break; - } - name.push(ch); - chars.next(); - } - - if name.is_empty() { - break; - } - - // Skip whitespace - while let Some(&ch) = chars.peek() { - if !ch.is_whitespace() { - break; - } - chars.next(); - } - - // Check for '=' - let has_value = chars.peek() == Some(&'='); - if has_value { - chars.next(); // consume '=' - - // Skip whitespace after '=' - while let Some(&ch) = chars.peek() { - if !ch.is_whitespace() { - break; - } - chars.next(); - } - - // Parse value - let value = if let Some("e) = chars.peek() { - if quote == '"' || quote == '\'' { - chars.next(); // consume opening quote - let mut val = String::new(); - for ch in chars.by_ref() { - if ch == quote { - break; // consume closing quote - } - val.push(ch); - } - val - } else { - // Unquoted value - let mut val = String::new(); - while let Some(&ch) = chars.peek() { - if ch.is_whitespace() { - break; - } - val.push(ch); - chars.next(); - } - val - } - } else { - String::new() - }; - - attrs.insert(name, value); - } else { - // Boolean attribute - attrs.insert(name, String::new()); - } - } - - attrs + /// Merge one page's registrations into the complete registry. + fn merge(&mut self, facts: &Facts) { + merge_url_map(&mut self.primary, &facts.primary); + merge_url_map(&mut self.secondary, &facts.secondary); + self.titles.extend(facts.titles.clone()); } /// Resolves the URL for an item identifier (internal implementation). @@ -479,125 +400,160 @@ impl Autorefs { )) } - /// Replaces autorefs and collects unresolved identifiers. + /// Renders one parsed autoref against the settled registry. #[allow(clippy::single_match_else)] - pub fn replace_in<S>( - &self, content: S, from_url: &str, + fn render( + &self, reference: &Reference, from_url: &str, + unresolved: &mut UnresolvedAutorefs, + ) -> String { + let title = reference.title(); + let identifier = reference.get("identifier").unwrap_or_default(); + let slug = reference.get("slug").unwrap_or_default(); + let optional = reference.contains("optional"); + let identifiers = if slug.is_empty() { + vec![identifier.to_string()] + } else { + vec![identifier.to_string(), slug.to_string()] + }; + + match self.get_url_and_title_from_ids(&identifiers, from_url) { + Ok((url, original_title)) => { + let external = !is_relative_url(&url); + let mut classes = vec![ + "autorefs".to_string(), + if external { + "autorefs-external".to_string() + } else { + "autorefs-internal".to_string() + }, + ]; + if let Some(class) = reference.get("class") { + classes.extend( + class.split_whitespace().map(ToString::to_string), + ); + } + let class = classes.join(" "); + + // Pass unknown attributes through in source order. html5gum + // decodes their values, so escape them when serializing. + let remaining = reference + .attributes() + .filter(|(name, _)| !HANDLED_ATTRS.contains(name)) + .map(|(name, value)| { + if value.is_empty() { + name.to_string() + } else { + format!("{name}=\"{}\"", html_escape(value)) + } + }) + .collect::<Vec<_>>(); + let remaining = if remaining.is_empty() { + String::new() + } else { + format!(" {}", remaining.join(" ")) + }; + + let tooltip = if optional { + original_title.as_deref().unwrap_or(identifier) + } else { + original_title.as_deref().unwrap_or_default() + }; + let title_attr = if !tooltip.is_empty() + && !format!("<code>{title}</code>").contains(tooltip) + { + format!(" title=\"{}\"", html_escape(tooltip)) + } else { + String::new() + }; + + format!( + "<a class=\"{class}\"{title_attr} href=\"{}\"{remaining}>{title}</a>", + html_escape(&url) + ) + } + Err(_) => { + if optional { + format!("<span title=\"{identifier}\">{title}</span>") + } else { + unresolved.insert(identifier); + if title == identifier { + format!("[{identifier}][]") + } else if title == format!("<code>{identifier}</code>") + && slug.is_empty() + { + format!("[<code>{identifier}</code>][]") + } else { + format!("[{title}][{identifier}]") + } + } + } + } + } + + /// Expands page-local slots in one linear pass. + fn replace_slots( + &self, content: String, references: &References, from_url: &str, + unresolved: &mut UnresolvedAutorefs, + ) -> String { + if references.is_empty() || !content.contains(SLOT_PREFIX) { + return content; + } + + let mut output = String::with_capacity(content.len()); + let mut cursor = 0; + while let Some(offset) = content[cursor..].find(SLOT_PREFIX) { + let start = cursor + offset; + let index_start = start + SLOT_PREFIX.len(); + let Some(offset) = content[index_start..].find(SLOT_SUFFIX) else { + break; + }; + let index_end = index_start + offset; + let end = index_end + SLOT_SUFFIX.len(); + + output.push_str(&content[cursor..start]); + if let Ok(index) = content[index_start..index_end].parse::<usize>() + && let Some(reference) = references.get(index) + { + output.push_str(&self.render(reference, from_url, unresolved)); + } else { + output.push_str(&content[start..end]); + } + cursor = end; + } + output.push_str(&content[cursor..]); + output + } + + /// Replaces cached slots and raw markers introduced by templates. + fn replace_in<S>( + &self, content: S, references: &References, from_url: &str, ) -> (String, UnresolvedAutorefs) where S: Into<String>, { - let content = content.into(); let mut unresolved = UnresolvedAutorefs::default(); - let output = AUTOREF_RE.replace_all(&content, |captures: &Captures| { - let attrs_str = - captures.name("attrs").map_or("", |m| m.as_str()); - let title = - captures.name("title").map_or("", |m| m.as_str()); + let mut content = self.replace_slots( + content.into(), + references, + from_url, + &mut unresolved, + ); - // Parse the HTML attributes - let attrs = Self::parse_attributes(attrs_str); - let identifier = - attrs.get("identifier").cloned().unwrap_or_default(); - let slug = attrs.get("slug").cloned().unwrap_or_default(); - let optional = attrs.contains_key("optional"); + // Templates may introduce autorefs after the cached Markdown pass. + // Autorefs therefore participates in the deliberate final HTML pass + // whenever it is enabled, using the same visitor and slot expansion + // path as page-produced markers. + let mut parser = Parser::default(); + if let Some(prepared) = html::scan(&content, &mut [&mut parser]) { + content = self.replace_slots( + prepared, + &parser.finish(), + from_url, + &mut unresolved, + ); + } - let identifiers = if slug.is_empty() { - vec![identifier.clone()] - } else { - vec![identifier.clone(), slug.clone()] - }; - - match self.get_url_and_title_from_ids(&identifiers, from_url) { - Ok((url, original_title)) => { - // Check if URL is external (not relative) - let external = !is_relative_url(&url); - - // Build CSS classes - let mut classes = vec![ - "autorefs".to_string(), - if external { - "autorefs-external".to_string() - } else { - "autorefs-internal".to_string() - }, - ]; - - // Add existing classes from attrs - if let Some(class_str) = attrs.get("class") { - classes.extend( - class_str - .split_whitespace() - .map(ToString::to_string), - ); - } - let class_attr = classes.join(" "); - - // Build remaining attributes (those not in the handled set) - let remaining_attrs: Vec<String> = attrs - .iter() - .filter(|(k, _)| !HANDLED_ATTRS.contains(&k.as_str())) - .map(|(k, v)| { - if v.is_empty() { - // Boolean attribute (no value) - k.clone() - } else { - // Attribute with value - format!("{k}=\"{v}\"") - } - }) - .collect(); - - let remaining = if remaining_attrs.is_empty() { - String::new() - } else { - format!(" {}", remaining_attrs.join(" ")) - }; - - // Build title attribute (link_titles is always true, strip_title_tags is always false) - let tooltip = if optional { - // For optional, we use identifier as fallback if no original_title - original_title.as_deref().unwrap_or(&identifier).to_string() - } else { - // For non-optional, use original_title or empty - original_title.as_deref().unwrap_or("").to_string() - }; - - let title_attr = if !tooltip.is_empty() && !format!("<code>{title}</code>").contains(&tooltip) { - format!(" title=\"{}\"", html_escape(&tooltip)) - } else { - String::new() - }; - - let escaped_url = html_escape(&url); - format!( - "<a class=\"{class_attr}\"{title_attr} href=\"{escaped_url}\"{remaining}>{title}</a>" - ) - } - Err(_) => { - if optional { - format!("<span title=\"{identifier}\">{title}</span>") - } else { - unresolved.insert(&identifier); - if title == identifier { - format!("[{identifier}][]") - } else if title == format!("<code>{identifier}</code>") - && slug.is_empty() - { - format!("[<code>{identifier}</code>][]") - } else { - format!("[{title}][{identifier}]") - } - } - } - } - }); - - let output = match output { - std::borrow::Cow::Borrowed(_) => content, - std::borrow::Cow::Owned(output) => output, - }; - (output, unresolved) + (content, unresolved) } } @@ -605,6 +561,28 @@ impl Autorefs { // Trait implementations // ---------------------------------------------------------------------------- +impl Registry { + /// Replace autoref placeholders using this immutable registry. + pub(crate) fn replace_in<S>( + &self, content: S, references: &References, from_url: &str, + ) -> (String, UnresolvedAutorefs) + where + S: Into<String>, + { + if let Some(autorefs) = &self.0 { + autorefs.replace_in(content, references, from_url) + } else { + (content.into(), UnresolvedAutorefs::default()) + } + } +} + +// ---------------------------------------------------------------------------- + +impl Value for Registry {} + +// ---------------------------------------------------------------------------- + impl Value for UnresolvedAutorefs {} // ---------------------------------------------------------------------------- @@ -629,25 +607,122 @@ impl UnresolvedAutorefs { // Functions // ---------------------------------------------------------------------------- -/// Retain URL-map entries based on their owning page URL. -fn retain_url_map( - map: &mut HashMap<String, Vec<String>>, predicate: &impl Fn(&str) -> bool, -) { - map.retain(|_, urls| { - urls.retain(|url| predicate(page_url_from_autoref(url))); - !urls.is_empty() - }); +/// Assemble a complete immutable registry from settled page-local facts. +pub(crate) fn assemble( + config: &Config, mut facts: Vec<(Key<Id>, Arc<Facts>)>, +) -> Registry { + if !is_enabled(config) { + return Registry(None); + } + + facts.sort_by_key(|(key, _)| file_sort_key(&key[0])); + + let mut registry = Autorefs::new(); + for (_, facts) in facts { + registry.merge(&facts); + } + registry.inventory = inventory(&config.get_cache_dir()); + Registry(Some(Arc::new(registry))) } -/// Return the page URL portion of an autoref URL. -fn page_url_from_autoref(url: &str) -> &str { - url.split_once('#').map_or(url, |(page, _)| page) +/// Returns whether autorefs is active after configuration shims are applied. +pub(super) fn is_enabled(config: &Config) -> bool { + config.has_markdown_extension(EXTENSION_NAME) +} + +/// Collect and cache global inventory URLs supplied by mkdocstrings. +fn inventory(cache_dir: &Path) -> HashMap<String, String> { + let path = cache_dir.join("autorefs.json"); + let mut cache = fs::read(&path) + .ok() + .and_then(|data| serde_json::from_slice::<InventoryCache>(&data).ok()) + .unwrap_or_default(); + + // An absent value means all pages came from the Markdown cache and Python + // never loaded mkdocstrings handlers. An empty map means rendering ran and + // no external inventory is configured, so it deliberately clears cache. + if let Some(inventory) = collect_inventory() { + cache.inventory = inventory; + } + + if let Ok(data) = serde_json::to_vec_pretty(&cache) { + let _ = fs::create_dir_all(cache_dir); + let _ = fs::write(path, data); + } + cache.inventory +} + +/// Take registrations produced by the most recently rendered page. +pub(crate) fn take_page(url: &str) -> Arc<Facts> { + Arc::new( + Python::attach(|py| { + let module = py.import("zensical.extensions.autorefs")?; + module + .call_method1("get_autorefs_page_data", (url,))? + .extract::<Facts>() + }) + .unwrap_or_default(), + ) +} + +/// Collect global inventory URLs if Python rendered at least one page. +fn collect_inventory() -> Option<HashMap<String, String>> { + Python::attach(|py| { + let module = py.import("zensical.extensions.autorefs")?; + module + .call_method0("get_autorefs_inventory_data")? + .extract::<Option<HashMap<String, String>>>() + }) + .unwrap_or_default() +} + +/// Merge URL lists while preserving registration order and uniqueness. +fn merge_url_map( + target: &mut HashMap<String, Vec<String>>, + source: &HashMap<String, Vec<String>>, +) { + for (identifier, urls) in source { + let target = target.entry(identifier.clone()).or_default(); + for url in urls { + if !target.contains(url) { + target.push(url.clone()); + } + } + } } #[cfg(test)] mod tests { use super::*; + fn prepare(input: &str) -> (String, References) { + let mut parser = Parser::default(); + let content = html::scan(input, &mut [&mut parser]) + .unwrap_or_else(|| input.to_string()); + (content, parser.finish()) + } + + #[test] + fn page_facts_merge_without_overwriting_shared_identifiers() { + let mut autorefs = Autorefs::new(); + autorefs.merge(&Facts { + primary: HashMap::from_iter([( + "shared".to_string(), + vec!["one/#shared".to_string()], + )]), + ..Default::default() + }); + autorefs.merge(&Facts { + primary: HashMap::from_iter([( + "shared".to_string(), + vec!["two/#shared".to_string()], + )]), + ..Default::default() + }); + + assert_eq!(autorefs.primary["shared"], ["one/#shared", "two/#shared"]); + } + #[test] fn test_resolve_closest_url() { let test_cases = vec![ @@ -726,6 +801,7 @@ mod tests { "<autoref identifier=\"missing\">Missing</autoref>", "<autoref identifier=\"skipped\" optional>Skipped</autoref>", ), + &References::default(), "guide/", ); @@ -735,4 +811,60 @@ mod tests { assert!(output.contains("[Missing][missing]")); assert!(output.contains("<span title=\"skipped\">Skipped</span>")); } + + #[test] + fn cached_slots_preserve_autoref_rendering_contract() { + let mut autorefs = Autorefs::new(); + autorefs + .primary + .insert("known".to_string(), vec!["reference/#known".to_string()]); + autorefs.titles.insert( + "reference/#known".to_string(), + "Canonical title".to_string(), + ); + let (content, references) = prepare(concat!( + "<autoref identifier=\"known\" class=\"custom\" ", + "data-kind=\"a&b\" download>", + "<code>Known</code></autoref>", + )); + + let (output, unresolved) = + autorefs.replace_in(content, &references, "guide/"); + + assert_eq!( + output, + concat!( + "<a class=\"autorefs autorefs-internal custom\" ", + "title=\"Canonical title\" ", + "href=\"../reference/#known\" ", + "data-kind=\"a&b\" download>", + "<code>Known</code></a>", + ) + ); + assert!(unresolved.iter().next().is_none()); + } + + #[test] + fn slug_is_used_as_a_resolution_fallback() { + let mut autorefs = Autorefs::new(); + autorefs.primary.insert( + "foo-bar".to_string(), + vec!["reference/#foo-bar".to_string()], + ); + let (content, references) = prepare( + "<autoref identifier=\"Foo bar\" slug=\"foo-bar\">Foo bar</autoref>", + ); + + let (output, unresolved) = + autorefs.replace_in(content, &references, "guide/"); + + assert_eq!( + output, + concat!( + "<a class=\"autorefs autorefs-internal\" ", + "href=\"../reference/#foo-bar\">Foo bar</a>", + ) + ); + assert!(unresolved.iter().next().is_none()); + } } diff --git a/crates/zensical/src/compat/mkdocs/plugin/autorefs/parser.rs b/crates/zensical/src/compat/mkdocs/plugin/autorefs/parser.rs new file mode 100644 index 0000000..a757d84 --- /dev/null +++ b/crates/zensical/src/compat/mkdocs/plugin/autorefs/parser.rs @@ -0,0 +1,271 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +//! Streaming extraction of MkDocs-compatible autoref placeholders. + +use html5gum::emitters::callback::CallbackEvent; +use html5gum::Span; +use serde::{Deserialize, Serialize}; + +use crate::compat::mkdocs::html::{Editor, Visitor}; + +// ---------------------------------------------------------------------------- +// Constants +// ---------------------------------------------------------------------------- + +/// Prefix of an internal page-local autoref slot. +pub(super) const SLOT_PREFIX: &str = "<!-- zensical:autoref:"; + +/// Suffix of an internal page-local autoref slot. +pub(super) const SLOT_SUFFIX: &str = " -->"; + +// ---------------------------------------------------------------------------- +// Structs +// ---------------------------------------------------------------------------- + +/// Autoref placeholders extracted from one rendered Markdown page. +#[derive( + Clone, Debug, Default, Deserialize, Hash, PartialEq, Eq, Serialize, +)] +pub(crate) struct References { + /// References in document order; their positions are stable slot IDs. + references: Vec<Reference>, +} + +/// One unresolved autoref placeholder. +#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)] +pub(super) struct Reference { + /// Attributes in their source order. + attributes: Vec<Attribute>, + /// Raw inner HTML used as link content. + title: String, +} + +/// One parsed HTML attribute. +#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)] +pub(super) struct Attribute { + /// Decoded attribute name. + name: String, + /// Decoded attribute value, or an empty string for boolean attributes. + value: String, +} + +/// Page-local autoref visitor. +#[derive(Default)] +pub(crate) struct Parser { + /// Autoref start tag currently being assembled. + pending: Option<Pending>, + /// Completed page-local references. + references: Vec<Reference>, +} + +/// Autoref element currently being assembled. +struct Pending { + /// Start of the complete element. + start: usize, + /// Start of its raw inner HTML after the start tag closes. + content: Option<usize>, + /// Parsed start-tag attributes. + attributes: Vec<Attribute>, + /// Attribute currently receiving a value. + attribute: Option<usize>, + /// Nested autoref elements, which are retained inside the outer title. + nested: usize, +} + +// ---------------------------------------------------------------------------- +// Implementations +// ---------------------------------------------------------------------------- + +impl References { + /// Returns the reference at a page-local slot index. + pub(super) fn get(&self, index: usize) -> Option<&Reference> { + self.references.get(index) + } + + /// Returns whether no page-local autorefs were extracted. + pub(crate) fn is_empty(&self) -> bool { + self.references.is_empty() + } +} + +// ---------------------------------------------------------------------------- + +impl Reference { + /// Returns the last value for an attribute, matching the old map parser. + pub(super) fn get(&self, name: &str) -> Option<&str> { + self.attributes + .iter() + .rev() + .find(|attribute| attribute.name == name) + .map(|attribute| attribute.value.as_str()) + } + + /// Returns whether an attribute is present. + pub(super) fn contains(&self, name: &str) -> bool { + self.attributes + .iter() + .any(|attribute| attribute.name == name) + } + + /// Iterates over parsed attributes in source order. + pub(super) fn attributes(&self) -> impl Iterator<Item = (&str, &str)> { + self.attributes.iter().map(|attribute| { + (attribute.name.as_str(), attribute.value.as_str()) + }) + } + + /// Returns raw inner HTML. + pub(super) fn title(&self) -> &str { + &self.title + } +} + +// ---------------------------------------------------------------------------- + +impl Parser { + /// Converts the visitor into cached page-local references. + pub(crate) fn finish(self) -> References { + References { references: self.references } + } + + /// Handles one tokenizer event. + fn handle( + &mut self, event: &CallbackEvent<'_>, span: Span<usize>, + editor: &mut Editor<'_>, + ) { + match event { + CallbackEvent::OpenStartTag { name } if *name == b"autoref" => { + if let Some(pending) = &mut self.pending { + pending.nested += 1; + } else { + self.pending = Some(Pending { + start: span.start, + content: None, + attributes: Vec::new(), + attribute: None, + nested: 0, + }); + } + } + CallbackEvent::AttributeName { name } => { + if let Some(pending) = &mut self.pending + && pending.content.is_none() + { + pending.attributes.push(Attribute { + name: String::from_utf8_lossy(name).into_owned(), + value: String::new(), + }); + pending.attribute = Some(pending.attributes.len() - 1); + } + } + CallbackEvent::AttributeValue { value } => { + if let Some(pending) = &mut self.pending + && pending.content.is_none() + && let Some(index) = pending.attribute + { + pending.attributes[index].value = + String::from_utf8_lossy(value).into_owned(); + } + } + CallbackEvent::CloseStartTag { self_closing } => { + if let Some(pending) = &mut self.pending + && pending.content.is_none() + { + if *self_closing { + self.pending = None; + } else { + pending.content = Some(span.end); + } + } + } + CallbackEvent::EndTag { name } if *name == b"autoref" => { + let Some(mut pending) = self.pending.take() else { + return; + }; + if pending.nested > 0 { + pending.nested -= 1; + self.pending = Some(pending); + return; + } + + let Some(content) = pending.content else { + return; + }; + let index = self.references.len(); + self.references.push(Reference { + attributes: pending.attributes, + title: editor.text(content..span.start).to_string(), + }); + editor.replace(pending.start..span.end, slot(index)); + } + _ => {} + } + } +} + +// ---------------------------------------------------------------------------- +// Trait implementations +// ---------------------------------------------------------------------------- + +impl Visitor for Parser { + fn visit( + &mut self, event: &CallbackEvent<'_>, span: Span<usize>, + editor: &mut Editor<'_>, + ) { + self.handle(event, span, editor); + } +} + +// ---------------------------------------------------------------------------- +// Functions +// ---------------------------------------------------------------------------- + +/// Creates the stable marker for a page-local autoref slot. +fn slot(index: usize) -> String { + format!("{SLOT_PREFIX}{index}{SLOT_SUFFIX}") +} + +// ---------------------------------------------------------------------------- +// Tests +// ---------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::compat::mkdocs::html; + + #[test] + fn extracts_attributes_and_raw_inner_html() { + let input = concat!( + "<p>Before ", + "<autoref\n identifier='Foo & Bar' optional>", + "<code>Foo & Bar</code>", + "</autoref> after</p>", + ); + let mut parser = Parser::default(); + let output = html::scan(input, &mut [&mut parser]).expect("slot edit"); + let references = parser.finish(); + let reference = references.get(0).expect("reference"); + + assert_eq!( + output, + format!("<p>Before {SLOT_PREFIX}0{SLOT_SUFFIX} after</p>") + ); + assert_eq!(reference.get("identifier"), Some("Foo & Bar")); + assert!(reference.contains("optional")); + assert_eq!(reference.title(), "<code>Foo & Bar</code>"); + } + + #[test] + fn leaves_unclosed_and_self_closing_elements_untouched() { + for input in ["<autoref identifier=x>Title", "<autoref identifier=x/>"] + { + let mut parser = Parser::default(); + assert_eq!(html::scan(input, &mut [&mut parser]), None); + assert!(parser.finish().is_empty()); + } + } +} diff --git a/crates/zensical/src/compat/mkdocs/mkdocstrings.rs b/crates/zensical/src/compat/mkdocs/plugin/mkdocstrings.rs similarity index 97% rename from crates/zensical/src/compat/mkdocs/mkdocstrings.rs rename to crates/zensical/src/compat/mkdocs/plugin/mkdocstrings.rs index ecc5ba3..c35d819 100644 --- a/crates/zensical/src/compat/mkdocs/mkdocstrings.rs +++ b/crates/zensical/src/compat/mkdocs/plugin/mkdocstrings.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT // All contributions are certified under the DCO -//! Mkdocstrings compatibility artifacts. +//! Mkdocstrings compatibility plugin. use pyo3::types::PyAnyMethods; use pyo3::Python; diff --git a/crates/zensical/src/compat/mkdocs/search.rs b/crates/zensical/src/compat/mkdocs/plugin/search.rs similarity index 82% rename from crates/zensical/src/compat/mkdocs/search.rs rename to crates/zensical/src/compat/mkdocs/plugin/search.rs index c6a394d..f6d1545 100644 --- a/crates/zensical/src/compat/mkdocs/search.rs +++ b/crates/zensical/src/compat/mkdocs/plugin/search.rs @@ -23,12 +23,13 @@ // ---------------------------------------------------------------------------- -//! MkDocs-compatible search index. +//! MkDocs-compatible search plugin. -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fs; use std::io::{BufWriter, Write}; +use std::sync::Arc; use zrx::id::Id; use zrx::scheduler::Value; use zrx::stream::function::Collection; @@ -44,7 +45,7 @@ mod item; mod parser; use item::{SearchItem, SearchSection}; -pub(crate) use parser::extract; +use parser::Parser; // ---------------------------------------------------------------------------- // Structs @@ -68,17 +69,24 @@ struct SearchIndex { items: Vec<SearchItem>, } -/// Compact page facts retained by the search branch. +/// Search facts extracted while rendering one Markdown page. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +pub(crate) struct Facts { + /// Page-local search sections. + sections: Vec<SearchSection>, +} + +/// Compact page document retained by the search branch. #[derive(Clone, Debug, PartialEq, Eq)] -struct SearchDocument { +pub(crate) struct Document { /// Page target URL. url: String, /// Page title. title: String, /// Page tag names. tags: Vec<String>, - /// Page-local search sections. - sections: Vec<SearchSection>, + /// Page-local facts extracted before page construction. + facts: Arc<Facts>, } // ---------------------------------------------------------------------------- @@ -97,25 +105,34 @@ impl SearchConfig { // ---------------------------------------------------------------------------- -impl SearchDocument { - /// Extracts the facts search needs from a rendered page. - fn new(page: &Page) -> Self { +impl Document { + /// Attaches page properties to previously extracted search facts. + pub(crate) fn new(page: &Page, facts: Arc<Facts>) -> Self { Self { url: page.url.clone(), title: page.title.clone(), tags: page.tags().into_iter().map(|tag| tag.name).collect(), - sections: extract(&page.content), + facts, } } } // ---------------------------------------------------------------------------- +impl Facts { + /// Returns whether this page contributes anything to the search index. + pub(crate) fn is_empty(&self) -> bool { + self.sections.is_empty() + } +} + +// ---------------------------------------------------------------------------- + impl SearchIndex { /// Creates a search index from compact page facts. #[allow(clippy::assigning_clones)] fn new( - documents: Vec<(Key<Id>, SearchDocument)>, nav: &Navigation, + documents: Vec<(Key<Id>, Document)>, nav: &Navigation, config: SearchPluginConfig, language: &str, ) -> Self { let mut items: Vec<SearchItem> = Vec::new(); @@ -139,21 +156,21 @@ impl SearchIndex { path.push(document.title.clone()); } - for section in document.sections { - let location = match section.location { + for section in &document.facts.sections { + let location = match §ion.location { Some(id) => format!("{}#{}", document.url, id), _ => document.url.clone(), }; let title = if section.title.is_empty() { document.title.clone() } else { - section.title + section.title.clone() }; items.push(SearchItem { location: Some(location), level: section.level, title, - text: section.text, + text: section.text.clone(), path: path.clone(), tags: document.tags.clone(), }); @@ -172,7 +189,7 @@ impl SearchIndex { // Trait implementations // ---------------------------------------------------------------------------- -impl Value for SearchDocument {} +impl Value for Document {} // ---------------------------------------------------------------------------- // Functions @@ -180,7 +197,8 @@ impl Value for SearchDocument {} /// Attach MkDocs-compatible search artifact generation to the build graph. pub(crate) fn attach( - config: &Config, pages: &Stream<Id, Page>, nav: &Signal<Id, Navigation>, + config: &Config, documents: &Stream<Id, Document>, + nav: &Signal<Id, Navigation>, ) { if !config.project.plugins.search.config.enabled { let config = config.clone(); @@ -196,22 +214,18 @@ pub(crate) fn attach( return; } - let documents = pages - .filter(|page: &Page| !is_search_excluded(&page.meta)) - .map(SearchDocument::new); - let documents = documents.reduce( - |documents: &dyn Collection<Key<Id>, SearchDocument>| { + let documents = + documents.reduce(|documents: &dyn Collection<Key<Id>, Document>| { Some( documents .iter() .map(|(key, document)| (key.clone(), document.clone())) .collect::<Vec<_>>(), ) - }, - ); + }); let config = config.clone(); let _ = documents.product(nav).map( - move |documents: &Vec<(Key<Id>, SearchDocument)>, nav: &Navigation| { + move |documents: &Vec<(Key<Id>, Document)>, nav: &Navigation| { let search = SearchIndex::new( documents.clone(), nav, @@ -223,6 +237,20 @@ pub(crate) fn attach( ); } +/// Creates the page-local search visitor. +pub(crate) fn parser(meta: &BTreeMap<String, Dynamic>) -> Parser { + if is_search_excluded(meta) { + Parser::discarding() + } else { + Parser::default() + } +} + +/// Converts a completed visitor into cached page-local facts. +pub(crate) fn finish(parser: Parser) -> Arc<Facts> { + Arc::new(Facts { sections: parser.finish() }) +} + /// Write search artifacts without retaining a second serialized copy. fn write(config: &Config, search: &SearchIndex) -> anyhow::Result<()> { let site_dir = config.get_site_dir(); diff --git a/crates/zensical/src/compat/mkdocs/search/item.rs b/crates/zensical/src/compat/mkdocs/plugin/search/item.rs similarity index 94% rename from crates/zensical/src/compat/mkdocs/search/item.rs rename to crates/zensical/src/compat/mkdocs/plugin/search/item.rs index 05a4ef7..996fd7a 100644 --- a/crates/zensical/src/compat/mkdocs/search/item.rs +++ b/crates/zensical/src/compat/mkdocs/plugin/search/item.rs @@ -23,9 +23,9 @@ // ---------------------------------------------------------------------------- -//! MkDocs-compatible search item. +//! MkDocs-compatible search items. -use serde::Serialize; +use serde::{Deserialize, Serialize}; // ---------------------------------------------------------------------------- // Structs @@ -51,7 +51,7 @@ pub struct SearchItem { // ---------------------------------------------------------------------------- /// Page-local search section before site-wide facts are attached. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] pub(crate) struct SearchSection { /// Heading fragment, if present. pub location: Option<String>, diff --git a/crates/zensical/src/compat/mkdocs/search/parser.rs b/crates/zensical/src/compat/mkdocs/plugin/search/parser.rs similarity index 89% rename from crates/zensical/src/compat/mkdocs/search/parser.rs rename to crates/zensical/src/compat/mkdocs/plugin/search/parser.rs index 9f30ada..8c5f06d 100644 --- a/crates/zensical/src/compat/mkdocs/search/parser.rs +++ b/crates/zensical/src/compat/mkdocs/plugin/search/parser.rs @@ -5,40 +5,22 @@ //! MkDocs-compatible search extraction from rendered HTML. -use html5gum::emitters::callback::{CallbackEmitter, CallbackEvent}; -use html5gum::{Span, Tokenizer}; -use std::convert::Infallible; +use html5gum::emitters::callback::CallbackEvent; +use html5gum::Span; + +use crate::compat::mkdocs::html::{Editor, Visitor}; use super::SearchSection; -/// Extract page-local search sections from rendered HTML. -pub(crate) fn extract(html: &str) -> Vec<SearchSection> { - let mut parser = SearchParser::default(); - - { - let mut emitter = CallbackEmitter::new( - |event: CallbackEvent<'_>, _span: Span<()>| -> Option<Infallible> { - parser.handle(event); - None - }, - ); - emitter.naively_switch_states(true); - - Tokenizer::new_with_emitter(html, emitter) - .finish() - .expect("string input is infallible"); - } - - parser.finish() -} - // ---------------------------------------------------------------------------- // Parser // ---------------------------------------------------------------------------- /// Streaming search parser. #[derive(Default)] -struct SearchParser { +pub(crate) struct Parser { + /// Whether extraction is disabled for a page excluded through metadata. + discard: bool, /// Open HTML elements. context: Vec<Element>, /// Section currently receiving text. @@ -53,9 +35,29 @@ struct SearchParser { attribute: Attribute, } -impl SearchParser { +impl Parser { + /// Creates a parser that only applies search-related HTML cleanup. + pub(crate) fn discarding() -> Self { + Self { + discard: true, + ..Self::default() + } + } + /// Handles a tokenizer event. - fn handle(&mut self, event: CallbackEvent<'_>) { + fn handle( + &mut self, event: &CallbackEvent<'_>, span: Span<usize>, + editor: &mut Editor<'_>, + ) { + if let CallbackEvent::AttributeName { name } = event + && *name == b"data-search-exclude" + { + editor.remove_attribute(name, span); + } + if self.discard { + return; + } + match event { CallbackEvent::OpenStartTag { name } => { self.start = Some(StartTag::new(Tag::from_bytes(name))); @@ -76,7 +78,7 @@ impl SearchParser { if let Some(start) = self.start.take() { let tag = start.tag.clone(); self.start(start); - if self_closing { + if *self_closing { self.end(&tag); } } @@ -286,7 +288,7 @@ impl SearchParser { } /// Converts parser state into page-local search sections. - fn finish(self) -> Vec<SearchSection> { + pub(crate) fn finish(self) -> Vec<SearchSection> { self.sections .into_iter() .filter(|section| !section.excluded) @@ -300,6 +302,19 @@ impl SearchParser { } } +// ---------------------------------------------------------------------------- +// Trait implementations +// ---------------------------------------------------------------------------- + +impl Visitor for Parser { + fn visit( + &mut self, event: &CallbackEvent<'_>, span: Span<usize>, + editor: &mut Editor<'_>, + ) { + self.handle(event, span, editor); + } +} + // ---------------------------------------------------------------------------- // State // ---------------------------------------------------------------------------- @@ -621,6 +636,13 @@ fn trim(mut value: String) -> String { #[cfg(test)] mod tests { use super::*; + use crate::compat::mkdocs::html::scan; + + fn extract(html: &str) -> Vec<SearchSection> { + let mut parser = Parser::default(); + let _ = scan(html, &mut [&mut parser]); + parser.finish() + } fn item( location: Option<&str>, level: u32, title: &str, text: &str, @@ -684,6 +706,35 @@ mod tests { ); } + #[test] + fn removes_exclusion_attributes_from_rendered_html() { + let html = concat!( + r#"<h1 id="top">Top</h1><p>Keep</p>"#, + r#"<div data-search-exclude="true"><p>Drop</p></div>"#, + ); + let mut parser = Parser::default(); + let output = scan(html, &mut [&mut parser]).expect("search edit"); + + assert_eq!( + output, + concat!( + r#"<h1 id="top">Top</h1><p>Keep</p>"#, + r#"<div><p>Drop</p></div>"#, + ) + ); + assert_eq!(parser.finish(), vec![item(None, 1, "Top", "<p>Keep</p>")]); + } + + #[test] + fn excluded_pages_only_apply_html_cleanup() { + let html = r"<p data-search-exclude>Drop</p>"; + let mut parser = Parser::discarding(); + let output = scan(html, &mut [&mut parser]).expect("search edit"); + + assert_eq!(output, "<p>Drop</p>"); + assert!(parser.finish().is_empty()); + } + #[test] fn preserves_selected_markup_and_empty_elements() { let html = concat!( diff --git a/crates/zensical/src/config.rs b/crates/zensical/src/config.rs index 719ce7f..f8a92e6 100644 --- a/crates/zensical/src/config.rs +++ b/crates/zensical/src/config.rs @@ -63,6 +63,8 @@ pub struct Config { pub project: Arc<Project>, /// Theme directories. pub theme_dirs: Vec<PathBuf>, + /// Resolved Python Markdown extensions after compatibility shims. + markdown_extensions: Arc<[String]>, /// Configuration hash. pub hash: u64, } @@ -94,14 +96,17 @@ impl Config { // but we'll move it through the same pipeline for consistency. let module = py.import("zensical.config")?; let config = module - .call_method1("parse_config", (path.to_string_lossy(),))? - .extract::<Project>()?; + .call_method1("parse_config", (path.to_string_lossy(),))?; + let markdown_extensions = config + .get_item("markdown_extensions")? + .extract::<Vec<String>>()?; + let project = config.extract::<Project>()?; // Return configuration and theme directory - Ok::<_, PyErr>(config) + Ok::<_, PyErr>((project, markdown_extensions)) }) .map_err(Into::into) - .and_then(|project| { + .and_then(|(project, markdown_extensions)| { // Merge theme directories, giving precedence to custom directory // over the main theme directory to allow for overrides let iter = project.theme_dirs.clone().into_iter(); @@ -121,11 +126,19 @@ impl Config { path: path.canonicalize()?, project: Arc::new(project), theme_dirs, + markdown_extensions: markdown_extensions.into(), hash, }) }) } + /// Returns whether a resolved Python Markdown extension is active. + pub(crate) fn has_markdown_extension(&self, name: &str) -> bool { + self.markdown_extensions + .iter() + .any(|extension| extension == name) + } + /// Returns the directory the configuration file is located in. pub fn get_root_dir(&self) -> PathBuf { let mut path = self.path.clone(); diff --git a/crates/zensical/src/python/issues.rs b/crates/zensical/src/python/issues.rs index 31a9c92..10d736b 100644 --- a/crates/zensical/src/python/issues.rs +++ b/crates/zensical/src/python/issues.rs @@ -34,8 +34,8 @@ use std::slice::Iter; use zrx::id::Id; use zrx::stream::Key; +use crate::compat::mkdocs::plugin::autorefs::UnresolvedAutorefs; use crate::config::validation::Validation; -use crate::structure::markdown::UnresolvedAutorefs; use super::collector::reference::{ LinkReference, LinkReferenceKind, Reference, diff --git a/crates/zensical/src/structure/markdown.rs b/crates/zensical/src/structure/markdown.rs index cb356e6..38c2c81 100644 --- a/crates/zensical/src/structure/markdown.rs +++ b/crates/zensical/src/structure/markdown.rs @@ -39,10 +39,6 @@ use crate::structure::dynamic::Dynamic; use crate::structure::nav::to_title; use crate::structure::toc::Section; -mod autorefs; - -pub use autorefs::{Autorefs, UnresolvedAutorefs}; - // ---------------------------------------------------------------------------- // Structs // ---------------------------------------------------------------------------- @@ -122,6 +118,13 @@ impl Markdown { Markdown { data: Arc::new(data) } }) } + + /// Replaces rendered HTML before the Markdown value enters the workflow. + pub(crate) fn replace_content(&mut self, content: String) { + Arc::get_mut(&mut self.data) + .expect("rendered Markdown is not shared yet") + .content = content; + } } // ---------------------------------------------------------------------------- diff --git a/crates/zensical/src/structure/nav.rs b/crates/zensical/src/structure/nav.rs index 2876835..84ace2d 100644 --- a/crates/zensical/src/structure/nav.rs +++ b/crates/zensical/src/structure/nav.rs @@ -25,21 +25,17 @@ //! Navigation. -use std::fs; use std::hash::{DefaultHasher, Hash, Hasher}; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; -use ahash::{HashMap, HashSet}; +use ahash::HashMap; use pyo3::types::{PyAny, PyAnyMethods}; -use pyo3::{Bound, FromPyObject, PyResult, Python}; +use pyo3::{Bound, FromPyObject, PyResult}; use serde::Serialize; use zrx::id::Id; use zrx::scheduler::Value; use zrx::stream::Key; -use crate::structure::markdown::Autorefs; - use super::page::Page; mod item; @@ -50,13 +46,6 @@ pub use item::NavigationItem; use iter::Iter; pub(crate) use view::NavigationView; -// ---------------------------------------------------------------------------- -// Constants -// ---------------------------------------------------------------------------- - -/// Lock serializing collection and caching of global autorefs data. -static AUTOREFS_CACHE_LOCK: Mutex<()> = Mutex::new(()); - // ---------------------------------------------------------------------------- // Structs // ---------------------------------------------------------------------------- @@ -74,10 +63,6 @@ pub struct Navigation { pub items: Arc<Vec<NavigationItem>>, /// Homepage, if defined. pub homepage: Option<NavigationItem>, - /// Autorefs (mkdocstrings), kept internal to the rendering pipeline. - #[pyo3(from_py_with = extract_shared_autorefs)] - #[serde(skip)] - pub autorefs: Arc<Autorefs>, /// Precomputed navigation-structure hash. pub hash: u64, /// Site snapshot generation this navigation was created from. @@ -92,21 +77,10 @@ pub struct Navigation { impl Navigation { /// Creates a navigation from the given items. pub fn new( - cache_dir: PathBuf, mut items: Vec<NavigationItem>, - pages: Vec<(Key<Id>, Page)>, + mut items: Vec<NavigationItem>, pages: Vec<(Key<Id>, Page)>, ) -> Self { - let page_urls = pages - .iter() - .map(|(_, page)| page.url.clone()) - .collect::<HashSet<_>>(); - - // Fetch and cache autorefs once, used by both branches below - let autorefs = get_autorefs_cached(&cache_dir, &page_urls); - if items.is_empty() { - let mut nav = Self::from(pages); - nav.autorefs = Arc::new(autorefs); - return nav; + return Self::from(pages); } // Create a map of pages for easy lookup, so we can resolve titles and @@ -188,7 +162,6 @@ impl Navigation { Self { items: Arc::new(items), homepage, - autorefs: Arc::new(autorefs), hash, generation: 0, } @@ -352,19 +325,12 @@ impl From<Vec<(Key<Id>, Page)>> for Navigation { }); } - // Start from empty autorefs — Navigation::new() overrides this with - // the cached+merged result when called through the normal build path. - // Fetching from Python here would consume the updated-pages tracking - // outside of the cache lock, silently losing update flags. - let autorefs = Autorefs::new(); - // Precompute hash let hash = navigation_hash(&items); // Determine homepage and return navigation Self { homepage: items.iter().find(|item| item.is_index).cloned(), - autorefs: Arc::new(autorefs), items: Arc::new(items), hash, generation: 0, @@ -430,64 +396,12 @@ pub(crate) fn to_title(component: &str) -> String { } } -fn extract_shared_autorefs( - value: &Bound<'_, PyAny>, -) -> PyResult<Arc<Autorefs>> { - value.extract::<Autorefs>().map(Arc::new) -} - fn extract_shared_items( value: &Bound<'_, PyAny>, ) -> PyResult<Arc<Vec<NavigationItem>>> { value.extract::<Vec<NavigationItem>>().map(Arc::new) } -fn get_autorefs_cached( - cache_dir: &Path, page_urls: &HashSet<String>, -) -> Autorefs { - let _guard = AUTOREFS_CACHE_LOCK.lock().expect("invariant"); - let path = cache_dir.join("autorefs.json"); - - // Load previously cached autorefs, falling back to empty if unavailable - let mut autorefs = fs::read(&path) - .ok() - .and_then(|data| serde_json::from_slice::<Autorefs>(&data).ok()) - .unwrap_or_default(); - - // Fetch fresh data from the Python process. Remove registrations for pages - // that were reprocessed before merging, so removed anchors don't survive - // in the cache. Fresh data takes precedence, while identifiers from pages - // that stayed cached are preserved. - let fresh = get_autorefs(); - let updated_pages = - fresh.updated_pages.iter().cloned().collect::<HashSet<_>>(); - autorefs.remove_pages(&updated_pages); - autorefs.merge(fresh); - - // Drop registrations for pages that no longer exist. - autorefs.retain_pages(page_urls); - - // Write merged autorefs back to cache - if let Ok(data) = serde_json::to_string_pretty(&autorefs) { - let _ = fs::create_dir_all(cache_dir); - let _ = fs::write(&path, data); - } - - autorefs -} - -fn get_autorefs() -> Autorefs { - match Python::attach(|py| { - let module = py.import("zensical.extensions.autorefs")?; - module - .call_method0("get_autorefs_data")? - .extract::<Autorefs>() - }) { - Ok(autorefs) => autorefs, - Err(_) => Autorefs::new(), - } -} - // ---------------------------------------------------------------------------- // Tests // ---------------------------------------------------------------------------- @@ -501,33 +415,25 @@ mod tests { let nav = Navigation { items: Arc::new(Vec::new()), homepage: None, - autorefs: Arc::new(Autorefs::new()), hash: 0, generation: 0, }; let clone = nav.clone(); - assert!(Arc::ptr_eq(&nav.autorefs, &clone.autorefs)); assert!(Arc::ptr_eq(&nav.items, &clone.items)); } #[test] - fn serialization_omits_internal_autorefs_state() { - let mut autorefs = Autorefs::new(); - autorefs - .primary - .insert("item".to_string(), vec!["reference/#item".to_string()]); + fn serialization_omits_internal_generation() { let nav = Navigation { items: Arc::new(Vec::new()), homepage: None, hash: navigation_hash(&[]), - autorefs: Arc::new(autorefs), generation: 0, }; let value = serde_json::to_value(nav).expect("invariant"); - assert!(value.get("autorefs").is_none()); assert!(value.get("generation").is_none()); assert!(value.get("hash").is_some()); } diff --git a/crates/zensical/src/structure/nav/view.rs b/crates/zensical/src/structure/nav/view.rs index afe92d7..4e2a56a 100644 --- a/crates/zensical/src/structure/nav/view.rs +++ b/crates/zensical/src/structure/nav/view.rs @@ -255,7 +255,6 @@ mod tests { use minijinja::{context, Environment}; use super::*; - use crate::structure::markdown::Autorefs; /// Creates the same tree in immutable and page-active forms. fn navigation(active: bool) -> Navigation { @@ -289,7 +288,6 @@ mod tests { Navigation { items: Arc::new(vec![root, sibling]), homepage: None, - autorefs: Arc::new(Autorefs::new()), hash: 42, generation: 0, } diff --git a/crates/zensical/src/workflow.rs b/crates/zensical/src/workflow.rs index 082ba7a..0007da2 100644 --- a/crates/zensical/src/workflow.rs +++ b/crates/zensical/src/workflow.rs @@ -26,6 +26,7 @@ //! Workflow definitions use regex::Regex; +use serde::{Deserialize, Serialize}; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -39,7 +40,7 @@ use zrx::stream::{ concurrent, Key, Signal, Stream, StreamTupleExt, Value, Workflow, }; -use super::compat::mkdocs::{mkdocstrings, search}; +use super::compat::mkdocs::plugin::{self, autorefs, mkdocstrings, search}; use super::config::Config; use super::structure::markdown::Markdown; use super::structure::nav::Navigation; @@ -47,10 +48,8 @@ use super::structure::page::Page; use super::template::Template; use super::watcher::Source; -use super::{ - python::{Anchors, Issues, References, SharedReferences}, - structure::markdown::UnresolvedAutorefs, -}; +use super::compat::mkdocs::plugin::autorefs::UnresolvedAutorefs; +use super::python::{Anchors, Issues, References, SharedReferences}; // TODO: Migrate aggregation after the basic workflow runs on the new runtime. // mod aggregate; @@ -91,13 +90,58 @@ pub struct Main { #[derive(Clone, Debug)] struct Site { /// Complete pages selected for this batch. - pages: Arc<Vec<(Key<Id>, Page)>>, + pages: Arc<Vec<(Key<Id>, SitePage)>>, /// Navigation derived from the current pages. nav: Navigation, + /// Autoref registry derived from the same settled page snapshot. + autorefs: autorefs::Registry, } impl Value for Site {} +// ---------------------------------------------------------------------------- + +/// Page render input retained after site-wide settlement. +#[derive(Clone, Debug)] +struct SitePage { + /// Page passed to the template renderer. + page: Page, + /// Page-local autorefs replaced with stable slots. + autorefs: Arc<autorefs::References>, +} + +impl Value for SitePage {} + +// ---------------------------------------------------------------------------- + +/// Cached output of rendering one Markdown source. +#[derive(Clone, Debug, Serialize, Deserialize)] +struct RenderedMarkdown { + /// Rendered Markdown consumed by page construction. + markdown: Markdown, + /// Page-local registrations consumed during site settlement. + registrations: Arc<autorefs::Facts>, + /// Facts extracted by the shared MkDocs-compatible HTML pass. + html: plugin::HtmlFacts, +} + +impl Value for RenderedMarkdown {} + +// ---------------------------------------------------------------------------- + +/// Page plus compatibility facts derived from the same Markdown render. +#[derive(Clone, Debug, PartialEq, Eq)] +struct RenderedPage { + /// Page consumed by site-wide and page-local branches. + page: Page, + /// Autoref registrations revision-aligned with the page. + registrations: Arc<autorefs::Facts>, + /// HTML compatibility facts revision-aligned with the page. + html: plugin::HtmlFacts, +} + +impl Value for RenderedPage {} + // ---------------------------------------------------------------------------- // Implementations // ---------------------------------------------------------------------------- @@ -110,14 +154,24 @@ impl Main { // Set up workflow to process static assets and Markdown files. process_theme_assets(&self.config, &files); process_assets(&self.config, &files); - let markdown = process_markdown(&self.config, &files); + let rendered = process_markdown(&self.config, &files); // Cross the one global settlement boundary, derive all site-wide // state, then expand the resulting batch into independent page work. - let page = generate_page(&self.config, &markdown); - let site = generate_site(&self.config, &page); + let rendered_page = generate_page(&self.config, &rendered); + let page = + rendered_page.map(|rendered: &RenderedPage| rendered.page.clone()); + let document = rendered_page + .filter(|rendered: &RenderedPage| !rendered.html.search.is_empty()) + .map(|rendered: &RenderedPage| { + search::Document::new( + &rendered.page, + rendered.html.search.clone(), + ) + }); + let site = generate_site(&self.config, &rendered_page); let nav = generate_nav(&site); - search::attach(&self.config, &page, &nav); + search::attach(&self.config, &document, &nav); mkdocstrings::attach(&self.config, &nav); let _ = render_templates(&self.config, &files, &nav); let unresolved = render_pages(&self.config, &site); @@ -209,10 +263,11 @@ fn validate( } /// Compute a hash of the page content relevant to template rendering. -fn page_hash(page: &Page) -> u64 { +fn page_hash(page: &Page, autorefs: &autorefs::References) -> u64 { let mut hasher = DefaultHasher::new(); page.content.hash(&mut hasher); page.meta.hash(&mut hasher); + autorefs.hash(&mut hasher); hasher.finish() } @@ -296,9 +351,9 @@ fn copy_file( } /// Create a stream to process Markdown files. -pub fn process_markdown( +fn process_markdown( config: &Config, files: &Stream<Id, Source>, -) -> Stream<Id, Markdown> { +) -> Stream<Id, RenderedMarkdown> { let matcher = Arc::new( Matcher::from_str(&format!( "zrs::::{}:**/*.md:", @@ -308,6 +363,7 @@ pub fn process_markdown( ); // Create pipeline to render Markdown files + let plugins = plugin::Settings::new(config); let config = config.clone(); files .filter(move |id: &Id| matcher.is_match(id).expect("invariant")) @@ -361,44 +417,72 @@ pub fn process_markdown( // This is a hack while waiting for CommonMark (AST) and components, // as well as topic-based authoring functionality. if SNIPPET_RE.is_match(&data) { - Markdown::new(id, url, data) + render_markdown(id, url, data, plugins) } else { cached( &config, id.as_str(), (config.hash, data.clone(), url.clone()), - |(_, data, url)| Markdown::new(id, url, data), + |(_, data, url)| render_markdown(id, url, data, plugins), ) } })) } +/// Render Markdown and collect the page-local facts produced alongside it. +fn render_markdown( + id: &Id, url: String, content: String, plugins: plugin::Settings, +) -> anyhow::Result<RenderedMarkdown> { + let mut markdown = Markdown::new(id, url.clone(), content)?; + let html = plugin::prepare(&mut markdown, plugins); + let registrations = if plugins.autorefs { + autorefs::take_page(&url) + } else { + Arc::default() + }; + Ok(RenderedMarkdown { markdown, registrations, html }) +} + /// Generate pages from Markdown files. -pub fn generate_page( - config: &Config, markdown: &Stream<Id, Markdown>, -) -> Stream<Id, Page> { +fn generate_page( + config: &Config, markdown: &Stream<Id, RenderedMarkdown>, +) -> Stream<Id, RenderedPage> { let config = config.clone(); - markdown.map(move |id: &Id, markdown: &Markdown| { - Page::new(&config, id, markdown.clone()) + markdown.map(move |id: &Id, markdown: &RenderedMarkdown| RenderedPage { + page: Page::new(&config, id, markdown.markdown.clone()), + registrations: markdown.registrations.clone(), + html: markdown.html.clone(), }) } /// Derive one complete site batch at the page-relation terminal. fn generate_site( - config: &Config, pages: &Stream<Id, Page>, + config: &Config, pages: &Stream<Id, RenderedPage>, ) -> Signal<Id, Site> { let config = config.clone(); - pages.reduce(move |pages: &dyn Collection<Key<Id>, Page>| { - let pages: Vec<_> = pages - .iter() - .map(|(key, page)| (key.clone(), page.clone())) - .collect(); - let nav = Navigation::new( - config.get_cache_dir(), - config.project.nav.clone(), - pages.clone(), - ); - Some(Site { pages: Arc::new(pages), nav }) + pages.reduce(move |pages: &dyn Collection<Key<Id>, RenderedPage>| { + let mut nav_pages = Vec::new(); + let mut site_pages = Vec::new(); + let mut facts = Vec::new(); + for (key, rendered) in pages.iter() { + nav_pages.push((key.clone(), rendered.page.clone())); + site_pages.push(( + key.clone(), + SitePage { + page: rendered.page.clone(), + autorefs: rendered.html.autorefs.clone(), + }, + )); + facts.push((key.clone(), rendered.registrations.clone())); + } + + let nav = Navigation::new(config.project.nav.clone(), nav_pages); + let autorefs = autorefs::assemble(&config, facts); + Some(Site { + pages: Arc::new(site_pages), + nav, + autorefs, + }) }) } @@ -464,36 +548,48 @@ fn render_pages( let pages = site.flat_map(|site: &Site| { site.pages .iter() - .map(|(key, page)| (key.clone(), (page.clone(), site.nav.clone()))) + .map(|(key, page)| { + ( + key.clone(), + (page.clone(), site.nav.clone(), site.autorefs.clone()), + ) + }) .collect::<Vec<_>>() }); let template = OnceLock::new(); let theme_dirs = config.theme_dirs.clone(); let config = config.clone(); - pages.map(move |page: &Page, nav: &Navigation| { - let mut page = page.clone(); - let id = page.url.clone(); + pages.map( + move |input: &SitePage, + nav: &Navigation, + autorefs: &autorefs::Registry| { + let mut page = input.page.clone(); + let references = &input.autorefs; + let id = page.url.clone(); - // Cache template rendering independently of autorefs, which are - // substituted below on every pass. Deriving a cache key for the - // substitution would require the same resolution scan that the - // substitution itself performs, so caching it can't pay off. - let args = (config.hash, nav.hash, page_hash(&page)); - let rendered = cached(&config, ("template", id), args, |(_, _, _)| { - let template = - template.get_or_init(|| Template::new(theme_dirs.clone())); - Ok(page.render_template(template, &config, nav.clone())?) - })?; + // Cache template rendering independently of autorefs, which are + // substituted below on every pass. Deriving a cache key for the + // substitution would require the same resolution scan that the + // substitution itself performs, so caching it can't pay off. + let args = (config.hash, nav.hash, page_hash(&page, references)); + let rendered = + cached(&config, ("template", id), args, |(_, _, _)| { + let template = template + .get_or_init(|| Template::new(theme_dirs.clone())); + Ok(page.render_template(template, &config, nav.clone())?) + })?; - // Replace autorefs and retain unresolved identifiers - let (data, unresolved) = nav.autorefs.replace_in(rendered, &page.url); + // Replace autorefs and retain unresolved identifiers + let (data, unresolved) = + autorefs.replace_in(rendered, references, &page.url); - let path = Path::new(&page.path); - fs::create_dir_all(path.parent().expect("invariant"))?; - fs::write(path, &data)?; - Ok::<_, anyhow::Error>(unresolved) - }) + let path = Path::new(&page.path); + fs::create_dir_all(path.parent().expect("invariant"))?; + fs::write(path, &data)?; + Ok::<_, anyhow::Error>(unresolved) + }, + ) } /// Creates a workflow for the given config. diff --git a/python/tests/integration/test_search.py b/python/tests/integration/test_search.py index c6d426a..e73d3c4 100644 --- a/python/tests/integration/test_search.py +++ b/python/tests/integration/test_search.py @@ -174,6 +174,32 @@ Not indexed. assert _read_index(disabled)["items"] == [] +def test_search_exclusion_attribute_is_removed_from_page( + tmp_path: Path, +) -> None: + """Search pragmas affect the index but do not leak into final HTML.""" + config = _write_project(tmp_path, plugins=" - search") + (tmp_path / "docs" / "index.md").write_text( + """\ +# Landing + +Visible body. + +<div data-search-exclude><p>Hidden body.</p></div> +""", + encoding="utf-8", + ) + zensical.build(str(config), _BUILD_OPTIONS) + + index = _read_index(tmp_path) + assert "Visible body." in index["items"][0]["text"] + assert "Hidden body." not in index["items"][0]["text"] + + page = (tmp_path / "site" / "index.html").read_text() + assert "Hidden body." in page + assert "data-search-exclude" not in page + + def test_search_rebuild_replaces_changed_and_removed_pages( tmp_path: Path, ) -> None: diff --git a/python/tests/integration/test_validation.py b/python/tests/integration/test_validation.py index 15da309..3f38a15 100644 --- a/python/tests/integration/test_validation.py +++ b/python/tests/integration/test_validation.py @@ -396,3 +396,43 @@ custom_dir = "overrides" captured = capfd.readouterr() assert "No issues found" in captured.err + + +def test_cached_template_refreshes_when_page_autoref_changes( + tmp_path: Path, +) -> None: + """Page-local autoref facts participate in the template cache key.""" + docs = tmp_path / "docs" + docs.mkdir() + source = docs / "index.md" + source.write_text( + "# Home\n\n[First title][first-target]\n", encoding="utf-8" + ) + (docs / "other.md").write_text( + "# Other\n\n## first-target\n\n## second-target\n", + encoding="utf-8", + ) + config = tmp_path / "zensical.toml" + config.write_text( + """ +[project] +site_name = "Test" + +[project.plugins.autorefs] +""".lstrip(), + encoding="utf-8", + ) + + zensical.build(str(config), {"clean": True, "strict": False}) + output = (tmp_path / "site" / "index.html").read_text(encoding="utf-8") + assert 'href="other/#first-target">First title</a>' in output + + # Both references occupy page-local slot zero. Only their cached facts + # distinguish the template inputs after the Markdown pass. + source.write_text( + "# Home\n\n[Second title][second-target]\n", encoding="utf-8" + ) + zensical.build(str(config), {"clean": False, "strict": False}) + output = (tmp_path / "site" / "index.html").read_text(encoding="utf-8") + assert 'href="other/#second-target">Second title</a>' in output + assert 'href="other/#first-target">First title</a>' not in output diff --git a/python/tests/unit/extensions/test_autorefs.py b/python/tests/unit/extensions/test_autorefs.py index 1830a41..e8b93a0 100644 --- a/python/tests/unit/extensions/test_autorefs.py +++ b/python/tests/unit/extensions/test_autorefs.py @@ -29,7 +29,13 @@ from typing import TYPE_CHECKING, Any import pytest from tests.unit.extensions.conftest import soup -from zensical.extensions.autorefs import get_autorefs_store, reset +from zensical.extensions.autorefs import ( + get_autorefs_inventory_data, + get_autorefs_page_data, + get_autorefs_store, + reset, +) +from zensical.extensions.context import Page if TYPE_CHECKING: from collections.abc import Generator @@ -80,6 +86,38 @@ def _reset_autorefs_store() -> Generator[None, None, None]: reset() +# --------------------------------------------------------------------------- +# Store +# --------------------------------------------------------------------------- + + +class TestStore: + """Tests for page-local fact extraction from the transient store.""" + + def test_page_data_is_taken_without_consuming_inventory(self) -> None: + """Page registrations leave the global inventory available.""" + store = get_autorefs_store() + page = Page(url="guide/", path="guide.md", meta={}) + store.set_page(page) + store.register_anchor(page, "target", title="Target") + store.register_anchor(page, "alias", anchor="target", primary=False) + store.register_url("external", "https://example.com/external") + + assert get_autorefs_page_data("guide/") == { + "primary": {"target": ["guide/#target"]}, + "secondary": {"alias": ["guide/#target"]}, + "titles": {"guide/#target": "Target"}, + } + assert get_autorefs_page_data("guide/") == { + "primary": {}, + "secondary": {}, + "titles": {}, + } + assert get_autorefs_inventory_data() == { + "external": "https://example.com/external" + } + + # --------------------------------------------------------------------------- # Inline processor # --------------------------------------------------------------------------- diff --git a/python/zensical/extensions/autorefs.py b/python/zensical/extensions/autorefs.py index 11d28d5..3ba0628 100644 --- a/python/zensical/extensions/autorefs.py +++ b/python/zensical/extensions/autorefs.py @@ -58,9 +58,6 @@ if TYPE_CHECKING: HTAGS = {"h1", "h2", "h3", "h4", "h5", "h6"} -AUTOREF_RE = re.compile( - r"<autoref (?P<attrs>.*?)>(?P<title>.*?)</autoref>", flags=re.DOTALL -) # ---------------------------------------------------------------------------- @@ -88,24 +85,59 @@ class AutorefsStore: self._secondary_url_map: dict[str, list[str]] = {} self._abs_url_map: dict[str, str] = {} self._title_map: dict[str, str] = {} - self._updated_pages: set[str] = set() self._page_registrations: dict[str, set[tuple[bool, str, str]]] = {} def set_page(self, page: Page) -> None: """Set the current page and discard its previous registrations.""" self.current_page = page - self._updated_pages.add(page.url) - for primary, identifier, url in self._page_registrations.pop( - page.url, set() - ): - url_map = ( - self._primary_url_map if primary else self._secondary_url_map - ) - urls = url_map[identifier] - urls.remove(url) - if not urls: - del url_map[identifier] - self._title_map.pop(url, None) + self.pop_page(page.url) + + def pop_page(self, page_url: str) -> dict[str, Any]: + """Remove and return registrations owned by one page.""" + registrations = self._page_registrations.pop(page_url, set()) + primary = self._pop_urls(self._primary_url_map, registrations, True) + secondary = self._pop_urls( + self._secondary_url_map, registrations, False + ) + urls = { + url + for values in (primary, secondary) + for entries in values.values() + for url in entries + } + titles = { + url: self._title_map.pop(url) + for url in urls + if url in self._title_map + } + return { + "primary": primary, + "secondary": secondary, + "titles": titles, + } + + @staticmethod + def _pop_urls( + url_map: dict[str, list[str]], + registrations: set[tuple[bool, str, str]], + primary: bool, + ) -> dict[str, list[str]]: + """Remove registered URLs from one URL map, preserving their order.""" + selected: dict[str, set[str]] = {} + for is_primary, identifier, url in registrations: + if is_primary == primary: + selected.setdefault(identifier, set()).add(url) + + result: dict[str, list[str]] = {} + for identifier, selected_urls in selected.items(): + urls = url_map.get(identifier, []) + result[identifier] = [url for url in urls if url in selected_urls] + remaining = [url for url in urls if url not in selected_urls] + if remaining: + url_map[identifier] = remaining + else: + url_map.pop(identifier, None) + return result def register_anchor( self, @@ -266,7 +298,7 @@ class AutorefsInlineProcessor(ReferenceInlineProcessor): def _make_tag( self, identifier: str, text: str, *, slug: str | None = None ) -> Element: - """Create a tag that can be matched by `AUTO_REF_RE`.""" + """Create a tag that can be resolved after site settlement.""" el = Element("autoref") if self.hook: identifier = self.hook.expand_identifier(identifier) @@ -436,25 +468,22 @@ def get_autorefs_store() -> AutorefsStore: return AUTOREFS -def get_autorefs_data() -> dict[str, Any]: - """Get autorefs data. +def get_autorefs_page_data(page_url: str) -> dict[str, Any]: + """Take page-local autorefs data. - This function is called from Rust to replace the `<autoref>` - elements written in the HTML output by both the autorefs - Markdown extension (for manual cross-references) and the - mkdocstrings extension (for automatic cross-references). + Rust combines these registrations into the settled URL registry used to + resolve `<autoref>` elements emitted by autorefs and mkdocstrings. """ if AUTOREFS: - updated_pages = list(AUTOREFS._updated_pages) - AUTOREFS._updated_pages.clear() - return { - "primary": AUTOREFS._primary_url_map, - "secondary": AUTOREFS._secondary_url_map, - "inventory": AUTOREFS._abs_url_map, - "titles": AUTOREFS._title_map, - "updated_pages": updated_pages, - } - return {} + return AUTOREFS.pop_page(page_url) + return {"primary": {}, "secondary": {}, "titles": {}} + + +def get_autorefs_inventory_data() -> dict[str, str] | None: + """Return global inventory URLs if Markdown rendering initialized them.""" + if AUTOREFS is None: + return None + return AUTOREFS._abs_url_map def set_autorefs_page(page: Page) -> None: