From ff11fe3ef493b9eb8b86ace09f8c256b49d2dd02 Mon Sep 17 00:00:00 2001 From: squidfunk Date: Tue, 1 Sep 2026 18:19:40 +0200 Subject: [PATCH] refactor: migrate `search` MkDocs plugin replacement Signed-off-by: squidfunk --- Cargo.lock | 7 + Cargo.toml | 1 + crates/zensical/Cargo.toml | 1 + crates/zensical/src/compat.rs | 8 + crates/zensical/src/compat/mkdocs.rs | 8 + crates/zensical/src/compat/mkdocs/search.rs | 275 +++++++ .../mkdocs}/search/item.rs | 23 +- .../src/compat/mkdocs/search/parser.rs | 776 ++++++++++++++++++ crates/zensical/src/lib.rs | 1 + crates/zensical/src/structure.rs | 1 - crates/zensical/src/structure/markdown.rs | 33 +- crates/zensical/src/structure/nav.rs | 7 +- crates/zensical/src/structure/page.rs | 1 - crates/zensical/src/structure/search.rs | 151 ---- crates/zensical/src/workflow.rs | 44 +- python/tests/integration/test_search.py | 198 +++++ python/zensical/extensions/search.py | 397 --------- python/zensical/markdown/render.py | 29 +- 18 files changed, 1331 insertions(+), 630 deletions(-) create mode 100644 crates/zensical/src/compat.rs create mode 100644 crates/zensical/src/compat/mkdocs.rs create mode 100644 crates/zensical/src/compat/mkdocs/search.rs rename crates/zensical/src/{structure => compat/mkdocs}/search/item.rs (76%) create mode 100644 crates/zensical/src/compat/mkdocs/search/parser.rs delete mode 100644 crates/zensical/src/structure/search.rs create mode 100644 python/tests/integration/test_search.py delete mode 100644 python/zensical/extensions/search.py diff --git a/Cargo.lock b/Cargo.lock index 8ce5ac5..24acb6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,6 +293,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "html5gum" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "428502d3ec1742c35e015871aef742bc4722a9acfcb616b8ff79b519922e1c36" + [[package]] name = "httparse" version = "1.10.1" @@ -1474,6 +1480,7 @@ dependencies = [ "ariadne", "crossbeam", "fluent-uri", + "html5gum", "minijinja", "minijinja-contrib", "mio", diff --git a/Cargo.toml b/Cargo.toml index d7cc859..832ad0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ base64 = "0.22" crossbeam = "0.8" file-id = "0.2" fluent-uri = "0.4" +html5gum = { version = "0.8.4", default-features = false } httparse = "1.10" httpdate = "1.0" indicatif = "0.18" diff --git a/crates/zensical/Cargo.toml b/crates/zensical/Cargo.toml index 1b3124f..ee45754 100644 --- a/crates/zensical/Cargo.toml +++ b/crates/zensical/Cargo.toml @@ -50,6 +50,7 @@ anyhow.workspace = true ariadne.workspace = true crossbeam.workspace = true fluent-uri.workspace = true +html5gum.workspace = true minijinja = { workspace = true, features = [ "json", "loader", "builtins", "urlencode" ] } diff --git a/crates/zensical/src/compat.rs b/crates/zensical/src/compat.rs new file mode 100644 index 0000000..d600ffa --- /dev/null +++ b/crates/zensical/src/compat.rs @@ -0,0 +1,8 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +//! Compatibility implementations for established ecosystems. + +pub mod mkdocs; diff --git a/crates/zensical/src/compat/mkdocs.rs b/crates/zensical/src/compat/mkdocs.rs new file mode 100644 index 0000000..1520a1d --- /dev/null +++ b/crates/zensical/src/compat/mkdocs.rs @@ -0,0 +1,8 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +//! MkDocs compatibility modules. + +pub mod search; diff --git a/crates/zensical/src/compat/mkdocs/search.rs b/crates/zensical/src/compat/mkdocs/search.rs new file mode 100644 index 0000000..c6a394d --- /dev/null +++ b/crates/zensical/src/compat/mkdocs/search.rs @@ -0,0 +1,275 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. + +// ---------------------------------------------------------------------------- + +//! MkDocs-compatible search index. + +use serde::Serialize; +use std::collections::BTreeMap; +use std::fs; +use std::io::{BufWriter, Write}; +use zrx::id::Id; +use zrx::scheduler::Value; +use zrx::stream::function::Collection; +use zrx::stream::{Key, Signal, Stream}; + +use crate::config::plugins::SearchPluginConfig; +use crate::config::Config; +use crate::structure::dynamic::Dynamic; +use crate::structure::nav::{file_sort_key, Navigation}; +use crate::structure::page::Page; + +mod item; +mod parser; + +use item::{SearchItem, SearchSection}; +pub(crate) use parser::extract; + +// ---------------------------------------------------------------------------- +// Structs +// ---------------------------------------------------------------------------- + +/// Search configuration. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +struct SearchConfig { + /// Languages for tokenizer. + lang: Vec, + /// Separator for tokenizer. + separator: String, +} + +/// Complete search artifact. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +struct SearchIndex { + /// Search configuration. + config: SearchConfig, + /// Search items. + items: Vec, +} + +/// Compact page facts retained by the search branch. +#[derive(Clone, Debug, PartialEq, Eq)] +struct SearchDocument { + /// Page target URL. + url: String, + /// Page title. + title: String, + /// Page tag names. + tags: Vec, + /// Page-local search sections. + sections: Vec, +} + +// ---------------------------------------------------------------------------- +// Implementations +// ---------------------------------------------------------------------------- + +impl SearchConfig { + /// Creates search configuration for the configured theme language. + fn new(config: SearchPluginConfig, language: &str) -> Self { + Self { + lang: vec![language.to_string()], + separator: config.separator, + } + } +} + +// ---------------------------------------------------------------------------- + +impl SearchDocument { + /// Extracts the facts search needs from a rendered page. + fn new(page: &Page) -> Self { + Self { + url: page.url.clone(), + title: page.title.clone(), + tags: page.tags().into_iter().map(|tag| tag.name).collect(), + sections: extract(&page.content), + } + } +} + +// ---------------------------------------------------------------------------- + +impl SearchIndex { + /// Creates a search index from compact page facts. + #[allow(clippy::assigning_clones)] + fn new( + documents: Vec<(Key, SearchDocument)>, nav: &Navigation, + config: SearchPluginConfig, language: &str, + ) -> Self { + let mut items: Vec = Vec::new(); + + let mut documents = Vec::from_iter(documents); + documents.sort_by_key(|(id, _)| file_sort_key(&id[0])); + + // Attach site-wide navigation facts only while assembling the final + // artifact, keeping them out of each page-local stream value. + for (_id, document) in documents { + let iter = nav.ancestors_for_url(&document.url).into_iter().rev(); + let mut path = iter + .filter_map(|item| { + item.display_title().map(ToString::to_string) + }) + .collect::>(); + + // Add page title to path if not already present - this might be + // the true in case of index pages + if path.last() != Some(&document.title) { + path.push(document.title.clone()); + } + + for section in document.sections { + let location = match section.location { + Some(id) => format!("{}#{}", document.url, id), + _ => document.url.clone(), + }; + let title = if section.title.is_empty() { + document.title.clone() + } else { + section.title + }; + items.push(SearchItem { + location: Some(location), + level: section.level, + title, + text: section.text, + path: path.clone(), + tags: document.tags.clone(), + }); + } + } + + // Return search + Self { + config: SearchConfig::new(config, language), + items, + } + } +} + +// ---------------------------------------------------------------------------- +// Trait implementations +// ---------------------------------------------------------------------------- + +impl Value for SearchDocument {} + +// ---------------------------------------------------------------------------- +// Functions +// ---------------------------------------------------------------------------- + +/// Attach MkDocs-compatible search artifact generation to the build graph. +pub(crate) fn attach( + config: &Config, pages: &Stream, nav: &Signal, +) { + if !config.project.plugins.search.config.enabled { + let config = config.clone(); + let _ = nav.map(move |nav: &Navigation| { + let search = SearchIndex::new( + Vec::new(), + nav, + config.project.plugins.search.config.clone(), + &config.project.theme.language, + ); + write(&config, &search) + }); + return; + } + + let documents = pages + .filter(|page: &Page| !is_search_excluded(&page.meta)) + .map(SearchDocument::new); + let documents = documents.reduce( + |documents: &dyn Collection, SearchDocument>| { + Some( + documents + .iter() + .map(|(key, document)| (key.clone(), document.clone())) + .collect::>(), + ) + }, + ); + let config = config.clone(); + let _ = documents.product(nav).map( + move |documents: &Vec<(Key, SearchDocument)>, nav: &Navigation| { + let search = SearchIndex::new( + documents.clone(), + nav, + config.project.plugins.search.config.clone(), + &config.project.theme.language, + ); + write(&config, &search) + }, + ); +} + +/// Write search artifacts without retaining a second serialized copy. +fn write(config: &Config, search: &SearchIndex) -> anyhow::Result<()> { + let site_dir = config.get_site_dir(); + let path = site_dir.join("search.json"); + fs::create_dir_all(path.parent().expect("invariant"))?; + let mut writer = BufWriter::new(fs::File::create(path)?); + serde_json::to_writer(&mut writer, search)?; + writer.flush()?; + + if config.project.plugins.offline.config.enabled { + let path = site_dir.join("search.js"); + fs::create_dir_all(path.parent().expect("invariant"))?; + let mut writer = BufWriter::new(fs::File::create(path)?); + writer.write_all(b"var __index = ")?; + serde_json::to_writer(&mut writer, search)?; + writer.write_all(b";")?; + writer.flush()?; + } + Ok(()) +} + +/// Returns whether a page is excluded from search through its metadata. +fn is_search_excluded(meta: &BTreeMap) -> bool { + let Some(Dynamic::Map(search)) = meta.get("search") else { + return false; + }; + matches!(search.get("exclude"), Some(Dynamic::Bool(true))) +} + +// ---------------------------------------------------------------------------- +// Tests +// ---------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn search_exclusion_is_read_from_page_metadata() { + let mut search = BTreeMap::new(); + search.insert(String::from("exclude"), Dynamic::Bool(true)); + let mut meta = BTreeMap::new(); + meta.insert(String::from("search"), Dynamic::Map(search)); + + assert!(is_search_excluded(&meta)); + + meta.clear(); + assert!(!is_search_excluded(&meta)); + } +} diff --git a/crates/zensical/src/structure/search/item.rs b/crates/zensical/src/compat/mkdocs/search/item.rs similarity index 76% rename from crates/zensical/src/structure/search/item.rs rename to crates/zensical/src/compat/mkdocs/search/item.rs index ea7a263..05a4ef7 100644 --- a/crates/zensical/src/structure/search/item.rs +++ b/crates/zensical/src/compat/mkdocs/search/item.rs @@ -23,18 +23,16 @@ // ---------------------------------------------------------------------------- -//! Search item. +//! MkDocs-compatible search item. -use pyo3::FromPyObject; -use serde::{Deserialize, Serialize}; +use serde::Serialize; // ---------------------------------------------------------------------------- // Structs // ---------------------------------------------------------------------------- /// Search item. -#[derive(Clone, Debug, PartialEq, Eq, FromPyObject, Serialize, Deserialize)] -#[pyo3(from_item_all)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct SearchItem { /// Search location. pub location: Option, @@ -49,3 +47,18 @@ pub struct SearchItem { /// Section tags. pub tags: Vec, } + +// ---------------------------------------------------------------------------- + +/// Page-local search section before site-wide facts are attached. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct SearchSection { + /// Heading fragment, if present. + pub location: Option, + /// Section level. + pub level: u32, + /// Section title. + pub title: String, + /// Section text. + pub text: String, +} diff --git a/crates/zensical/src/compat/mkdocs/search/parser.rs b/crates/zensical/src/compat/mkdocs/search/parser.rs new file mode 100644 index 0000000..9f30ada --- /dev/null +++ b/crates/zensical/src/compat/mkdocs/search/parser.rs @@ -0,0 +1,776 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +//! MkDocs-compatible search extraction from rendered HTML. + +use html5gum::emitters::callback::{CallbackEmitter, CallbackEvent}; +use html5gum::{Span, Tokenizer}; +use std::convert::Infallible; + +use super::SearchSection; + +/// Extract page-local search sections from rendered HTML. +pub(crate) fn extract(html: &str) -> Vec { + let mut parser = SearchParser::default(); + + { + let mut emitter = CallbackEmitter::new( + |event: CallbackEvent<'_>, _span: Span<()>| -> Option { + parser.handle(event); + None + }, + ); + emitter.naively_switch_states(true); + + Tokenizer::new_with_emitter(html, emitter) + .finish() + .expect("string input is infallible"); + } + + parser.finish() +} + +// ---------------------------------------------------------------------------- +// Parser +// ---------------------------------------------------------------------------- + +/// Streaming search parser. +#[derive(Default)] +struct SearchParser { + /// Open HTML elements. + context: Vec, + /// Section currently receiving text. + current: Option, + /// Sections extracted from the input. + sections: Vec, + /// Number of excluded elements currently open. + skip: usize, + /// Start tag currently emitted by the tokenizer. + start: Option, + /// Current attribute of the pending start tag. + attribute: Attribute, +} + +impl SearchParser { + /// Handles a tokenizer event. + fn handle(&mut self, event: CallbackEvent<'_>) { + match event { + CallbackEvent::OpenStartTag { name } => { + self.start = Some(StartTag::new(Tag::from_bytes(name))); + self.attribute = Attribute::Other; + } + CallbackEvent::AttributeName { name } => { + self.attribute = Attribute::from_bytes(name); + if let Some(start) = &mut self.start { + start.observe_attribute(self.attribute, None); + } + } + CallbackEvent::AttributeValue { value } => { + if let Some(start) = &mut self.start { + start.observe_attribute(self.attribute, Some(value)); + } + } + CallbackEvent::CloseStartTag { self_closing } => { + if let Some(start) = self.start.take() { + let tag = start.tag.clone(); + self.start(start); + if self_closing { + self.end(&tag); + } + } + } + CallbackEvent::EndTag { name } => { + self.end(&Tag::from_bytes(name)); + } + CallbackEvent::String { value } => { + self.text(String::from_utf8_lossy(value).as_ref()); + } + CallbackEvent::Comment { .. } + | CallbackEvent::Doctype { .. } + | CallbackEvent::Error(_) => {} + } + } + + /// Handles a complete start tag. + fn start(&mut self, start: StartTag) { + if start.tag.is_void() { + return; + } + + let heading = start.tag.heading_level(); + let skipped = start.tag.is_skipped() + || start.excluded + || start.class.as_deref() == Some(b"linenodiv"); + let headerlink = start.tag == Tag::A + && start.class.as_deref() == Some(b"headerlink"); + let tag = start.tag.clone(); + + self.context.push(Element { + tag: start.tag, + skipped, + headerlink, + kept: None, + }); + + if let (Some(level), true) = (heading, start.id_present) { + let depth = self.context.len(); + + if level != 1 && self.sections.is_empty() { + self.push_preface(); + } + + let location = if self.sections.is_empty() { + None + } else { + start.id + }; + let section = SectionState::new( + Some(level), + level.into(), + depth, + location, + start.excluded, + ); + self.sections.push(section); + self.current = Some(self.sections.len() - 1); + } + + self.ensure_section(); + + if skipped { + self.skip += 1; + return; + } + + if self.skip == 0 && tag.is_kept() { + let (section, title) = self.output_target(); + let data = self.sections[section].output_mut(title); + let start = data.value.len(); + let previous_whitespace = data.last_whitespace; + data.value.push('<'); + data.value.push_str(tag.name()); + data.value.push('>'); + data.last_whitespace = false; + + self.context.last_mut().expect("context").kept = + Some(KeptElement { + section, + title, + start, + previous_whitespace, + }); + } + } + + /// Handles an end tag. + fn end(&mut self, tag: &Tag) { + if self.context.last().is_none_or(|el| el.tag != *tag) { + return; + } + + let depth = self.context.len(); + if let Some(index) = self.current + && self.sections[index].exited_or_deeper_than(depth) + && let Some(parent) = self + .sections + .iter() + .rposition(|section| !section.exited && section.depth <= depth) + { + self.sections[index].exited = true; + self.current = Some(parent); + } + + let element = self.context.pop().expect("context"); + if element.skipped { + self.skip -= 1; + return; + } + + if self.skip == 0 && tag.is_kept() { + let (section, title) = self.output_target(); + let data = self.sections[section].output_mut(title); + let opening = format!("<{}>", tag.name()); + + if let Some(start) = data.value.find(&opening) { + let following = &data.value[start + opening.len()..]; + if following.chars().any(|char| !char.is_whitespace()) { + data.value.push_str("'); + data.last_whitespace = false; + } else { + data.value.truncate(start); + data.last_whitespace = element.kept.map_or_else( + || { + data.value + .chars() + .last() + .is_some_and(char::is_whitespace) + }, + |kept| { + debug_assert_eq!(kept.section, section); + debug_assert_eq!(kept.title, title); + debug_assert_eq!(kept.start, start); + kept.previous_whitespace + }, + ); + } + } + } + } + + /// Handles text content. + fn text(&mut self, value: &str) { + if self.skip > 0 { + return; + } + + let preformatted = self.context.iter().any(|el| el.tag == Tag::Pre); + let whitespace = value.chars().all(char::is_whitespace); + let text; + let value = if preformatted { + value + } else if whitespace { + " " + } else if value.contains('\n') { + text = value.replace('\n', " "); + &text + } else { + value + }; + + self.ensure_section(); + let (section, title) = self.output_target(); + + if title { + if self.context.iter().any(|el| el.headerlink) { + return; + } + escape(value, &mut self.sections[section].title.value); + self.sections[section].title.last_whitespace = whitespace; + } else { + let data = &mut self.sections[section].text; + if !whitespace || preformatted || !data.last_whitespace { + escape(value, &mut data.value); + data.last_whitespace = whitespace; + } + } + } + + /// Returns the current section and whether its title receives output. + fn output_target(&self) -> (usize, bool) { + let section = self.current.expect("section"); + let heading = self.sections[section].heading; + let title = heading.is_some_and(|level| { + self.context + .iter() + .any(|el| el.tag.heading_level() == Some(level)) + }); + (section, title) + } + + /// Ensures a section exists for preface content. + fn ensure_section(&mut self) { + if self.current.is_none() { + self.push_preface(); + } + } + + /// Adds the implicit top-level preface section. + fn push_preface(&mut self) { + self.sections + .push(SectionState::new(None, 1, 0, None, false)); + self.current = Some(self.sections.len() - 1); + } + + /// Converts parser state into page-local search sections. + fn finish(self) -> Vec { + self.sections + .into_iter() + .filter(|section| !section.excluded) + .map(|section| SearchSection { + location: section.location, + level: section.level, + title: trim(section.title.value), + text: trim(section.text.value), + }) + .collect() + } +} + +// ---------------------------------------------------------------------------- +// State +// ---------------------------------------------------------------------------- + +/// Section being assembled. +struct SectionState { + heading: Option, + level: u32, + depth: usize, + exited: bool, + excluded: bool, + location: Option, + title: Output, + text: Output, +} + +impl SectionState { + fn new( + heading: Option, level: u32, depth: usize, + location: Option, excluded: bool, + ) -> Self { + Self { + heading, + level, + depth, + exited: false, + excluded, + location, + title: Output::default(), + text: Output::default(), + } + } + + fn exited_or_deeper_than(&self, depth: usize) -> bool { + self.exited || self.depth > depth + } + + fn output_mut(&mut self, title: bool) -> &mut Output { + if title { + &mut self.title + } else { + &mut self.text + } + } +} + +/// Output buffer and whitespace state. +#[derive(Default)] +struct Output { + value: String, + last_whitespace: bool, +} + +/// Open HTML element. +struct Element { + tag: Tag, + skipped: bool, + headerlink: bool, + kept: Option, +} + +/// Location at which a retained element was opened. +#[derive(Clone, Copy)] +struct KeptElement { + section: usize, + title: bool, + start: usize, + previous_whitespace: bool, +} + +/// Start tag under construction. +struct StartTag { + tag: Tag, + id_present: bool, + id: Option, + excluded: bool, + class: Option>, +} + +impl StartTag { + fn new(tag: Tag) -> Self { + Self { + tag, + id_present: false, + id: None, + excluded: false, + class: None, + } + } + + fn observe_attribute( + &mut self, attribute: Attribute, value: Option<&[u8]>, + ) { + match attribute { + Attribute::Id => { + self.id_present = true; + self.id = value + .map(|value| String::from_utf8_lossy(value).into_owned()); + } + Attribute::Class => { + self.class = value.map(<[u8]>::to_vec); + } + Attribute::SearchExclude => self.excluded = true, + Attribute::Other => {} + } + } +} + +/// Attributes relevant to extraction. +#[derive(Clone, Copy, Default)] +enum Attribute { + Id, + Class, + SearchExclude, + #[default] + Other, +} + +impl Attribute { + fn from_bytes(value: &[u8]) -> Self { + match value { + b"id" => Self::Id, + b"class" => Self::Class, + b"data-search-exclude" => Self::SearchExclude, + _ => Self::Other, + } + } +} + +/// HTML tags relevant to extraction. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Tag { + A, + Area, + Base, + Br, + Code, + Col, + Embed, + H(u8), + Hr, + Img, + Input, + Li, + Link, + Meta, + Object, + Ol, + P, + Param, + Pre, + Script, + Small, + Source, + Style, + Sub, + Sup, + Track, + Ul, + Wbr, + Other(Box), +} + +impl Tag { + fn from_bytes(value: &[u8]) -> Self { + match value { + b"a" => Self::A, + b"area" => Self::Area, + b"base" => Self::Base, + b"br" => Self::Br, + b"code" => Self::Code, + b"col" => Self::Col, + b"embed" => Self::Embed, + b"h1" => Self::H(1), + b"h2" => Self::H(2), + b"h3" => Self::H(3), + b"h4" => Self::H(4), + b"h5" => Self::H(5), + b"h6" => Self::H(6), + b"hr" => Self::Hr, + b"img" => Self::Img, + b"input" => Self::Input, + b"li" => Self::Li, + b"link" => Self::Link, + b"meta" => Self::Meta, + b"object" => Self::Object, + b"ol" => Self::Ol, + b"p" => Self::P, + b"param" => Self::Param, + b"pre" => Self::Pre, + b"script" => Self::Script, + b"small" => Self::Small, + b"source" => Self::Source, + b"style" => Self::Style, + b"sub" => Self::Sub, + b"sup" => Self::Sup, + b"track" => Self::Track, + b"ul" => Self::Ul, + b"wbr" => Self::Wbr, + _ => { + Self::Other(String::from_utf8_lossy(value).into_owned().into()) + } + } + } + + fn name(&self) -> &str { + match self { + Self::A => "a", + Self::Area => "area", + Self::Base => "base", + Self::Br => "br", + Self::Code => "code", + Self::Col => "col", + Self::Embed => "embed", + Self::H(1) => "h1", + Self::H(2) => "h2", + Self::H(3) => "h3", + Self::H(4) => "h4", + Self::H(5) => "h5", + Self::H(6) => "h6", + Self::H(_) => unreachable!("heading level"), + Self::Hr => "hr", + Self::Img => "img", + Self::Input => "input", + Self::Li => "li", + Self::Link => "link", + Self::Meta => "meta", + Self::Object => "object", + Self::Ol => "ol", + Self::P => "p", + Self::Param => "param", + Self::Pre => "pre", + Self::Script => "script", + Self::Small => "small", + Self::Source => "source", + Self::Style => "style", + Self::Sub => "sub", + Self::Sup => "sup", + Self::Track => "track", + Self::Ul => "ul", + Self::Wbr => "wbr", + Self::Other(name) => name, + } + } + + fn heading_level(&self) -> Option { + if let Self::H(level) = self { + Some(*level) + } else { + None + } + } + + fn is_kept(&self) -> bool { + matches!( + self, + Self::P + | Self::Code + | Self::Pre + | Self::Li + | Self::Ol + | Self::Ul + | Self::Small + | Self::Sub + | Self::Sup + ) + } + + fn is_skipped(&self) -> bool { + matches!(self, Self::Object | Self::Script | Self::Style) + } + + fn is_void(&self) -> bool { + matches!( + self, + Self::Area + | Self::Base + | Self::Br + | Self::Col + | Self::Embed + | Self::Hr + | Self::Img + | Self::Input + | Self::Link + | Self::Meta + | Self::Param + | Self::Source + | Self::Track + | Self::Wbr + ) + } +} + +// ---------------------------------------------------------------------------- +// Helpers +// ---------------------------------------------------------------------------- + +/// Escapes text like `html.escape(..., quote=False)`. +fn escape(value: &str, output: &mut String) { + for char in value.chars() { + match char { + '&' => output.push_str("&"), + '<' => output.push_str("<"), + '>' => output.push_str(">"), + _ => output.push(char), + } + } +} + +/// Trims Unicode whitespace without retaining the original allocation twice. +fn trim(mut value: String) -> String { + let end = value.trim_end().len(); + value.truncate(end); + let start = value.len() - value.trim_start().len(); + value.drain(..start); + value +} + +#[cfg(test)] +mod tests { + use super::*; + + fn item( + location: Option<&str>, level: u32, title: &str, text: &str, + ) -> SearchSection { + SearchSection { + location: location.map(ToString::to_string), + level, + title: title.to_string(), + text: text.to_string(), + } + } + + #[test] + fn extracts_preface() { + let html = "

Before heading.

"; + assert_eq!( + extract(html), + vec![item(None, 1, "", "

Before heading.

")] + ); + } + + #[test] + fn divides_content_into_sections() { + let html = concat!( + r##"

Top code"##, + r##"

"##, + r#"

First & second.

"#, + r#"

Child

Body

"#, + ); + assert_eq!( + extract(html), + vec![ + item( + None, + 1, + "Top code", + "

First & second.

" + ), + item(Some("child"), 2, "Child", "

Body

"), + ] + ); + } + + #[test] + fn preserves_missing_heading_id_behavior() { + let html = "

No ID

Body

"; + assert_eq!(extract(html), vec![item(None, 1, "", "No ID

Body

")]); + } + + #[test] + fn excludes_configured_content() { + let html = concat!( + r#"

Top

Keep

"#, + r#"

Drop

After

"#, + r#"
1
"#, + r#""#, + ); + assert_eq!( + extract(html), + vec![item(None, 1, "Top", "

Keep

After

")] + ); + } + + #[test] + fn preserves_selected_markup_and_empty_elements() { + let html = concat!( + r#"

Top

Text small

"#, + r#"

  • One
  • x
"#, + ); + assert_eq!( + extract(html), + vec![item( + None, + 1, + "Top", + "

Text small

  • One
  • x
", + )] + ); + } + + #[test] + fn preserves_whitespace_and_preformatted_content() { + let html = concat!( + "

Top

one\n two

", + "
a < b\n  c
", + ); + assert_eq!( + extract(html), + vec![item( + None, + 1, + "Top", + "

one two

a < b\n  c
", + )] + ); + } + + #[test] + fn decodes_and_escapes_entities_like_python() { + let html = concat!( + r#"

A & B ©

"#, + r#"

<tag> "x" 'y'  

"#, + ); + assert_eq!( + extract(html), + vec![item( + None, + 1, + "A & B ©", + "

<tag> \"x\" 'y' \u{a0}

" + )] + ); + } + + #[test] + fn restores_parent_section_after_nested_heading() { + let html = concat!( + r#"

Nested

Inside

"#, + r#"

Outside

"#, + ); + assert_eq!( + extract(html), + vec![ + item(None, 1, "", "

Outside

"), + item(Some("nested"), 2, "Nested", "

Inside

"), + ] + ); + } + + #[test] + fn preserves_malformed_html_behavior() { + let html = concat!( + r#"

Top

Before open

"#, + r#"

Next

After"#, + ); + assert_eq!( + extract(html), + vec![ + item(None, 1, "Top", "

Before open"), + item(Some("next"), 2, "Next", "

After"), + ] + ); + } + + #[test] + fn ignores_void_elements() { + let html = concat!( + r#"

Top

"#, + r#"

A
BC

D

"#, + ); + assert_eq!(extract(html), vec![item(None, 1, "Top", "

ABCD

")]); + } +} diff --git a/crates/zensical/src/lib.rs b/crates/zensical/src/lib.rs index 8c576ab..859fd0d 100644 --- a/crates/zensical/src/lib.rs +++ b/crates/zensical/src/lib.rs @@ -40,6 +40,7 @@ use std::time::{Duration, Instant}; use std::{fs, io, thread}; use zrx::id::Id; +mod compat; mod config; mod python; mod server; diff --git a/crates/zensical/src/structure.rs b/crates/zensical/src/structure.rs index 39a7db6..4538f38 100644 --- a/crates/zensical/src/structure.rs +++ b/crates/zensical/src/structure.rs @@ -29,6 +29,5 @@ pub mod dynamic; pub mod markdown; pub mod nav; pub mod page; -pub mod search; pub mod tag; pub mod toc; diff --git a/crates/zensical/src/structure/markdown.rs b/crates/zensical/src/structure/markdown.rs index 9b2239e..cb356e6 100644 --- a/crates/zensical/src/structure/markdown.rs +++ b/crates/zensical/src/structure/markdown.rs @@ -37,7 +37,6 @@ use zrx::stream::Value; use crate::structure::dynamic::Dynamic; use crate::structure::nav::to_title; -use crate::structure::search::SearchItem; use crate::structure::toc::Section; mod autorefs; @@ -61,21 +60,32 @@ pub struct Markdown { } /// Immutable rendered Markdown data. -#[derive(Debug, FromPyObject, Serialize, Deserialize)] -#[pyo3(from_item_all)] +#[derive(Debug, Serialize, Deserialize)] pub struct MarkdownData { /// Markdown metadata. pub meta: BTreeMap, /// Markdown content. pub content: String, - /// Search index. - pub search: Vec, /// Page title extracted from Markdown. pub title: String, /// Table of contents. pub toc: Vec
, } +/// Markdown data returned by the Python renderer. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct RenderedMarkdown { + /// Markdown metadata. + meta: BTreeMap, + /// Markdown content. + content: String, + /// Page title extracted from Markdown. + title: String, + /// Table of contents. + toc: Vec
, +} + // ---------------------------------------------------------------------------- // Implementations // ---------------------------------------------------------------------------- @@ -89,7 +99,7 @@ impl Markdown { let module = py.import("zensical.markdown.render")?; module .call_method1("render", (content, id.location(), url))? - .extract::() + .extract::() }) .map_err(|err| { Python::attach(|py| { @@ -101,7 +111,13 @@ impl Markdown { }) }); - res.map(|mut data| { + res.map(|data| { + let mut data = MarkdownData { + meta: data.meta, + content: data.content, + title: data.title, + toc: data.toc, + }; data.title = extract_title(&id, &data); Markdown { data: Arc::new(data) } }) @@ -173,7 +189,6 @@ fn extract_title(id: &Id, markdown: &MarkdownData) -> String { to_title(&file) } -// ---------------------------------------------------------------------------- // Tests // ---------------------------------------------------------------------------- @@ -186,7 +201,6 @@ mod tests { data: Arc::new(MarkdownData { meta: BTreeMap::new(), content: String::from("

Home

"), - search: Vec::new(), title: String::from("Home"), toc: Vec::new(), }), @@ -208,6 +222,7 @@ mod tests { assert_eq!(value["content"], "

Home

"); assert_eq!(value["title"], "Home"); assert!(value.get("data").is_none()); + assert!(value.get("search").is_none()); let markdown: Markdown = serde_json::from_value(value).unwrap(); assert_eq!(markdown.content, "

Home

"); diff --git a/crates/zensical/src/structure/nav.rs b/crates/zensical/src/structure/nav.rs index f5563f0..2876835 100644 --- a/crates/zensical/src/structure/nav.rs +++ b/crates/zensical/src/structure/nav.rs @@ -199,6 +199,11 @@ impl Navigation { /// Note that only the ancestors, not the page itself is returned, which /// again, mirrors MkDocs' behavior, and is necessary for breadcrumbs. pub fn ancestors(&self, page: &Page) -> Vec { + self.ancestors_for_url(&page.url) + } + + /// Returns ancestors of a page URL without requiring the complete page. + pub(crate) fn ancestors_for_url(&self, url: &str) -> Vec { // Recursively find ancestors of the page with the given URL. fn recurse<'a>( items: &'a [NavigationItem], url: &str, @@ -227,7 +232,7 @@ impl Navigation { // Clone the ancestors into owned items and reverse them, so we start // at the ancestor closest to the page, not the root itself let mut items: Vec<&NavigationItem> = Vec::new(); - let _ = recurse(&self.items, &page.url, &mut items); + let _ = recurse(&self.items, url, &mut items); items.into_iter().rev().cloned().collect() } diff --git a/crates/zensical/src/structure/page.rs b/crates/zensical/src/structure/page.rs index bceeeaa..1ed2ba6 100644 --- a/crates/zensical/src/structure/page.rs +++ b/crates/zensical/src/structure/page.rs @@ -256,7 +256,6 @@ impl PartialEq for PageData { && self.path == other.path && self.content == other.content && self.toc == other.toc - && self.search == other.search } } diff --git a/crates/zensical/src/structure/search.rs b/crates/zensical/src/structure/search.rs deleted file mode 100644 index 4d292fb..0000000 --- a/crates/zensical/src/structure/search.rs +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright (c) 2025-2026 Zensical and contributors - -// SPDX-License-Identifier: MIT -// All contributions are certified under the DCO - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to -// deal in the Software without restriction, including without limitation the -// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -// sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -// IN THE SOFTWARE. - -// ---------------------------------------------------------------------------- - -//! Search index. - -use pyo3::FromPyObject; -use serde::Serialize; -use zrx::id::Id; -use zrx::scheduler::Value; -use zrx::stream::Key; - -use crate::config::plugins::SearchPluginConfig; - -use super::nav::{file_sort_key, Navigation}; -use super::page::Page; - -mod item; - -pub use item::SearchItem; - -// ---------------------------------------------------------------------------- -// Structs -// ---------------------------------------------------------------------------- - -/// Search configuration. -#[derive(Clone, Debug, PartialEq, Eq, FromPyObject, Serialize)] -pub struct SearchConfig { - /// Languages for tokenizer. - pub lang: Vec, - /// Separator for tokenizer. - pub separator: String, -} - -/// Search index. -/// -/// Later, when the module system is available, we'll move search into a module -/// of its own, but for now, we'll just keep it here for simplicity. -#[derive(Clone, Debug, PartialEq, Eq, FromPyObject, Serialize)] -pub struct SearchIndex { - /// Search configuration. - pub config: SearchConfig, - /// Search items. - pub items: Vec, -} - -// ---------------------------------------------------------------------------- -// Implementations -// ---------------------------------------------------------------------------- - -impl SearchConfig { - /// Creates search configuration for the configured theme language. - fn new(config: SearchPluginConfig, language: &str) -> Self { - Self { - lang: vec![language.to_string()], - separator: config.separator, - } - } -} - -// ---------------------------------------------------------------------------- - -impl SearchIndex { - /// Creates a search index from pages. - #[allow(clippy::assigning_clones)] - pub fn new( - pages: Vec<(Key, Page)>, nav: &Navigation, - config: SearchPluginConfig, language: &str, - ) -> Self { - let mut items: Vec = Vec::new(); - - // Convert chunk into a vector for easier processing, and sort pages by - // the exact same method that MkDocs uses - let mut pages = Vec::from_iter(pages); - pages.sort_by_key(|(id, _)| file_sort_key(&id[0])); - - // Assemble search index, combining all items from all pages into a - // single, flat list, adjusting the location to include the page URL - for (_id, page) in pages { - let iter = nav.ancestors(&page).into_iter().rev(); - let mut path = iter - .filter_map(|item| { - item.display_title().map(ToString::to_string) - }) - .collect::>(); - - // Add page title to path if not already present - this might be - // the true in case of index pages - if path.last() != Some(&page.title) { - path.push(page.title.clone()); - } - - // Extract page tags, if any - let tags: Vec = - page.tags().into_iter().map(|tag| tag.name).collect(); - - // For each page, adjust the location of each item and add it to - // the overall list - for mut item in page.search.iter().cloned() { - let location = match item.location { - Some(id) => format!("{}#{}", page.url, id), - _ => page.url.clone(), - }; - - // Fall back to page title, if item title is empty - if item.title.is_empty() { - item.title = page.title.clone(); - } - - // Update location and path and add item - item.location = Some(location); - item.path = path.clone(); - item.tags = tags.clone(); - items.push(item); - } - } - - // Return search - Self { - config: SearchConfig::new(config, language), - items, - } - } -} - -// ---------------------------------------------------------------------------- -// Trait implementations -// ---------------------------------------------------------------------------- - -impl Value for SearchIndex {} diff --git a/crates/zensical/src/workflow.rs b/crates/zensical/src/workflow.rs index 8d5f76b..ed3cbd6 100644 --- a/crates/zensical/src/workflow.rs +++ b/crates/zensical/src/workflow.rs @@ -29,7 +29,6 @@ use pyo3::types::PyAnyMethods; use pyo3::Python; use regex::Regex; use std::hash::{DefaultHasher, Hash, Hasher}; -use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::{Arc, LazyLock, OnceLock}; @@ -42,11 +41,11 @@ use zrx::stream::{ concurrent, Key, Signal, Stream, StreamTupleExt, Value, Workflow, }; +use super::compat::mkdocs::search; use super::config::Config; use super::structure::markdown::Markdown; use super::structure::nav::Navigation; use super::structure::page::Page; -use super::structure::search::SearchIndex; use super::template::Template; use super::watcher::Source; @@ -120,7 +119,7 @@ impl Main { let page = generate_page(&self.config, &markdown); let site = generate_site(&self.config, &page); let nav = generate_nav(&site); - generate_search_index(&self.config, &site); + search::attach(&self.config, &page, &nav); generate_object_inventory(&self.config, &nav); let _ = render_templates(&self.config, &files, &nav); let unresolved = render_pages(&self.config, &site); @@ -360,7 +359,6 @@ pub fn process_markdown( } else { url }; - // Don't cache page if it inserts (pymdownx) snippets. // This is a hack while waiting for CommonMark (AST) and components, // as well as topic-based authoring functionality. @@ -443,44 +441,6 @@ pub fn generate_object_inventory( }); } -/// Generate search index -fn generate_search_index(config: &Config, site: &Signal) { - let config = config.clone(); - let _ = site.map(move |site: &Site| { - // Derive search only in this terminal branch, so its complete item - // relation is released immediately after the files are written. - let search = SearchIndex::new( - site.pages.as_ref().clone(), - &site.nav, - config.project.plugins.search.config.clone(), - &config.project.theme.language, - ); - let site_dir = config.get_site_dir(); - - // Stream the search index directly to disk without retaining an - // additional JSON string alongside the structured representation. - let path = site_dir.join("search.json"); - fs::create_dir_all(path.parent().expect("invariant"))?; - let mut writer = BufWriter::new(fs::File::create(path)?); - serde_json::to_writer(&mut writer, &search)?; - writer.flush()?; - - // If offline plugin is enabled, create search.js as well - if config.project.plugins.offline.config.enabled { - let path = site_dir.join("search.js"); - fs::create_dir_all(path.parent().expect("invariant"))?; - let mut writer = BufWriter::new(fs::File::create(path)?); - writer.write_all(b"var __index = ")?; - serde_json::to_writer(&mut writer, &search)?; - writer.write_all(b";")?; - writer.flush()?; - } - - // All files were written successfully - Ok::<_, anyhow::Error>(()) - }); -} - /// Render static and extra templates. pub fn render_templates( config: &Config, files: &Stream, nav: &Signal, diff --git a/python/tests/integration/test_search.py b/python/tests/integration/test_search.py new file mode 100644 index 0000000..c6d426a --- /dev/null +++ b/python/tests/integration/test_search.py @@ -0,0 +1,198 @@ +# Copyright (c) 2025-2026 Zensical and contributors + +# SPDX-License-Identifier: MIT +# All contributions are certified under the DCO + +"""Integration tests for MkDocs-compatible search artifacts.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import zensical + +if TYPE_CHECKING: + from pathlib import Path + + +_BUILD_OPTIONS: dict[str, Any] = {"clean": False, "strict": False} + + +def _write_project(root: Path, *, plugins: str) -> Path: + """Create a representative search project.""" + docs = root / "docs" + (docs / "guide").mkdir(parents=True) + (docs / "index.md").write_text( + """\ +--- +tags: + - alpha + - beta +--- + +# Landing + +Intro with fine print. + +## Overview + +Overview body. +""", + encoding="utf-8", + ) + (docs / "guide" / "topic.md").write_text( + """\ +--- +title: Metadata title +tags: + - guide +--- + +Preface before a heading. + +## Details + +Detailed body. +""", + encoding="utf-8", + ) + config = root / "mkdocs.yml" + config.write_text( + f"""\ +site_name: Search +nav: + - Home: index.md + - Guides: + - Topic: guide/topic.md +plugins: +{plugins} +""", + encoding="utf-8", + ) + return config + + +def _read_index(root: Path) -> dict[str, Any]: + """Read the generated search index.""" + return json.loads((root / "site" / "search.json").read_text()) + + +def test_search_artifacts_match_mkdocs_contract(tmp_path: Path) -> None: + """Search output preserves ordering, page facts, and offline framing.""" + config = _write_project( + tmp_path, + plugins=' - search:\n separator: "[\\\\s-]+"\n - offline', + ) + zensical.build(str(config), _BUILD_OPTIONS) + + expected = { + "config": {"lang": ["en"], "separator": "[\\s-]+"}, + "items": [ + { + "location": "index.html", + "level": 1, + "title": "Landing", + "text": "

Intro with fine print.

", + "path": ["Landing"], + "tags": ["alpha", "beta"], + }, + { + "location": "index.html#overview", + "level": 2, + "title": "Overview", + "text": "

Overview body.

", + "path": ["Landing"], + "tags": ["alpha", "beta"], + }, + { + "location": "guide/topic.html", + "level": 1, + "title": "Metadata title", + "text": "

Preface before a heading.

", + "path": ["Guides", "Metadata title"], + "tags": ["guide"], + }, + { + "location": "guide/topic.html#details", + "level": 2, + "title": "Details", + "text": "

Detailed body.

", + "path": ["Guides", "Metadata title"], + "tags": ["guide"], + }, + ], + } + assert _read_index(tmp_path) == expected + + compact = json.dumps(expected, separators=(",", ":"), ensure_ascii=False) + assert (tmp_path / "site" / "search.js").read_text() == ( + f"var __index = {compact};" + ) + + +def test_search_exclusion_and_disabled_output(tmp_path: Path) -> None: + """Excluded pages contribute no items and disabled search stays valid.""" + config = _write_project(tmp_path, plugins=" search:\n enabled: true") + topic = tmp_path / "docs" / "guide" / "topic.md" + topic.write_text( + """\ +--- +search: + exclude: true +--- + +# Hidden + +Not indexed. +""", + encoding="utf-8", + ) + zensical.build(str(config), _BUILD_OPTIONS) + assert [item["title"] for item in _read_index(tmp_path)["items"]] == [ + "Landing", + "Overview", + ] + + all_excluded = tmp_path / "all-excluded" + all_excluded.mkdir() + config = _write_project(all_excluded, plugins=" - search") + for page in (all_excluded / "docs").rglob("*.md"): + page.write_text( + "---\nsearch:\n exclude: true\n---\n\n# Hidden\n", + encoding="utf-8", + ) + zensical.build(str(config), _BUILD_OPTIONS) + assert _read_index(all_excluded)["items"] == [] + + disabled = tmp_path / "disabled" + disabled.mkdir() + config = _write_project( + disabled, plugins=" search:\n enabled: false" + ) + zensical.build(str(config), _BUILD_OPTIONS) + assert _read_index(disabled)["items"] == [] + + +def test_search_rebuild_replaces_changed_and_removed_pages( + tmp_path: Path, +) -> None: + """Successive builds do not retain stale page search facts.""" + config = _write_project(tmp_path, plugins=" - search") + zensical.build(str(config), _BUILD_OPTIONS) + + index = tmp_path / "docs" / "index.md" + index.write_text("# Changed\n\nFresh body.\n", encoding="utf-8") + (tmp_path / "docs" / "guide" / "topic.md").unlink() + zensical.build(str(config), _BUILD_OPTIONS) + + assert _read_index(tmp_path)["items"] == [ + { + "location": "", + "level": 1, + "title": "Changed", + "text": "

Fresh body.

", + "path": ["Changed"], + "tags": [], + } + ] diff --git a/python/zensical/extensions/search.py b/python/zensical/extensions/search.py deleted file mode 100644 index 9a5445f..0000000 --- a/python/zensical/extensions/search.py +++ /dev/null @@ -1,397 +0,0 @@ -# Copyright (c) 2025-2026 Zensical and contributors - -# SPDX-License-Identifier: MIT -# All contributions are certified under the DCO - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from __future__ import annotations - -from dataclasses import dataclass, field -from html import escape -from html.parser import HTMLParser -from typing import TYPE_CHECKING, Any - -from markdown import Extension -from markdown.postprocessors import Postprocessor - -if TYPE_CHECKING: - from markdown import Markdown - - -# ----------------------------------------------------------------------------- -# Classes -# ----------------------------------------------------------------------------- - - -@dataclass -class SearchConfig: - """Configuration for the Search Markdown extension.""" - - keep: set[str] = field(default_factory=set) - - -# ----------------------------------------------------------------------------- - - -class SearchProcessor(Postprocessor): - """Post processor to extract searchable content from the rendered HTML.""" - - name = "search" - - def __init__(self, md: Markdown, config: SearchConfig) -> None: - super().__init__(md) - self.config = config - self.data: list[dict[str, Any]] = [] - - def run(self, text: str) -> str: - """Process the rendered HTML and extract text length.""" - # Divide page content into sections - parser = Parser() - parser.feed(text) - parser.close() - - # Extract data from sections that are not excluded - self.data = [] - for section in parser.data: - if not section.is_excluded(): - # Compute title and text - title = "".join(section.title).strip() - content = "".join(section.text).strip() - - # Store data for external access - self.data.append( - { - "location": section.id, - "level": section.level, - "title": title, - "text": content, - "path": [], - "tags": [], - } - ) - - # Return the original HTML unchanged - return text - - -class SearchExtension(Extension): - """Markdown extension for search indexing.""" - - name = "zensical.extensions.search" - - def __init__(self, **kwargs: Any) -> None: - self._kwargs = kwargs - - def extendMarkdown(self, md: Markdown) -> None: - """Register the PostProcessor with Markdown.""" - config = SearchConfig(**self._kwargs) - processor = SearchProcessor(md, config) - md.postprocessors.register(processor, processor.name, 0) - - -def makeExtension(**kwargs: Any) -> SearchExtension: - """Register Markdown extension.""" - return SearchExtension(**kwargs) - - -# ----------------------------------------------------------------------------- - - -# HTML element -class Element: - """HTML element. - - An element with attributes, essentially a small wrapper object for the - parser to access attributes in other callbacks than handle_starttag. - """ - - # Initialize HTML element - def __init__( - self, tag: str, attrs: dict[str, str | None] | None = None - ) -> None: - self.tag = tag - self.attrs = attrs or {} - - # String representation - def __repr__(self): - return self.tag - - # Support comparison (compare by tag only) - def __eq__(self, other: object) -> bool: - if isinstance(other, Element): - return self.tag == other.tag - return self.tag == other - - # Support set operations - def __hash__(self): - return hash(self.tag) - - # Check whether the element should be excluded - def is_excluded(self) -> bool: - return "data-search-exclude" in self.attrs - - -# ----------------------------------------------------------------------------- - - -# HTML section -class Section: - """HTML section. - - A block of text with markup, preceded by a title (with markup), i.e., a - headline with a certain level (h1-h6). Internally used by the parser. - """ - - # Initialize HTML section - def __init__(self, el: Element, level: int, depth: int = 0) -> None: - self.el = el - self.depth: int | float = depth - self.level = level - - # Initialize section data - self.text: list[str] = [] - self.title: list[str] = [] - self.id: str | None = None - - # String representation - def __repr__(self): - if self.id: - return f"{self.el.tag}#{self.id}" - return self.el.tag - - # Check whether the section should be excluded - def is_excluded(self) -> bool: - return self.el.is_excluded() - - -# ----------------------------------------------------------------------------- - - -# HTML parser -class Parser(HTMLParser): - """Section divider. - - This parser divides the given string of HTML into a list of sections, each - of which are preceded by a h1-h6 level heading. A white- and blacklist of - tags dictates which tags should be preserved as part of the index, and - which should be ignored in their entirety. - """ - - # Initialize HTML parser - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - - # Tags to skip - self.skip: set[str | Element] = { - "object", # Objects - "script", # Scripts - "style", # Styles - } - - # Current context and section - self.context: list[Element] = [] - self.section: Section | None = None - - # All parsed sections - self.data: list[Section] = [] - - # Called at the start of every HTML tag - def handle_starttag( - self, tag: str, attrs: list[tuple[str, str | None]] - ) -> None: - attrs_dict = dict(attrs) - - # Ignore self-closing tags - el = Element(tag, attrs_dict) - if tag not in void: - self.context.append(el) - else: - return - - # Handle heading - if tag in ([f"h{x}" for x in range(1, 7)]): - depth = len(self.context) - if "id" in attrs_dict: - # Ensure top-level section - if tag != "h1" and not self.data: - self.section = Section(Element("hx"), 1, depth) - self.data.append(self.section) - - # Set identifier, if not first section - self.section = Section(el, int(tag[1:2]), depth) - if self.data: - self.section.id = attrs_dict["id"] - - # Append section to list - self.data.append(self.section) - - # Handle preface - ensure top-level section - if not self.section: - self.section = Section(Element("hx"), 1) - self.data.append(self.section) - - # Handle special cases to skip - for key, value in attrs_dict.items(): - # Skip block if explicitly excluded from search - if key == "data-search-exclude": - self.skip.add(el) - return - - # Skip line numbers - see https://bit.ly/3GvubZx - if key == "class" and value == "linenodiv": - self.skip.add(el) - return - - # Render opening tag if kept - if not self.skip.intersection(self.context) and tag in keep: - # Check whether we're inside the section title - data = self.section.text - if self.section.el in self.context: - data = self.section.title - - # Append to section title or text - data.append(f"<{tag}>") - - # Called at the end of every HTML tag - def handle_endtag(self, tag: str) -> None: - if not self.context or self.context[-1] != tag: - return - - # Check whether we're exiting the current context, which happens when - # a headline is nested in another element. In that case, we close the - # current section, continuing to append data to the previous section, - # which could also be a nested section – see https://bit.ly/3IxxIJZ - assert self.section is not None # noqa: S101 - if self.section.depth > len(self.context): - for section in reversed(self.data): - if section.depth <= len(self.context): - # Set depth to infinity in order to denote that the current - # section is exited and must never be considered again. - self.section.depth = float("inf") - self.section = section - break - - # Remove element from skip list - el = self.context.pop() - if el in self.skip: - if el.tag not in ["script", "style", "object"]: - self.skip.remove(el) - return - - # Render closing tag if kept - if not self.skip.intersection(self.context) and tag in keep: - # Check whether we're inside the section title - data = self.section.text - if self.section.el in self.context: - data = self.section.title - - # Search for corresponding opening tag - index = data.index(f"<{tag}>") - for i in range(index + 1, len(data)): - if not data[i].isspace(): - index = len(data) - break - - # Remove element if empty (or only whitespace) - if len(data) > index: - while len(data) > index: - data.pop() - - # Append to section title or text - else: - data.append(f"") - - # Called for the text contents of each tag - def handle_data(self, data: str) -> None: - if self.skip.intersection(self.context): - return - - # Collapse whitespace in non-pre contexts - if "pre" not in self.context: - if not data.isspace(): - data = data.replace("\n", " ") - else: - data = " " - - # Handle preface - ensure top-level section - if not self.section: - self.section = Section(Element("hx"), 1) - self.data.append(self.section) - - # Handle section headline - if self.section.el in self.context: - permalink = False - for el in self.context: - if el.tag == "a" and el.attrs.get("class") == "headerlink": - permalink = True - - # Ignore permalinks - if not permalink: - self.section.title.append(escape(data, quote=False)) - - # Collapse adjacent whitespace - elif data.isspace(): - if ( - not self.section.text - or not self.section.text[-1].isspace() - or "pre" in self.context - ): - self.section.text.append(data) - - # Handle everything else - else: - self.section.text.append(escape(data, quote=False)) - - -# ----------------------------------------------------------------------------- -# Data -# ----------------------------------------------------------------------------- - - -# Tags to keep -keep = { - "p", - "code", - "pre", - "li", - "ol", - "ul", - "small", - "sub", - "sup", -} - -# Tags that are self-closing -void = { - "area", - "base", - "br", - "col", - "embed", - "hr", - "img", - "input", - "link", - "meta", - "param", - "source", - "track", - "wbr", -} diff --git a/python/zensical/markdown/render.py b/python/zensical/markdown/render.py index ac598cd..97869a9 100644 --- a/python/zensical/markdown/render.py +++ b/python/zensical/markdown/render.py @@ -25,7 +25,7 @@ from __future__ import annotations import re from datetime import date, datetime -from typing import TYPE_CHECKING, Any +from typing import Any import yaml from markdown import Markdown @@ -35,10 +35,6 @@ from zensical.config import get_config from zensical.extensions.autorefs import set_autorefs_page from zensical.extensions.context import ContextExtension, Page from zensical.extensions.links import LinksExtension -from zensical.extensions.search import SearchExtension - -if TYPE_CHECKING: - from zensical.extensions.search import SearchProcessor # ---------------------------------------------------------------------------- # Constants @@ -110,14 +106,11 @@ def render(content: str, path: str, url: str) -> dict: extension_configs=config["mdx_configs"], ) - # Note: mkdocstrings and markdown-exec do not need to propagate - # the links and search extensions to their inner Markdown instances: - # their postprocessors run last and can see inner layer contents. - # More importantly, inner layers *must not* run the links and search - # extensions: the inner links treeprocessor would transform links once, - # and the outer links postprocessor would transform them again. - # The search postprocessor would run twice for generated content, - # incurring a performance cost. + # Note: mkdocstrings and markdown-exec do not need to propagate the links + # extension to their inner Markdown instances. Its postprocessor runs last + # and can see inner layer contents. More importantly, inner layers *must + # not* run the extension: the inner treeprocessor would transform links + # once, and the outer postprocessor would transform them again. # Register links extension, which is equivalent to MkDocs' path resolution # Markdown extension. This is a bandaid, until we move this to Rust @@ -126,10 +119,6 @@ def render(content: str, path: str, url: str) -> dict: ) links.extendMarkdown(md) - # Register search extension, which extracts text for search indexing - search_extension = SearchExtension() - search_extension.extendMarkdown(md) - # Inform markdown-exec that it runs through Zensical. try: import markdown_exec # noqa: PLC0415 # ty:ignore[unresolved-import] @@ -141,11 +130,6 @@ def render(content: str, path: str, url: str) -> dict: # Convert content to HTML content = md.convert(content) - # Obtain search index data, unless page is excluded - search_processor: SearchProcessor = md.postprocessors["search"] - if meta.get("search", {}).get("exclude", False): - search_processor.data = [] - # Sanitize metadata before passing it to Rust meta = {k: _sanitize(v) for k, v in meta.items()} @@ -154,7 +138,6 @@ def render(content: str, path: str, url: str) -> dict: "meta": meta, "title": "", "content": content, - "search": search_processor.data, "toc": [_convert_toc(item) for item in getattr(md, "toc_tokens", [])], }