refactor: migrate search MkDocs plugin replacement

Signed-off-by: squidfunk <martin.donath@squidfunk.com>

Cargo.lock

@@ -293,6 +293,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "html5gum"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "428502d3ec1742c35e015871aef742bc4722a9acfcb616b8ff79b519922e1c36"
+
 [[package]]
 name = "httparse"
 version = "1.10.1"
@@ -1474,6 +1480,7 @@ dependencies = [
  "ariadne",
  "crossbeam",
  "fluent-uri",
+ "html5gum",
  "minijinja",
  "minijinja-contrib",
  "mio",

Cargo.toml

@@ -50,6 +50,7 @@ base64 = "0.22"
 crossbeam = "0.8"
 file-id = "0.2"
 fluent-uri = "0.4"
+html5gum = { version = "0.8.4", default-features = false }
 httparse = "1.10"
 httpdate = "1.0"
 indicatif = "0.18"

crates/zensical/Cargo.toml

@@ -50,6 +50,7 @@ anyhow.workspace = true
 ariadne.workspace = true
 crossbeam.workspace = true
 fluent-uri.workspace = true
+html5gum.workspace = true
 minijinja = { workspace = true, features = [
   "json", "loader", "builtins", "urlencode"
 ] }

crates/zensical/src/compat.rs

@@ -0,0 +1,8 @@
+// Copyright (c) 2025-2026 Zensical and contributors
+
+// SPDX-License-Identifier: MIT
+// All contributions are certified under the DCO
+
+//! Compatibility implementations for established ecosystems.
+
+pub mod mkdocs;

crates/zensical/src/compat/mkdocs.rs

@@ -0,0 +1,8 @@
+// Copyright (c) 2025-2026 Zensical and contributors
+
+// SPDX-License-Identifier: MIT
+// All contributions are certified under the DCO
+
+//! MkDocs compatibility modules.
+
+pub mod search;

crates/zensical/src/compat/mkdocs/search.rs

@@ -0,0 +1,275 @@
+// Copyright (c) 2025-2026 Zensical and contributors
+
+// SPDX-License-Identifier: MIT
+// All contributions are certified under the DCO
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+
+// ----------------------------------------------------------------------------
+
+//! MkDocs-compatible search index.
+
+use serde::Serialize;
+use std::collections::BTreeMap;
+use std::fs;
+use std::io::{BufWriter, Write};
+use zrx::id::Id;
+use zrx::scheduler::Value;
+use zrx::stream::function::Collection;
+use zrx::stream::{Key, Signal, Stream};
+
+use crate::config::plugins::SearchPluginConfig;
+use crate::config::Config;
+use crate::structure::dynamic::Dynamic;
+use crate::structure::nav::{file_sort_key, Navigation};
+use crate::structure::page::Page;
+
+mod item;
+mod parser;
+
+use item::{SearchItem, SearchSection};
+pub(crate) use parser::extract;
+
+// ----------------------------------------------------------------------------
+// Structs
+// ----------------------------------------------------------------------------
+
+/// Search configuration.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+struct SearchConfig {
+    /// Languages for tokenizer.
+    lang: Vec<String>,
+    /// Separator for tokenizer.
+    separator: String,
+}
+
+/// Complete search artifact.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+struct SearchIndex {
+    /// Search configuration.
+    config: SearchConfig,
+    /// Search items.
+    items: Vec<SearchItem>,
+}
+
+/// Compact page facts retained by the search branch.
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct SearchDocument {
+    /// Page target URL.
+    url: String,
+    /// Page title.
+    title: String,
+    /// Page tag names.
+    tags: Vec<String>,
+    /// Page-local search sections.
+    sections: Vec<SearchSection>,
+}
+
+// ----------------------------------------------------------------------------
+// Implementations
+// ----------------------------------------------------------------------------
+
+impl SearchConfig {
+    /// Creates search configuration for the configured theme language.
+    fn new(config: SearchPluginConfig, language: &str) -> Self {
+        Self {
+            lang: vec![language.to_string()],
+            separator: config.separator,
+        }
+    }
+}
+
+// ----------------------------------------------------------------------------
+
+impl SearchDocument {
+    /// Extracts the facts search needs from a rendered page.
+    fn new(page: &Page) -> Self {
+        Self {
+            url: page.url.clone(),
+            title: page.title.clone(),
+            tags: page.tags().into_iter().map(|tag| tag.name).collect(),
+            sections: extract(&page.content),
+        }
+    }
+}
+
+// ----------------------------------------------------------------------------
+
+impl SearchIndex {
+    /// Creates a search index from compact page facts.
+    #[allow(clippy::assigning_clones)]
+    fn new(
+        documents: Vec<(Key<Id>, SearchDocument)>, nav: &Navigation,
+        config: SearchPluginConfig, language: &str,
+    ) -> Self {
+        let mut items: Vec<SearchItem> = Vec::new();
+
+        let mut documents = Vec::from_iter(documents);
+        documents.sort_by_key(|(id, _)| file_sort_key(&id[0]));
+
+        // Attach site-wide navigation facts only while assembling the final
+        // artifact, keeping them out of each page-local stream value.
+        for (_id, document) in documents {
+            let iter = nav.ancestors_for_url(&document.url).into_iter().rev();
+            let mut path = iter
+                .filter_map(|item| {
+                    item.display_title().map(ToString::to_string)
+                })
+                .collect::<Vec<_>>();
+
+            // Add page title to path if not already present - this might be
+            // the true in case of index pages
+            if path.last() != Some(&document.title) {
+                path.push(document.title.clone());
+            }
+
+            for section in document.sections {
+                let location = match section.location {
+                    Some(id) => format!("{}#{}", document.url, id),
+                    _ => document.url.clone(),
+                };
+                let title = if section.title.is_empty() {
+                    document.title.clone()
+                } else {
+                    section.title
+                };
+                items.push(SearchItem {
+                    location: Some(location),
+                    level: section.level,
+                    title,
+                    text: section.text,
+                    path: path.clone(),
+                    tags: document.tags.clone(),
+                });
+            }
+        }
+
+        // Return search
+        Self {
+            config: SearchConfig::new(config, language),
+            items,
+        }
+    }
+}
+
+// ----------------------------------------------------------------------------
+// Trait implementations
+// ----------------------------------------------------------------------------
+
+impl Value for SearchDocument {}
+
+// ----------------------------------------------------------------------------
+// Functions
+// ----------------------------------------------------------------------------
+
+/// Attach MkDocs-compatible search artifact generation to the build graph.
+pub(crate) fn attach(
+    config: &Config, pages: &Stream<Id, Page>, nav: &Signal<Id, Navigation>,
+) {
+    if !config.project.plugins.search.config.enabled {
+        let config = config.clone();
+        let _ = nav.map(move |nav: &Navigation| {
+            let search = SearchIndex::new(
+                Vec::new(),
+                nav,
+                config.project.plugins.search.config.clone(),
+                &config.project.theme.language,
+            );
+            write(&config, &search)
+        });
+        return;
+    }
+
+    let documents = pages
+        .filter(|page: &Page| !is_search_excluded(&page.meta))
+        .map(SearchDocument::new);
+    let documents = documents.reduce(
+        |documents: &dyn Collection<Key<Id>, SearchDocument>| {
+            Some(
+                documents
+                    .iter()
+                    .map(|(key, document)| (key.clone(), document.clone()))
+                    .collect::<Vec<_>>(),
+            )
+        },
+    );
+    let config = config.clone();
+    let _ = documents.product(nav).map(
+        move |documents: &Vec<(Key<Id>, SearchDocument)>, nav: &Navigation| {
+            let search = SearchIndex::new(
+                documents.clone(),
+                nav,
+                config.project.plugins.search.config.clone(),
+                &config.project.theme.language,
+            );
+            write(&config, &search)
+        },
+    );
+}
+
+/// Write search artifacts without retaining a second serialized copy.
+fn write(config: &Config, search: &SearchIndex) -> anyhow::Result<()> {
+    let site_dir = config.get_site_dir();
+    let path = site_dir.join("search.json");
+    fs::create_dir_all(path.parent().expect("invariant"))?;
+    let mut writer = BufWriter::new(fs::File::create(path)?);
+    serde_json::to_writer(&mut writer, search)?;
+    writer.flush()?;
+
+    if config.project.plugins.offline.config.enabled {
+        let path = site_dir.join("search.js");
+        fs::create_dir_all(path.parent().expect("invariant"))?;
+        let mut writer = BufWriter::new(fs::File::create(path)?);
+        writer.write_all(b"var __index = ")?;
+        serde_json::to_writer(&mut writer, search)?;
+        writer.write_all(b";")?;
+        writer.flush()?;
+    }
+    Ok(())
+}
+
+/// Returns whether a page is excluded from search through its metadata.
+fn is_search_excluded(meta: &BTreeMap<String, Dynamic>) -> bool {
+    let Some(Dynamic::Map(search)) = meta.get("search") else {
+        return false;
+    };
+    matches!(search.get("exclude"), Some(Dynamic::Bool(true)))
+}
+
+// ----------------------------------------------------------------------------
+// Tests
+// ----------------------------------------------------------------------------
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn search_exclusion_is_read_from_page_metadata() {
+        let mut search = BTreeMap::new();
+        search.insert(String::from("exclude"), Dynamic::Bool(true));
+        let mut meta = BTreeMap::new();
+        meta.insert(String::from("search"), Dynamic::Map(search));
+
+        assert!(is_search_excluded(&meta));
+
+        meta.clear();
+        assert!(!is_search_excluded(&meta));
+    }
+}

crates/zensical/src/compat/mkdocs/search/item.rs

@@ -23,18 +23,16 @@
 
 // ----------------------------------------------------------------------------
 
-//! Search item.
+//! MkDocs-compatible search item.
 
-use pyo3::FromPyObject;
-use serde::{Deserialize, Serialize};
+use serde::Serialize;
 
 // ----------------------------------------------------------------------------
 // Structs
 // ----------------------------------------------------------------------------
 
 /// Search item.
-#[derive(Clone, Debug, PartialEq, Eq, FromPyObject, Serialize, Deserialize)]
-#[pyo3(from_item_all)]
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
 pub struct SearchItem {
     /// Search location.
     pub location: Option<String>,
@@ -49,3 +47,18 @@ pub struct SearchItem {
     /// Section tags.
     pub tags: Vec<String>,
 }
+
+// ----------------------------------------------------------------------------
+
+/// Page-local search section before site-wide facts are attached.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) struct SearchSection {
+    /// Heading fragment, if present.
+    pub location: Option<String>,
+    /// Section level.
+    pub level: u32,
+    /// Section title.
+    pub title: String,
+    /// Section text.
+    pub text: String,
+}

crates/zensical/src/compat/mkdocs/search/parser.rs

@@ -0,0 +1,776 @@
+// Copyright (c) 2025-2026 Zensical and contributors
+
+// SPDX-License-Identifier: MIT
+// All contributions are certified under the DCO
+
+//! MkDocs-compatible search extraction from rendered HTML.
+
+use html5gum::emitters::callback::{CallbackEmitter, CallbackEvent};
+use html5gum::{Span, Tokenizer};
+use std::convert::Infallible;
+
+use super::SearchSection;
+
+/// Extract page-local search sections from rendered HTML.
+pub(crate) fn extract(html: &str) -> Vec<SearchSection> {
+    let mut parser = SearchParser::default();
+
+    {
+        let mut emitter = CallbackEmitter::new(
+            |event: CallbackEvent<'_>, _span: Span<()>| -> Option<Infallible> {
+                parser.handle(event);
+                None
+            },
+        );
+        emitter.naively_switch_states(true);
+
+        Tokenizer::new_with_emitter(html, emitter)
+            .finish()
+            .expect("string input is infallible");
+    }
+
+    parser.finish()
+}
+
+// ----------------------------------------------------------------------------
+// Parser
+// ----------------------------------------------------------------------------
+
+/// Streaming search parser.
+#[derive(Default)]
+struct SearchParser {
+    /// Open HTML elements.
+    context: Vec<Element>,
+    /// Section currently receiving text.
+    current: Option<usize>,
+    /// Sections extracted from the input.
+    sections: Vec<SectionState>,
+    /// Number of excluded elements currently open.
+    skip: usize,
+    /// Start tag currently emitted by the tokenizer.
+    start: Option<StartTag>,
+    /// Current attribute of the pending start tag.
+    attribute: Attribute,
+}
+
+impl SearchParser {
+    /// Handles a tokenizer event.
+    fn handle(&mut self, event: CallbackEvent<'_>) {
+        match event {
+            CallbackEvent::OpenStartTag { name } => {
+                self.start = Some(StartTag::new(Tag::from_bytes(name)));
+                self.attribute = Attribute::Other;
+            }
+            CallbackEvent::AttributeName { name } => {
+                self.attribute = Attribute::from_bytes(name);
+                if let Some(start) = &mut self.start {
+                    start.observe_attribute(self.attribute, None);
+                }
+            }
+            CallbackEvent::AttributeValue { value } => {
+                if let Some(start) = &mut self.start {
+                    start.observe_attribute(self.attribute, Some(value));
+                }
+            }
+            CallbackEvent::CloseStartTag { self_closing } => {
+                if let Some(start) = self.start.take() {
+                    let tag = start.tag.clone();
+                    self.start(start);
+                    if self_closing {
+                        self.end(&tag);
+                    }
+                }
+            }
+            CallbackEvent::EndTag { name } => {
+                self.end(&Tag::from_bytes(name));
+            }
+            CallbackEvent::String { value } => {
+                self.text(String::from_utf8_lossy(value).as_ref());
+            }
+            CallbackEvent::Comment { .. }
+            | CallbackEvent::Doctype { .. }
+            | CallbackEvent::Error(_) => {}
+        }
+    }
+
+    /// Handles a complete start tag.
+    fn start(&mut self, start: StartTag) {
+        if start.tag.is_void() {
+            return;
+        }
+
+        let heading = start.tag.heading_level();
+        let skipped = start.tag.is_skipped()
+            || start.excluded
+            || start.class.as_deref() == Some(b"linenodiv");
+        let headerlink = start.tag == Tag::A
+            && start.class.as_deref() == Some(b"headerlink");
+        let tag = start.tag.clone();
+
+        self.context.push(Element {
+            tag: start.tag,
+            skipped,
+            headerlink,
+            kept: None,
+        });
+
+        if let (Some(level), true) = (heading, start.id_present) {
+            let depth = self.context.len();
+
+            if level != 1 && self.sections.is_empty() {
+                self.push_preface();
+            }
+
+            let location = if self.sections.is_empty() {
+                None
+            } else {
+                start.id
+            };
+            let section = SectionState::new(
+                Some(level),
+                level.into(),
+                depth,
+                location,
+                start.excluded,
+            );
+            self.sections.push(section);
+            self.current = Some(self.sections.len() - 1);
+        }
+
+        self.ensure_section();
+
+        if skipped {
+            self.skip += 1;
+            return;
+        }
+
+        if self.skip == 0 && tag.is_kept() {
+            let (section, title) = self.output_target();
+            let data = self.sections[section].output_mut(title);
+            let start = data.value.len();
+            let previous_whitespace = data.last_whitespace;
+            data.value.push('<');
+            data.value.push_str(tag.name());
+            data.value.push('>');
+            data.last_whitespace = false;
+
+            self.context.last_mut().expect("context").kept =
+                Some(KeptElement {
+                    section,
+                    title,
+                    start,
+                    previous_whitespace,
+                });
+        }
+    }
+
+    /// Handles an end tag.
+    fn end(&mut self, tag: &Tag) {
+        if self.context.last().is_none_or(|el| el.tag != *tag) {
+            return;
+        }
+
+        let depth = self.context.len();
+        if let Some(index) = self.current
+            && self.sections[index].exited_or_deeper_than(depth)
+            && let Some(parent) = self
+                .sections
+                .iter()
+                .rposition(|section| !section.exited && section.depth <= depth)
+        {
+            self.sections[index].exited = true;
+            self.current = Some(parent);
+        }
+
+        let element = self.context.pop().expect("context");
+        if element.skipped {
+            self.skip -= 1;
+            return;
+        }
+
+        if self.skip == 0 && tag.is_kept() {
+            let (section, title) = self.output_target();
+            let data = self.sections[section].output_mut(title);
+            let opening = format!("<{}>", tag.name());
+
+            if let Some(start) = data.value.find(&opening) {
+                let following = &data.value[start + opening.len()..];
+                if following.chars().any(|char| !char.is_whitespace()) {
+                    data.value.push_str("</");
+                    data.value.push_str(tag.name());
+                    data.value.push('>');
+                    data.last_whitespace = false;
+                } else {
+                    data.value.truncate(start);
+                    data.last_whitespace = element.kept.map_or_else(
+                        || {
+                            data.value
+                                .chars()
+                                .last()
+                                .is_some_and(char::is_whitespace)
+                        },
+                        |kept| {
+                            debug_assert_eq!(kept.section, section);
+                            debug_assert_eq!(kept.title, title);
+                            debug_assert_eq!(kept.start, start);
+                            kept.previous_whitespace
+                        },
+                    );
+                }
+            }
+        }
+    }
+
+    /// Handles text content.
+    fn text(&mut self, value: &str) {
+        if self.skip > 0 {
+            return;
+        }
+
+        let preformatted = self.context.iter().any(|el| el.tag == Tag::Pre);
+        let whitespace = value.chars().all(char::is_whitespace);
+        let text;
+        let value = if preformatted {
+            value
+        } else if whitespace {
+            " "
+        } else if value.contains('\n') {
+            text = value.replace('\n', " ");
+            &text
+        } else {
+            value
+        };
+
+        self.ensure_section();
+        let (section, title) = self.output_target();
+
+        if title {
+            if self.context.iter().any(|el| el.headerlink) {
+                return;
+            }
+            escape(value, &mut self.sections[section].title.value);
+            self.sections[section].title.last_whitespace = whitespace;
+        } else {
+            let data = &mut self.sections[section].text;
+            if !whitespace || preformatted || !data.last_whitespace {
+                escape(value, &mut data.value);
+                data.last_whitespace = whitespace;
+            }
+        }
+    }
+
+    /// Returns the current section and whether its title receives output.
+    fn output_target(&self) -> (usize, bool) {
+        let section = self.current.expect("section");
+        let heading = self.sections[section].heading;
+        let title = heading.is_some_and(|level| {
+            self.context
+                .iter()
+                .any(|el| el.tag.heading_level() == Some(level))
+        });
+        (section, title)
+    }
+
+    /// Ensures a section exists for preface content.
+    fn ensure_section(&mut self) {
+        if self.current.is_none() {
+            self.push_preface();
+        }
+    }
+
+    /// Adds the implicit top-level preface section.
+    fn push_preface(&mut self) {
+        self.sections
+            .push(SectionState::new(None, 1, 0, None, false));
+        self.current = Some(self.sections.len() - 1);
+    }
+
+    /// Converts parser state into page-local search sections.
+    fn finish(self) -> Vec<SearchSection> {
+        self.sections
+            .into_iter()
+            .filter(|section| !section.excluded)
+            .map(|section| SearchSection {
+                location: section.location,
+                level: section.level,
+                title: trim(section.title.value),
+                text: trim(section.text.value),
+            })
+            .collect()
+    }
+}
+
+// ----------------------------------------------------------------------------
+// State
+// ----------------------------------------------------------------------------
+
+/// Section being assembled.
+struct SectionState {
+    heading: Option<u8>,
+    level: u32,
+    depth: usize,
+    exited: bool,
+    excluded: bool,
+    location: Option<String>,
+    title: Output,
+    text: Output,
+}
+
+impl SectionState {
+    fn new(
+        heading: Option<u8>, level: u32, depth: usize,
+        location: Option<String>, excluded: bool,
+    ) -> Self {
+        Self {
+            heading,
+            level,
+            depth,
+            exited: false,
+            excluded,
+            location,
+            title: Output::default(),
+            text: Output::default(),
+        }
+    }
+
+    fn exited_or_deeper_than(&self, depth: usize) -> bool {
+        self.exited || self.depth > depth
+    }
+
+    fn output_mut(&mut self, title: bool) -> &mut Output {
+        if title {
+            &mut self.title
+        } else {
+            &mut self.text
+        }
+    }
+}
+
+/// Output buffer and whitespace state.
+#[derive(Default)]
+struct Output {
+    value: String,
+    last_whitespace: bool,
+}
+
+/// Open HTML element.
+struct Element {
+    tag: Tag,
+    skipped: bool,
+    headerlink: bool,
+    kept: Option<KeptElement>,
+}
+
+/// Location at which a retained element was opened.
+#[derive(Clone, Copy)]
+struct KeptElement {
+    section: usize,
+    title: bool,
+    start: usize,
+    previous_whitespace: bool,
+}
+
+/// Start tag under construction.
+struct StartTag {
+    tag: Tag,
+    id_present: bool,
+    id: Option<String>,
+    excluded: bool,
+    class: Option<Vec<u8>>,
+}
+
+impl StartTag {
+    fn new(tag: Tag) -> Self {
+        Self {
+            tag,
+            id_present: false,
+            id: None,
+            excluded: false,
+            class: None,
+        }
+    }
+
+    fn observe_attribute(
+        &mut self, attribute: Attribute, value: Option<&[u8]>,
+    ) {
+        match attribute {
+            Attribute::Id => {
+                self.id_present = true;
+                self.id = value
+                    .map(|value| String::from_utf8_lossy(value).into_owned());
+            }
+            Attribute::Class => {
+                self.class = value.map(<[u8]>::to_vec);
+            }
+            Attribute::SearchExclude => self.excluded = true,
+            Attribute::Other => {}
+        }
+    }
+}
+
+/// Attributes relevant to extraction.
+#[derive(Clone, Copy, Default)]
+enum Attribute {
+    Id,
+    Class,
+    SearchExclude,
+    #[default]
+    Other,
+}
+
+impl Attribute {
+    fn from_bytes(value: &[u8]) -> Self {
+        match value {
+            b"id" => Self::Id,
+            b"class" => Self::Class,
+            b"data-search-exclude" => Self::SearchExclude,
+            _ => Self::Other,
+        }
+    }
+}
+
+/// HTML tags relevant to extraction.
+#[derive(Clone, Debug, PartialEq, Eq)]
+enum Tag {
+    A,
+    Area,
+    Base,
+    Br,
+    Code,
+    Col,
+    Embed,
+    H(u8),
+    Hr,
+    Img,
+    Input,
+    Li,
+    Link,
+    Meta,
+    Object,
+    Ol,
+    P,
+    Param,
+    Pre,
+    Script,
+    Small,
+    Source,
+    Style,
+    Sub,
+    Sup,
+    Track,
+    Ul,
+    Wbr,
+    Other(Box<str>),
+}
+
+impl Tag {
+    fn from_bytes(value: &[u8]) -> Self {
+        match value {
+            b"a" => Self::A,
+            b"area" => Self::Area,
+            b"base" => Self::Base,
+            b"br" => Self::Br,
+            b"code" => Self::Code,
+            b"col" => Self::Col,
+            b"embed" => Self::Embed,
+            b"h1" => Self::H(1),
+            b"h2" => Self::H(2),
+            b"h3" => Self::H(3),
+            b"h4" => Self::H(4),
+            b"h5" => Self::H(5),
+            b"h6" => Self::H(6),
+            b"hr" => Self::Hr,
+            b"img" => Self::Img,
+            b"input" => Self::Input,
+            b"li" => Self::Li,
+            b"link" => Self::Link,
+            b"meta" => Self::Meta,
+            b"object" => Self::Object,
+            b"ol" => Self::Ol,
+            b"p" => Self::P,
+            b"param" => Self::Param,
+            b"pre" => Self::Pre,
+            b"script" => Self::Script,
+            b"small" => Self::Small,
+            b"source" => Self::Source,
+            b"style" => Self::Style,
+            b"sub" => Self::Sub,
+            b"sup" => Self::Sup,
+            b"track" => Self::Track,
+            b"ul" => Self::Ul,
+            b"wbr" => Self::Wbr,
+            _ => {
+                Self::Other(String::from_utf8_lossy(value).into_owned().into())
+            }
+        }
+    }
+
+    fn name(&self) -> &str {
+        match self {
+            Self::A => "a",
+            Self::Area => "area",
+            Self::Base => "base",
+            Self::Br => "br",
+            Self::Code => "code",
+            Self::Col => "col",
+            Self::Embed => "embed",
+            Self::H(1) => "h1",
+            Self::H(2) => "h2",
+            Self::H(3) => "h3",
+            Self::H(4) => "h4",
+            Self::H(5) => "h5",
+            Self::H(6) => "h6",
+            Self::H(_) => unreachable!("heading level"),
+            Self::Hr => "hr",
+            Self::Img => "img",
+            Self::Input => "input",
+            Self::Li => "li",
+            Self::Link => "link",
+            Self::Meta => "meta",
+            Self::Object => "object",
+            Self::Ol => "ol",
+            Self::P => "p",
+            Self::Param => "param",
+            Self::Pre => "pre",
+            Self::Script => "script",
+            Self::Small => "small",
+            Self::Source => "source",
+            Self::Style => "style",
+            Self::Sub => "sub",
+            Self::Sup => "sup",
+            Self::Track => "track",
+            Self::Ul => "ul",
+            Self::Wbr => "wbr",
+            Self::Other(name) => name,
+        }
+    }
+
+    fn heading_level(&self) -> Option<u8> {
+        if let Self::H(level) = self {
+            Some(*level)
+        } else {
+            None
+        }
+    }
+
+    fn is_kept(&self) -> bool {
+        matches!(
+            self,
+            Self::P
+                | Self::Code
+                | Self::Pre
+                | Self::Li
+                | Self::Ol
+                | Self::Ul
+                | Self::Small
+                | Self::Sub
+                | Self::Sup
+        )
+    }
+
+    fn is_skipped(&self) -> bool {
+        matches!(self, Self::Object | Self::Script | Self::Style)
+    }
+
+    fn is_void(&self) -> bool {
+        matches!(
+            self,
+            Self::Area
+                | Self::Base
+                | Self::Br
+                | Self::Col
+                | Self::Embed
+                | Self::Hr
+                | Self::Img
+                | Self::Input
+                | Self::Link
+                | Self::Meta
+                | Self::Param
+                | Self::Source
+                | Self::Track
+                | Self::Wbr
+        )
+    }
+}
+
+// ----------------------------------------------------------------------------
+// Helpers
+// ----------------------------------------------------------------------------
+
+/// Escapes text like `html.escape(..., quote=False)`.
+fn escape(value: &str, output: &mut String) {
+    for char in value.chars() {
+        match char {
+            '&' => output.push_str("&amp;"),
+            '<' => output.push_str("&lt;"),
+            '>' => output.push_str("&gt;"),
+            _ => output.push(char),
+        }
+    }
+}
+
+/// Trims Unicode whitespace without retaining the original allocation twice.
+fn trim(mut value: String) -> String {
+    let end = value.trim_end().len();
+    value.truncate(end);
+    let start = value.len() - value.trim_start().len();
+    value.drain(..start);
+    value
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn item(
+        location: Option<&str>, level: u32, title: &str, text: &str,
+    ) -> SearchSection {
+        SearchSection {
+            location: location.map(ToString::to_string),
+            level,
+            title: title.to_string(),
+            text: text.to_string(),
+        }
+    }
+
+    #[test]
+    fn extracts_preface() {
+        let html = "<p>Before <em>heading</em>.</p>";
+        assert_eq!(
+            extract(html),
+            vec![item(None, 1, "", "<p>Before heading.</p>")]
+        );
+    }
+
+    #[test]
+    fn divides_content_into_sections() {
+        let html = concat!(
+            r##"<h1 id="top">Top <code>code</code>"##,
+            r##"<a class="headerlink" href="#top">¶</a></h1>"##,
+            r#"<p>First &amp; second.</p>"#,
+            r#"<h2 id="child">Child</h2><p>Body</p>"#,
+        );
+        assert_eq!(
+            extract(html),
+            vec![
+                item(
+                    None,
+                    1,
+                    "Top <code>code</code>",
+                    "<p>First &amp; second.</p>"
+                ),
+                item(Some("child"), 2, "Child", "<p>Body</p>"),
+            ]
+        );
+    }
+
+    #[test]
+    fn preserves_missing_heading_id_behavior() {
+        let html = "<h2>No ID</h2><p>Body</p>";
+        assert_eq!(extract(html), vec![item(None, 1, "", "No ID<p>Body</p>")]);
+    }
+
+    #[test]
+    fn excludes_configured_content() {
+        let html = concat!(
+            r#"<h1 id="top">Top</h1><p>Keep</p>"#,
+            r#"<div data-search-exclude><p>Drop</p></div><p>After</p>"#,
+            r#"<div class="linenodiv"><pre>1</pre></div>"#,
+            r#"<script>ignored <b>script</b></script>"#,
+        );
+        assert_eq!(
+            extract(html),
+            vec![item(None, 1, "Top", "<p>Keep</p><p>After</p>")]
+        );
+    }
+
+    #[test]
+    fn preserves_selected_markup_and_empty_elements() {
+        let html = concat!(
+            r#"<h1 id="top">Top</h1><p>Text <small>small</small></p>"#,
+            r#"<p>   </p><ul><li>One</li><li><code>x</code></li></ul>"#,
+        );
+        assert_eq!(
+            extract(html),
+            vec![item(
+                None,
+                1,
+                "Top",
+                "<p>Text <small>small</small></p><p> </p><ul><li>One</li><li><code>x</code></li></ul>",
+            )]
+        );
+    }
+
+    #[test]
+    fn preserves_whitespace_and_preformatted_content() {
+        let html = concat!(
+            "<h1 id=\"top\">Top</h1><p>one\n  two</p>",
+            "<pre><code>a &lt; b\n  c</code></pre>",
+        );
+        assert_eq!(
+            extract(html),
+            vec![item(
+                None,
+                1,
+                "Top",
+                "<p>one   two</p><pre><code>a &lt; b\n  c</code></pre>",
+            )]
+        );
+    }
+
+    #[test]
+    fn decodes_and_escapes_entities_like_python() {
+        let html = concat!(
+            r#"<h1 id="top">A &amp; B &#169;</h1>"#,
+            r#"<p>&lt;tag&gt; &quot;x&quot; &apos;y&apos; &nbsp;</p>"#,
+        );
+        assert_eq!(
+            extract(html),
+            vec![item(
+                None,
+                1,
+                "A &amp; B ©",
+                "<p>&lt;tag&gt; \"x\" 'y' \u{a0}</p>"
+            )]
+        );
+    }
+
+    #[test]
+    fn restores_parent_section_after_nested_heading() {
+        let html = concat!(
+            r#"<div><h2 id="nested">Nested</h2><p>Inside</p></div>"#,
+            r#"<p>Outside</p>"#,
+        );
+        assert_eq!(
+            extract(html),
+            vec![
+                item(None, 1, "", "<p>Outside</p>"),
+                item(Some("nested"), 2, "Nested", "<p>Inside</p>"),
+            ]
+        );
+    }
+
+    #[test]
+    fn preserves_malformed_html_behavior() {
+        let html = concat!(
+            r#"<h1 id="top">Top</h1><p>Before <code>open</p>"#,
+            r#"<h2 id="next">Next</h2><p>After"#,
+        );
+        assert_eq!(
+            extract(html),
+            vec![
+                item(None, 1, "Top", "<p>Before <code>open"),
+                item(Some("next"), 2, "Next", "<p>After"),
+            ]
+        );
+    }
+
+    #[test]
+    fn ignores_void_elements() {
+        let html = concat!(
+            r#"<h1 id="top">Top</h1>"#,
+            r#"<p>A<br>B<img src="x">C<hr>D</p>"#,
+        );
+        assert_eq!(extract(html), vec![item(None, 1, "Top", "<p>ABCD</p>")]);
+    }
+}

crates/zensical/src/lib.rs

@@ -40,6 +40,7 @@ use std::time::{Duration, Instant};
 use std::{fs, io, thread};
 use zrx::id::Id;
 
+mod compat;
 mod config;
 mod python;
 mod server;

crates/zensical/src/structure.rs

@@ -29,6 +29,5 @@ pub mod dynamic;
 pub mod markdown;
 pub mod nav;
 pub mod page;
-pub mod search;
 pub mod tag;
 pub mod toc;

crates/zensical/src/structure/markdown.rs

@@ -37,7 +37,6 @@ use zrx::stream::Value;
 
 use crate::structure::dynamic::Dynamic;
 use crate::structure::nav::to_title;
-use crate::structure::search::SearchItem;
 use crate::structure::toc::Section;
 
 mod autorefs;
@@ -61,21 +60,32 @@ pub struct Markdown {
 }
 
 /// Immutable rendered Markdown data.
-#[derive(Debug, FromPyObject, Serialize, Deserialize)]
-#[pyo3(from_item_all)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct MarkdownData {
     /// Markdown metadata.
     pub meta: BTreeMap<String, Dynamic>,
     /// Markdown content.
     pub content: String,
-    /// Search index.
-    pub search: Vec<SearchItem>,
     /// Page title extracted from Markdown.
     pub title: String,
     /// Table of contents.
     pub toc: Vec<Section>,
 }
 
+/// Markdown data returned by the Python renderer.
+#[derive(FromPyObject)]
+#[pyo3(from_item_all)]
+struct RenderedMarkdown {
+    /// Markdown metadata.
+    meta: BTreeMap<String, Dynamic>,
+    /// Markdown content.
+    content: String,
+    /// Page title extracted from Markdown.
+    title: String,
+    /// Table of contents.
+    toc: Vec<Section>,
+}
+
 // ----------------------------------------------------------------------------
 // Implementations
 // ----------------------------------------------------------------------------
@@ -89,7 +99,7 @@ impl Markdown {
             let module = py.import("zensical.markdown.render")?;
             module
                 .call_method1("render", (content, id.location(), url))?
-                .extract::<MarkdownData>()
+                .extract::<RenderedMarkdown>()
         })
         .map_err(|err| {
             Python::attach(|py| {
@@ -101,7 +111,13 @@ impl Markdown {
             })
         });
 
-        res.map(|mut data| {
+        res.map(|data| {
+            let mut data = MarkdownData {
+                meta: data.meta,
+                content: data.content,
+                title: data.title,
+                toc: data.toc,
+            };
             data.title = extract_title(&id, &data);
             Markdown { data: Arc::new(data) }
         })
@@ -173,7 +189,6 @@ fn extract_title(id: &Id, markdown: &MarkdownData) -> String {
     to_title(&file)
 }
 
-// ----------------------------------------------------------------------------
 // Tests
 // ----------------------------------------------------------------------------
 
@@ -186,7 +201,6 @@ mod tests {
             data: Arc::new(MarkdownData {
                 meta: BTreeMap::new(),
                 content: String::from("<h1>Home</h1>"),
-                search: Vec::new(),
                 title: String::from("Home"),
                 toc: Vec::new(),
             }),
@@ -208,6 +222,7 @@ mod tests {
         assert_eq!(value["content"], "<h1>Home</h1>");
         assert_eq!(value["title"], "Home");
         assert!(value.get("data").is_none());
+        assert!(value.get("search").is_none());
 
         let markdown: Markdown = serde_json::from_value(value).unwrap();
         assert_eq!(markdown.content, "<h1>Home</h1>");

crates/zensical/src/structure/nav.rs

@@ -199,6 +199,11 @@ impl Navigation {
     /// Note that only the ancestors, not the page itself is returned, which
     /// again, mirrors MkDocs' behavior, and is necessary for breadcrumbs.
     pub fn ancestors(&self, page: &Page) -> Vec<NavigationItem> {
+        self.ancestors_for_url(&page.url)
+    }
+
+    /// Returns ancestors of a page URL without requiring the complete page.
+    pub(crate) fn ancestors_for_url(&self, url: &str) -> Vec<NavigationItem> {
         // Recursively find ancestors of the page with the given URL.
         fn recurse<'a>(
             items: &'a [NavigationItem], url: &str,
@@ -227,7 +232,7 @@ impl Navigation {
         // Clone the ancestors into owned items and reverse them, so we start
         // at the ancestor closest to the page, not the root itself
         let mut items: Vec<&NavigationItem> = Vec::new();
-        let _ = recurse(&self.items, &page.url, &mut items);
+        let _ = recurse(&self.items, url, &mut items);
         items.into_iter().rev().cloned().collect()
     }
 

crates/zensical/src/structure/page.rs

@@ -256,7 +256,6 @@ impl PartialEq for PageData {
             && self.path == other.path
             && self.content == other.content
             && self.toc == other.toc
-            && self.search == other.search
     }
 }
 

crates/zensical/src/structure/search.rs

@@ -1,151 +0,0 @@
-// Copyright (c) 2025-2026 Zensical and contributors
-
-// SPDX-License-Identifier: MIT
-// All contributions are certified under the DCO
-
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to
-// deal in the Software without restriction, including without limitation the
-// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-// sell copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-// IN THE SOFTWARE.
-
-// ----------------------------------------------------------------------------
-
-//! Search index.
-
-use pyo3::FromPyObject;
-use serde::Serialize;
-use zrx::id::Id;
-use zrx::scheduler::Value;
-use zrx::stream::Key;
-
-use crate::config::plugins::SearchPluginConfig;
-
-use super::nav::{file_sort_key, Navigation};
-use super::page::Page;
-
-mod item;
-
-pub use item::SearchItem;
-
-// ----------------------------------------------------------------------------
-// Structs
-// ----------------------------------------------------------------------------
-
-/// Search configuration.
-#[derive(Clone, Debug, PartialEq, Eq, FromPyObject, Serialize)]
-pub struct SearchConfig {
-    /// Languages for tokenizer.
-    pub lang: Vec<String>,
-    /// Separator for tokenizer.
-    pub separator: String,
-}
-
-/// Search index.
-///
-/// Later, when the module system is available, we'll move search into a module
-/// of its own, but for now, we'll just keep it here for simplicity.
-#[derive(Clone, Debug, PartialEq, Eq, FromPyObject, Serialize)]
-pub struct SearchIndex {
-    /// Search configuration.
-    pub config: SearchConfig,
-    /// Search items.
-    pub items: Vec<SearchItem>,
-}
-
-// ----------------------------------------------------------------------------
-// Implementations
-// ----------------------------------------------------------------------------
-
-impl SearchConfig {
-    /// Creates search configuration for the configured theme language.
-    fn new(config: SearchPluginConfig, language: &str) -> Self {
-        Self {
-            lang: vec![language.to_string()],
-            separator: config.separator,
-        }
-    }
-}
-
-// ----------------------------------------------------------------------------
-
-impl SearchIndex {
-    /// Creates a search index from pages.
-    #[allow(clippy::assigning_clones)]
-    pub fn new(
-        pages: Vec<(Key<Id>, Page)>, nav: &Navigation,
-        config: SearchPluginConfig, language: &str,
-    ) -> Self {
-        let mut items: Vec<SearchItem> = Vec::new();
-
-        // Convert chunk into a vector for easier processing, and sort pages by
-        // the exact same method that MkDocs uses
-        let mut pages = Vec::from_iter(pages);
-        pages.sort_by_key(|(id, _)| file_sort_key(&id[0]));
-
-        // Assemble search index, combining all items from all pages into a
-        // single, flat list, adjusting the location to include the page URL
-        for (_id, page) in pages {
-            let iter = nav.ancestors(&page).into_iter().rev();
-            let mut path = iter
-                .filter_map(|item| {
-                    item.display_title().map(ToString::to_string)
-                })
-                .collect::<Vec<_>>();
-
-            // Add page title to path if not already present - this might be
-            // the true in case of index pages
-            if path.last() != Some(&page.title) {
-                path.push(page.title.clone());
-            }
-
-            // Extract page tags, if any
-            let tags: Vec<String> =
-                page.tags().into_iter().map(|tag| tag.name).collect();
-
-            // For each page, adjust the location of each item and add it to
-            // the overall list
-            for mut item in page.search.iter().cloned() {
-                let location = match item.location {
-                    Some(id) => format!("{}#{}", page.url, id),
-                    _ => page.url.clone(),
-                };
-
-                // Fall back to page title, if item title is empty
-                if item.title.is_empty() {
-                    item.title = page.title.clone();
-                }
-
-                // Update location and path and add item
-                item.location = Some(location);
-                item.path = path.clone();
-                item.tags = tags.clone();
-                items.push(item);
-            }
-        }
-
-        // Return search
-        Self {
-            config: SearchConfig::new(config, language),
-            items,
-        }
-    }
-}
-
-// ----------------------------------------------------------------------------
-// Trait implementations
-// ----------------------------------------------------------------------------
-
-impl Value for SearchIndex {}

crates/zensical/src/workflow.rs

@@ -29,7 +29,6 @@ use pyo3::types::PyAnyMethods;
 use pyo3::Python;
 use regex::Regex;
 use std::hash::{DefaultHasher, Hash, Hasher};
-use std::io::{BufWriter, Write};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::sync::{Arc, LazyLock, OnceLock};
@@ -42,11 +41,11 @@ use zrx::stream::{
     concurrent, Key, Signal, Stream, StreamTupleExt, Value, Workflow,
 };
 
+use super::compat::mkdocs::search;
 use super::config::Config;
 use super::structure::markdown::Markdown;
 use super::structure::nav::Navigation;
 use super::structure::page::Page;
-use super::structure::search::SearchIndex;
 use super::template::Template;
 use super::watcher::Source;
 
@@ -120,7 +119,7 @@ impl Main {
         let page = generate_page(&self.config, &markdown);
         let site = generate_site(&self.config, &page);
         let nav = generate_nav(&site);
-        generate_search_index(&self.config, &site);
+        search::attach(&self.config, &page, &nav);
         generate_object_inventory(&self.config, &nav);
         let _ = render_templates(&self.config, &files, &nav);
         let unresolved = render_pages(&self.config, &site);
@@ -360,7 +359,6 @@ pub fn process_markdown(
             } else {
                 url
             };
-
             // Don't cache page if it inserts (pymdownx) snippets.
             // This is a hack while waiting for CommonMark (AST) and components,
             // as well as topic-based authoring functionality.
@@ -443,44 +441,6 @@ pub fn generate_object_inventory(
     });
 }
 
-/// Generate search index
-fn generate_search_index(config: &Config, site: &Signal<Id, Site>) {
-    let config = config.clone();
-    let _ = site.map(move |site: &Site| {
-        // Derive search only in this terminal branch, so its complete item
-        // relation is released immediately after the files are written.
-        let search = SearchIndex::new(
-            site.pages.as_ref().clone(),
-            &site.nav,
-            config.project.plugins.search.config.clone(),
-            &config.project.theme.language,
-        );
-        let site_dir = config.get_site_dir();
-
-        // Stream the search index directly to disk without retaining an
-        // additional JSON string alongside the structured representation.
-        let path = site_dir.join("search.json");
-        fs::create_dir_all(path.parent().expect("invariant"))?;
-        let mut writer = BufWriter::new(fs::File::create(path)?);
-        serde_json::to_writer(&mut writer, &search)?;
-        writer.flush()?;
-
-        // If offline plugin is enabled, create search.js as well
-        if config.project.plugins.offline.config.enabled {
-            let path = site_dir.join("search.js");
-            fs::create_dir_all(path.parent().expect("invariant"))?;
-            let mut writer = BufWriter::new(fs::File::create(path)?);
-            writer.write_all(b"var __index = ")?;
-            serde_json::to_writer(&mut writer, &search)?;
-            writer.write_all(b";")?;
-            writer.flush()?;
-        }
-
-        // All files were written successfully
-        Ok::<_, anyhow::Error>(())
-    });
-}
-
 /// Render static and extra templates.
 pub fn render_templates(
     config: &Config, files: &Stream<Id, Source>, nav: &Signal<Id, Navigation>,

python/tests/integration/test_search.py

@@ -0,0 +1,198 @@
+# Copyright (c) 2025-2026 Zensical and contributors
+
+# SPDX-License-Identifier: MIT
+# All contributions are certified under the DCO
+
+"""Integration tests for MkDocs-compatible search artifacts."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+import zensical
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+_BUILD_OPTIONS: dict[str, Any] = {"clean": False, "strict": False}
+
+
+def _write_project(root: Path, *, plugins: str) -> Path:
+    """Create a representative search project."""
+    docs = root / "docs"
+    (docs / "guide").mkdir(parents=True)
+    (docs / "index.md").write_text(
+        """\
+---
+tags:
+  - alpha
+  - beta
+---
+
+# Landing
+
+Intro with <small>fine print</small>.
+
+## Overview
+
+Overview body.
+""",
+        encoding="utf-8",
+    )
+    (docs / "guide" / "topic.md").write_text(
+        """\
+---
+title: Metadata title
+tags:
+  - guide
+---
+
+Preface before a heading.
+
+## Details
+
+Detailed body.
+""",
+        encoding="utf-8",
+    )
+    config = root / "mkdocs.yml"
+    config.write_text(
+        f"""\
+site_name: Search
+nav:
+  - Home: index.md
+  - Guides:
+      - Topic: guide/topic.md
+plugins:
+{plugins}
+""",
+        encoding="utf-8",
+    )
+    return config
+
+
+def _read_index(root: Path) -> dict[str, Any]:
+    """Read the generated search index."""
+    return json.loads((root / "site" / "search.json").read_text())
+
+
+def test_search_artifacts_match_mkdocs_contract(tmp_path: Path) -> None:
+    """Search output preserves ordering, page facts, and offline framing."""
+    config = _write_project(
+        tmp_path,
+        plugins='  - search:\n      separator: "[\\\\s-]+"\n  - offline',
+    )
+    zensical.build(str(config), _BUILD_OPTIONS)
+
+    expected = {
+        "config": {"lang": ["en"], "separator": "[\\s-]+"},
+        "items": [
+            {
+                "location": "index.html",
+                "level": 1,
+                "title": "Landing",
+                "text": "<p>Intro with <small>fine print</small>.</p>",
+                "path": ["Landing"],
+                "tags": ["alpha", "beta"],
+            },
+            {
+                "location": "index.html#overview",
+                "level": 2,
+                "title": "Overview",
+                "text": "<p>Overview body.</p>",
+                "path": ["Landing"],
+                "tags": ["alpha", "beta"],
+            },
+            {
+                "location": "guide/topic.html",
+                "level": 1,
+                "title": "Metadata title",
+                "text": "<p>Preface before a heading.</p>",
+                "path": ["Guides", "Metadata title"],
+                "tags": ["guide"],
+            },
+            {
+                "location": "guide/topic.html#details",
+                "level": 2,
+                "title": "Details",
+                "text": "<p>Detailed body.</p>",
+                "path": ["Guides", "Metadata title"],
+                "tags": ["guide"],
+            },
+        ],
+    }
+    assert _read_index(tmp_path) == expected
+
+    compact = json.dumps(expected, separators=(",", ":"), ensure_ascii=False)
+    assert (tmp_path / "site" / "search.js").read_text() == (
+        f"var __index = {compact};"
+    )
+
+
+def test_search_exclusion_and_disabled_output(tmp_path: Path) -> None:
+    """Excluded pages contribute no items and disabled search stays valid."""
+    config = _write_project(tmp_path, plugins="  search:\n    enabled: true")
+    topic = tmp_path / "docs" / "guide" / "topic.md"
+    topic.write_text(
+        """\
+---
+search:
+  exclude: true
+---
+
+# Hidden
+
+Not indexed.
+""",
+        encoding="utf-8",
+    )
+    zensical.build(str(config), _BUILD_OPTIONS)
+    assert [item["title"] for item in _read_index(tmp_path)["items"]] == [
+        "Landing",
+        "Overview",
+    ]
+
+    all_excluded = tmp_path / "all-excluded"
+    all_excluded.mkdir()
+    config = _write_project(all_excluded, plugins="  - search")
+    for page in (all_excluded / "docs").rglob("*.md"):
+        page.write_text(
+            "---\nsearch:\n  exclude: true\n---\n\n# Hidden\n",
+            encoding="utf-8",
+        )
+    zensical.build(str(config), _BUILD_OPTIONS)
+    assert _read_index(all_excluded)["items"] == []
+
+    disabled = tmp_path / "disabled"
+    disabled.mkdir()
+    config = _write_project(
+        disabled, plugins="  search:\n    enabled: false"
+    )
+    zensical.build(str(config), _BUILD_OPTIONS)
+    assert _read_index(disabled)["items"] == []
+
+
+def test_search_rebuild_replaces_changed_and_removed_pages(
+    tmp_path: Path,
+) -> None:
+    """Successive builds do not retain stale page search facts."""
+    config = _write_project(tmp_path, plugins="  - search")
+    zensical.build(str(config), _BUILD_OPTIONS)
+
+    index = tmp_path / "docs" / "index.md"
+    index.write_text("# Changed\n\nFresh body.\n", encoding="utf-8")
+    (tmp_path / "docs" / "guide" / "topic.md").unlink()
+    zensical.build(str(config), _BUILD_OPTIONS)
+
+    assert _read_index(tmp_path)["items"] == [
+        {
+            "location": "",
+            "level": 1,
+            "title": "Changed",
+            "text": "<p>Fresh body.</p>",
+            "path": ["Changed"],
+            "tags": [],
+        }
+    ]

python/zensical/extensions/search.py

@@ -1,397 +0,0 @@
-# Copyright (c) 2025-2026 Zensical and contributors
-
-# SPDX-License-Identifier: MIT
-# All contributions are certified under the DCO
-
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to
-# deal in the Software without restriction, including without limitation the
-# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-# sell copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from html import escape
-from html.parser import HTMLParser
-from typing import TYPE_CHECKING, Any
-
-from markdown import Extension
-from markdown.postprocessors import Postprocessor
-
-if TYPE_CHECKING:
-    from markdown import Markdown
-
-
-# -----------------------------------------------------------------------------
-# Classes
-# -----------------------------------------------------------------------------
-
-
-@dataclass
-class SearchConfig:
-    """Configuration for the Search Markdown extension."""
-
-    keep: set[str] = field(default_factory=set)
-
-
-# -----------------------------------------------------------------------------
-
-
-class SearchProcessor(Postprocessor):
-    """Post processor to extract searchable content from the rendered HTML."""
-
-    name = "search"
-
-    def __init__(self, md: Markdown, config: SearchConfig) -> None:
-        super().__init__(md)
-        self.config = config
-        self.data: list[dict[str, Any]] = []
-
-    def run(self, text: str) -> str:
-        """Process the rendered HTML and extract text length."""
-        # Divide page content into sections
-        parser = Parser()
-        parser.feed(text)
-        parser.close()
-
-        # Extract data from sections that are not excluded
-        self.data = []
-        for section in parser.data:
-            if not section.is_excluded():
-                # Compute title and text
-                title = "".join(section.title).strip()
-                content = "".join(section.text).strip()
-
-                # Store data for external access
-                self.data.append(
-                    {
-                        "location": section.id,
-                        "level": section.level,
-                        "title": title,
-                        "text": content,
-                        "path": [],
-                        "tags": [],
-                    }
-                )
-
-        # Return the original HTML unchanged
-        return text
-
-
-class SearchExtension(Extension):
-    """Markdown extension for search indexing."""
-
-    name = "zensical.extensions.search"
-
-    def __init__(self, **kwargs: Any) -> None:
-        self._kwargs = kwargs
-
-    def extendMarkdown(self, md: Markdown) -> None:
-        """Register the PostProcessor with Markdown."""
-        config = SearchConfig(**self._kwargs)
-        processor = SearchProcessor(md, config)
-        md.postprocessors.register(processor, processor.name, 0)
-
-
-def makeExtension(**kwargs: Any) -> SearchExtension:
-    """Register Markdown extension."""
-    return SearchExtension(**kwargs)
-
-
-# -----------------------------------------------------------------------------
-
-
-# HTML element
-class Element:
-    """HTML element.
-
-    An element with attributes, essentially a small wrapper object for the
-    parser to access attributes in other callbacks than handle_starttag.
-    """
-
-    # Initialize HTML element
-    def __init__(
-        self, tag: str, attrs: dict[str, str | None] | None = None
-    ) -> None:
-        self.tag = tag
-        self.attrs = attrs or {}
-
-    # String representation
-    def __repr__(self):
-        return self.tag
-
-    # Support comparison (compare by tag only)
-    def __eq__(self, other: object) -> bool:
-        if isinstance(other, Element):
-            return self.tag == other.tag
-        return self.tag == other
-
-    # Support set operations
-    def __hash__(self):
-        return hash(self.tag)
-
-    # Check whether the element should be excluded
-    def is_excluded(self) -> bool:
-        return "data-search-exclude" in self.attrs
-
-
-# -----------------------------------------------------------------------------
-
-
-# HTML section
-class Section:
-    """HTML section.
-
-    A block of text with markup, preceded by a title (with markup), i.e., a
-    headline with a certain level (h1-h6). Internally used by the parser.
-    """
-
-    # Initialize HTML section
-    def __init__(self, el: Element, level: int, depth: int = 0) -> None:
-        self.el = el
-        self.depth: int | float = depth
-        self.level = level
-
-        # Initialize section data
-        self.text: list[str] = []
-        self.title: list[str] = []
-        self.id: str | None = None
-
-    # String representation
-    def __repr__(self):
-        if self.id:
-            return f"{self.el.tag}#{self.id}"
-        return self.el.tag
-
-    # Check whether the section should be excluded
-    def is_excluded(self) -> bool:
-        return self.el.is_excluded()
-
-
-# -----------------------------------------------------------------------------
-
-
-# HTML parser
-class Parser(HTMLParser):
-    """Section divider.
-
-    This parser divides the given string of HTML into a list of sections, each
-    of which are preceded by a h1-h6 level heading. A white- and blacklist of
-    tags dictates which tags should be preserved as part of the index, and
-    which should be ignored in their entirety.
-    """
-
-    # Initialize HTML parser
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        super().__init__(*args, **kwargs)
-
-        # Tags to skip
-        self.skip: set[str | Element] = {
-            "object",  # Objects
-            "script",  # Scripts
-            "style",  # Styles
-        }
-
-        # Current context and section
-        self.context: list[Element] = []
-        self.section: Section | None = None
-
-        # All parsed sections
-        self.data: list[Section] = []
-
-    # Called at the start of every HTML tag
-    def handle_starttag(
-        self, tag: str, attrs: list[tuple[str, str | None]]
-    ) -> None:
-        attrs_dict = dict(attrs)
-
-        # Ignore self-closing tags
-        el = Element(tag, attrs_dict)
-        if tag not in void:
-            self.context.append(el)
-        else:
-            return
-
-        # Handle heading
-        if tag in ([f"h{x}" for x in range(1, 7)]):
-            depth = len(self.context)
-            if "id" in attrs_dict:
-                # Ensure top-level section
-                if tag != "h1" and not self.data:
-                    self.section = Section(Element("hx"), 1, depth)
-                    self.data.append(self.section)
-
-                # Set identifier, if not first section
-                self.section = Section(el, int(tag[1:2]), depth)
-                if self.data:
-                    self.section.id = attrs_dict["id"]
-
-                # Append section to list
-                self.data.append(self.section)
-
-        # Handle preface - ensure top-level section
-        if not self.section:
-            self.section = Section(Element("hx"), 1)
-            self.data.append(self.section)
-
-        # Handle special cases to skip
-        for key, value in attrs_dict.items():
-            # Skip block if explicitly excluded from search
-            if key == "data-search-exclude":
-                self.skip.add(el)
-                return
-
-            # Skip line numbers - see https://bit.ly/3GvubZx
-            if key == "class" and value == "linenodiv":
-                self.skip.add(el)
-                return
-
-        # Render opening tag if kept
-        if not self.skip.intersection(self.context) and tag in keep:
-            # Check whether we're inside the section title
-            data = self.section.text
-            if self.section.el in self.context:
-                data = self.section.title
-
-            # Append to section title or text
-            data.append(f"<{tag}>")
-
-    # Called at the end of every HTML tag
-    def handle_endtag(self, tag: str) -> None:
-        if not self.context or self.context[-1] != tag:
-            return
-
-        # Check whether we're exiting the current context, which happens when
-        # a headline is nested in another element. In that case, we close the
-        # current section, continuing to append data to the previous section,
-        # which could also be a nested section – see https://bit.ly/3IxxIJZ
-        assert self.section is not None  # noqa: S101
-        if self.section.depth > len(self.context):
-            for section in reversed(self.data):
-                if section.depth <= len(self.context):
-                    # Set depth to infinity in order to denote that the current
-                    # section is exited and must never be considered again.
-                    self.section.depth = float("inf")
-                    self.section = section
-                    break
-
-        # Remove element from skip list
-        el = self.context.pop()
-        if el in self.skip:
-            if el.tag not in ["script", "style", "object"]:
-                self.skip.remove(el)
-            return
-
-        # Render closing tag if kept
-        if not self.skip.intersection(self.context) and tag in keep:
-            # Check whether we're inside the section title
-            data = self.section.text
-            if self.section.el in self.context:
-                data = self.section.title
-
-            # Search for corresponding opening tag
-            index = data.index(f"<{tag}>")
-            for i in range(index + 1, len(data)):
-                if not data[i].isspace():
-                    index = len(data)
-                    break
-
-            # Remove element if empty (or only whitespace)
-            if len(data) > index:
-                while len(data) > index:
-                    data.pop()
-
-            # Append to section title or text
-            else:
-                data.append(f"</{tag}>")
-
-    # Called for the text contents of each tag
-    def handle_data(self, data: str) -> None:
-        if self.skip.intersection(self.context):
-            return
-
-        # Collapse whitespace in non-pre contexts
-        if "pre" not in self.context:
-            if not data.isspace():
-                data = data.replace("\n", " ")
-            else:
-                data = " "
-
-        # Handle preface - ensure top-level section
-        if not self.section:
-            self.section = Section(Element("hx"), 1)
-            self.data.append(self.section)
-
-        # Handle section headline
-        if self.section.el in self.context:
-            permalink = False
-            for el in self.context:
-                if el.tag == "a" and el.attrs.get("class") == "headerlink":
-                    permalink = True
-
-            # Ignore permalinks
-            if not permalink:
-                self.section.title.append(escape(data, quote=False))
-
-        # Collapse adjacent whitespace
-        elif data.isspace():
-            if (
-                not self.section.text
-                or not self.section.text[-1].isspace()
-                or "pre" in self.context
-            ):
-                self.section.text.append(data)
-
-        # Handle everything else
-        else:
-            self.section.text.append(escape(data, quote=False))
-
-
-# -----------------------------------------------------------------------------
-# Data
-# -----------------------------------------------------------------------------
-
-
-# Tags to keep
-keep = {
-    "p",
-    "code",
-    "pre",
-    "li",
-    "ol",
-    "ul",
-    "small",
-    "sub",
-    "sup",
-}
-
-# Tags that are self-closing
-void = {
-    "area",
-    "base",
-    "br",
-    "col",
-    "embed",
-    "hr",
-    "img",
-    "input",
-    "link",
-    "meta",
-    "param",
-    "source",
-    "track",
-    "wbr",
-}

python/zensical/markdown/render.py

@@ -25,7 +25,7 @@ from __future__ import annotations
 
 import re
 from datetime import date, datetime
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 import yaml
 from markdown import Markdown
@@ -35,10 +35,6 @@ from zensical.config import get_config
 from zensical.extensions.autorefs import set_autorefs_page
 from zensical.extensions.context import ContextExtension, Page
 from zensical.extensions.links import LinksExtension
-from zensical.extensions.search import SearchExtension
-
-if TYPE_CHECKING:
-    from zensical.extensions.search import SearchProcessor
 
 # ----------------------------------------------------------------------------
 # Constants
@@ -110,14 +106,11 @@ def render(content: str, path: str, url: str) -> dict:
         extension_configs=config["mdx_configs"],
     )
 
-    # Note: mkdocstrings and markdown-exec do not need to propagate
-    # the links and search extensions to their inner Markdown instances:
-    # their postprocessors run last and can see inner layer contents.
-    # More importantly, inner layers *must not* run the links and search
-    # extensions: the inner links treeprocessor would transform links once,
-    # and the outer links postprocessor would transform them again.
-    # The search postprocessor would run twice for generated content,
-    # incurring a performance cost.
+    # Note: mkdocstrings and markdown-exec do not need to propagate the links
+    # extension to their inner Markdown instances. Its postprocessor runs last
+    # and can see inner layer contents. More importantly, inner layers *must
+    # not* run the extension: the inner treeprocessor would transform links
+    # once, and the outer postprocessor would transform them again.
 
     # Register links extension, which is equivalent to MkDocs' path resolution
     # Markdown extension. This is a bandaid, until we move this to Rust
@@ -126,10 +119,6 @@ def render(content: str, path: str, url: str) -> dict:
     )
     links.extendMarkdown(md)
 
-    # Register search extension, which extracts text for search indexing
-    search_extension = SearchExtension()
-    search_extension.extendMarkdown(md)
-
     # Inform markdown-exec that it runs through Zensical.
     try:
         import markdown_exec  # noqa: PLC0415  # ty:ignore[unresolved-import]
@@ -141,11 +130,6 @@ def render(content: str, path: str, url: str) -> dict:
     # Convert content to HTML
     content = md.convert(content)
 
-    # Obtain search index data, unless page is excluded
-    search_processor: SearchProcessor = md.postprocessors["search"]
-    if meta.get("search", {}).get("exclude", False):
-        search_processor.data = []
-
     # Sanitize metadata before passing it to Rust
     meta = {k: _sanitize(v) for k, v in meta.items()}
 
@@ -154,7 +138,6 @@ def render(content: str, path: str, url: str) -> dict:
         "meta": meta,
         "title": "",
         "content": content,
-        "search": search_processor.data,
         "toc": [_convert_toc(item) for item in getattr(md, "toc_tokens", [])],
     }