From e6c56cef609983befc43026127d11f5ec4df4adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Tue, 18 Aug 2026 15:08:20 +0000 Subject: [PATCH] refactor: report invalid autorefs under `invalid_links` validation setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Timothée Mazzucotelli --- crates/zensical/src/python.rs | 2 +- crates/zensical/src/python/collector.rs | 2 +- .../src/python/collector/reference.rs | 38 ++ crates/zensical/src/python/issues.rs | 118 +++++-- crates/zensical/src/structure/markdown.rs | 2 +- .../src/structure/markdown/autorefs.rs | 97 ++++-- crates/zensical/src/structure/nav.rs | 113 ++++-- crates/zensical/src/structure/page.rs | 12 +- crates/zensical/src/workflow.rs | 325 ++++++++---------- crates/zensical/src/workflow/aggregate.rs | 254 ++++++++++++++ python/tests/integration/test_validation.py | 324 ++++++++++++++++- python/zensical/extensions/autorefs.py | 26 +- 12 files changed, 1038 insertions(+), 275 deletions(-) create mode 100644 crates/zensical/src/workflow/aggregate.rs diff --git a/crates/zensical/src/python.rs b/crates/zensical/src/python.rs index e6d0bb6..8aba202 100644 --- a/crates/zensical/src/python.rs +++ b/crates/zensical/src/python.rs @@ -29,6 +29,6 @@ pub mod collector; mod issues; mod span; -pub use collector::{Anchors, References}; +pub use collector::{Anchors, References, SharedReferences}; pub use issues::Issues; pub use span::Span; diff --git a/crates/zensical/src/python/collector.rs b/crates/zensical/src/python/collector.rs index 215cf53..894fe7f 100644 --- a/crates/zensical/src/python/collector.rs +++ b/crates/zensical/src/python/collector.rs @@ -30,4 +30,4 @@ pub mod reference; pub mod snippet; pub use anchor::Anchors; -pub use reference::References; +pub use reference::{References, SharedReferences}; diff --git a/crates/zensical/src/python/collector/reference.rs b/crates/zensical/src/python/collector/reference.rs index 60acb6e..3936c1d 100644 --- a/crates/zensical/src/python/collector/reference.rs +++ b/crates/zensical/src/python/collector/reference.rs @@ -27,8 +27,10 @@ use pyo3::prelude::*; use std::fmt::{self, Debug}; +use std::ops::Deref; use std::slice::Iter; use std::str::FromStr; +use std::sync::Arc; use zrx::stream::Value; mod footnote; @@ -74,6 +76,16 @@ pub struct References { inner: Vec, } +// ---------------------------------------------------------------------------- + +/// Shared reference set. +/// +/// References embed the Markdown they were extracted from, so they're shared +/// behind an [`Arc`] to keep a single copy per page alive while joins, site +/// snapshots and validation state hold on to them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SharedReferences(Arc); + // ---------------------------------------------------------------------------- // Implementations // ---------------------------------------------------------------------------- @@ -100,6 +112,32 @@ impl Value for References {} // ---------------------------------------------------------------------------- +impl Value for SharedReferences {} + +// ---------------------------------------------------------------------------- + +impl From for SharedReferences { + /// Creates a shared reference set from a reference set. + #[inline] + fn from(references: References) -> Self { + Self(Arc::new(references)) + } +} + +// ---------------------------------------------------------------------------- + +impl Deref for SharedReferences { + type Target = References; + + /// Returns a reference to the underlying reference set. + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +// ---------------------------------------------------------------------------- + impl FromStr for References { type Err = PyErr; diff --git a/crates/zensical/src/python/issues.rs b/crates/zensical/src/python/issues.rs index a1a3389..3075c23 100644 --- a/crates/zensical/src/python/issues.rs +++ b/crates/zensical/src/python/issues.rs @@ -35,12 +35,12 @@ use zrx::id::Id; use zrx::scheduler::{Key, Value}; use crate::config::validation::Validation; -use crate::structure::markdown::AutorefResolutions; +use crate::structure::markdown::UnresolvedAutorefs; use super::collector::reference::{ LinkReference, LinkReferenceKind, Reference, }; -use super::collector::{Anchors, References}; +use super::collector::{Anchors, SharedReferences}; use super::span::Span; mod error; @@ -54,6 +54,15 @@ pub use error::{Error, Result}; /// Issue. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Issue { + /// Autoref with no matching identifier. + /// + /// The span is optional, since autorefs might be introduced by templates + /// or transformations, in which case they cannot be located in the source. + UnresolvedAutoref { + path: PathBuf, + span: Option, + id: String, + }, /// Link or image reference with no matching definition. UnresolvedReference { path: PathBuf, @@ -124,7 +133,8 @@ impl Issue { /// Returns the path of the issue. pub fn path(&self) -> &Path { match self { - Issue::UnresolvedReference { path, .. } + Issue::UnresolvedAutoref { path, .. } + | Issue::UnresolvedReference { path, .. } | Issue::UnresolvedFootnote { path, .. } | Issue::UnusedDefinition { path, .. } | Issue::UnusedFootnote { path, .. } @@ -135,9 +145,10 @@ impl Issue { } } - /// Returns the span of the issue. - pub fn span(&self) -> &Span { + /// Returns the span of the issue, if it can be located in the source. + pub fn span(&self) -> Option<&Span> { match self { + Issue::UnresolvedAutoref { span, .. } => span.as_ref(), Issue::UnresolvedReference { span, .. } | Issue::UnresolvedFootnote { span, .. } | Issue::UnusedDefinition { span, .. } @@ -145,7 +156,7 @@ impl Issue { | Issue::ShadowedDefinition { span, .. } | Issue::ShadowedFootnote { span, .. } | Issue::InvalidLink { span, .. } - | Issue::InvalidLinkAnchor { span, .. } => span, + | Issue::InvalidLinkAnchor { span, .. } => Some(span), } } } @@ -160,7 +171,7 @@ impl Issues { pub fn new(iter: T) -> Self where T: IntoIterator< - Item = (Key, (References, Anchors, AutorefResolutions)), + Item = (Key, (SharedReferences, Anchors, UnresolvedAutorefs)), >, { let mut issues = Vec::new(); @@ -183,7 +194,7 @@ impl Issues { // Collect all links for each page for cross-page checking later let mut mappings = Vec::new(); #[allow(clippy::case_sensitive_file_extension_comparisons)] - for reference in &references { + for reference in references.iter() { if let Reference::Link(link) = reference { let href = &references.markdown()[link.href.start..link.href.end]; @@ -211,7 +222,7 @@ impl Issues { // 1st pass - collect link and footnote definitions let markdown = references.markdown(); - for reference in &references { + for reference in references.iter() { match reference { Reference::LinkDefinition(link) => { let id = &markdown[link.id.start..link.id.end]; @@ -242,18 +253,12 @@ impl Issues { let mut used_note_defs = HashSet::default(); // 2nd pass - check link and footnote references - for reference in &references { + for reference in references.iter() { match reference { Reference::LinkReference(link) => { let id = &markdown[link.id.start..link.id.end]; if link_defs.contains_key(&to_id(id)) { used_link_defs.insert(to_id(id)); - } else if autoref_id(markdown, link) - .is_some_and(|id| autorefs.is_resolved(id)) - { - // Autorefs resolved this reference while rendering - // the page, so it is valid despite having no link - // definition in the Markdown source. } else { issues.push(Issue::UnresolvedReference { path: path.clone().into(), @@ -301,6 +306,40 @@ impl Issues { }); } } + + // Report autorefs that failed to resolve while rendering the + // page. Index the identifiers autorefs derives from the source + // link references, so issues point at the exact location. When + // an identifier cannot be located, e.g. because the autoref was + // introduced by a template, fall back to a page-level issue. + let mut spans = HashMap::<_, Vec>::default(); + for reference in references.iter() { + if let Reference::LinkReference(link) = reference { + if let Some(id) = autoref_id(markdown, link) { + spans + .entry(id) + .or_default() + .push((link.id.start..link.id.end).into()); + } + } + } + for id in autorefs.iter() { + if let Some(spans) = spans.get(id.as_str()) { + for span in spans { + issues.push(Issue::UnresolvedAutoref { + path: path.clone().into(), + span: Some(*span), + id: id.clone(), + }); + } + } else { + issues.push(Issue::UnresolvedAutoref { + path: path.clone().into(), + span: None, + id: id.clone(), + }); + } + } } // Check links across pages for issues @@ -400,9 +439,11 @@ impl Issues { // Sort issues by path, then by span issues.sort_by(|a, b| { - a.path() - .cmp(b.path()) - .then_with(|| a.span().start.cmp(&b.span().start)) + a.path().cmp(b.path()).then_with(|| { + let a = a.span().map_or(0, |span| span.start); + let b = b.span().map_or(0, |span| span.start); + a.cmp(&b) + }) }); // Return issues @@ -418,6 +459,12 @@ impl Issues { // Determine the path and kind of report let path = issue.path().to_string_lossy(); let kind = match issue { + Issue::UnresolvedAutoref { .. } => { + if !validation.invalid_links { + continue; + } + ReportKind::Warning + } Issue::UnresolvedReference { .. } => { if !validation.unresolved_references { continue; @@ -470,6 +517,9 @@ impl Issues { // Determine the label message and color let (message, color) = match issue { + Issue::UnresolvedAutoref { .. } => { + ("unresolved autoref", Color::Yellow) + } Issue::UnresolvedReference { .. } => { ("unresolved link reference", Color::Yellow) } @@ -496,17 +546,29 @@ impl Issues { } }; + // Issues without a span cannot be located in the source, so we + // print a plain report that only mentions the page they occur on + let Some(span) = issue.span() else { + let Issue::UnresolvedAutoref { id, .. } = issue else { + unreachable!("only autorefs may lack spans"); + }; + Report::build(kind, (path.as_ref(), 0..0)) + .with_message(format!("{message} `{id}` in {path}")) + .finish() + .eprint((path.as_ref(), Source::from("")))?; + count += 1; + continue; + }; + // Create report - let builder = Report::build( - kind, - (path.as_ref(), Range::from(*issue.span())), - ) - .with_message(message) - .with_label( - Label::new((path.as_ref(), Range::from(*issue.span()))) + let builder = + Report::build(kind, (path.as_ref(), Range::from(*span))) .with_message(message) - .with_color(color), - ); + .with_label( + Label::new((path.as_ref(), Range::from(*span))) + .with_message(message) + .with_color(color), + ); // Obtain Markdown source let source = self diff --git a/crates/zensical/src/structure/markdown.rs b/crates/zensical/src/structure/markdown.rs index c791397..60b5ea5 100644 --- a/crates/zensical/src/structure/markdown.rs +++ b/crates/zensical/src/structure/markdown.rs @@ -41,7 +41,7 @@ use crate::structure::toc::Section; mod autorefs; -pub use autorefs::{AutorefResolutions, Autorefs}; +pub use autorefs::{Autorefs, UnresolvedAutorefs}; // ---------------------------------------------------------------------------- // Constants diff --git a/crates/zensical/src/structure/markdown/autorefs.rs b/crates/zensical/src/structure/markdown/autorefs.rs index 6ac3099..cedf302 100644 --- a/crates/zensical/src/structure/markdown/autorefs.rs +++ b/crates/zensical/src/structure/markdown/autorefs.rs @@ -202,11 +202,11 @@ fn is_relative_url(url: &str) -> bool { // Structs // ---------------------------------------------------------------------------- -/// Resolution results for autoreferences in a single page. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct AutorefResolutions { - /// Identifiers that were resolved successfully. - resolved: HashSet, +/// Autoref identifiers that could not be resolved in a single page. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct UnresolvedAutorefs { + /// Identifiers in order of first appearance. + identifiers: Vec, } // ---------------------------------------------------------------------------- @@ -259,6 +259,9 @@ pub struct Autorefs { pub inventory: HashMap, // Titles. pub titles: HashMap, + // Pages reprocessed since autorefs data was last collected. + #[serde(skip)] + pub updated_pages: Vec, } // ---------------------------------------------------------------------------- @@ -285,6 +288,24 @@ impl Autorefs { self.titles.extend(other.titles); } + /// Removes registrations owned by any of the given pages. + pub(crate) fn remove_pages(&mut self, pages: &HashSet) { + self.retain_page_urls(|page| !pages.contains(page)); + } + + /// Retains registrations owned by one of the given pages. + pub(crate) fn retain_pages(&mut self, pages: &HashSet) { + self.retain_page_urls(|page| pages.contains(page)); + } + + /// Retains registrations whose page URL matches the predicate. + fn retain_page_urls(&mut self, predicate: impl Fn(&str) -> bool) { + retain_url_map(&mut self.primary, &predicate); + retain_url_map(&mut self.secondary, &predicate); + self.titles + .retain(|url, _| predicate(page_url_from_autoref(url))); + } + /// Parses HTML attributes string into a HashMap. /// /// @todo Document that this is not the most resilient HTML parser @@ -458,13 +479,13 @@ impl Autorefs { )) } - /// Replaces autorefs and collects their resolution results. + /// Replaces autorefs and collects unresolved identifiers. #[allow(clippy::single_match_else)] pub fn replace_in( - &self, content: String, from_url: &str, - ) -> (String, AutorefResolutions) { - let mut resolutions = AutorefResolutions::default(); - let output = AUTOREF_RE.replace_all(&content, |captures: &Captures| { + &self, content: &str, from_url: &str, + ) -> (String, UnresolvedAutorefs) { + let mut unresolved = UnresolvedAutorefs::default(); + let output = AUTOREF_RE.replace_all(content, |captures: &Captures| { let attrs_str = captures.name("attrs").map_or("", |m| m.as_str()); let title = @@ -485,8 +506,6 @@ impl Autorefs { match self.get_url_and_title_from_ids(&identifiers, from_url) { Ok((url, original_title)) => { - resolutions.resolved.insert(identifier.clone()); - // Check if URL is external (not relative) let external = !is_relative_url(&url); @@ -555,7 +574,7 @@ impl Autorefs { if optional { format!("{title}") } else { - // @todo: unmapped.append((identifier, attrs.context)) + unresolved.insert(&identifier); if title == identifier { format!("[{identifier}][]") } else if title == format!("{identifier}") @@ -570,7 +589,7 @@ impl Autorefs { } }); - (output.to_string(), resolutions) + (output.to_string(), unresolved) } } @@ -578,17 +597,43 @@ impl Autorefs { // Trait implementations // ---------------------------------------------------------------------------- -impl Value for AutorefResolutions {} +impl Value for UnresolvedAutorefs {} // ---------------------------------------------------------------------------- // Implementations // ---------------------------------------------------------------------------- -impl AutorefResolutions { - /// Returns whether an identifier was resolved. - pub fn is_resolved(&self, identifier: &str) -> bool { - self.resolved.contains(identifier) +impl UnresolvedAutorefs { + /// Records an identifier that failed to resolve. + fn insert(&mut self, identifier: &str) { + if !self.identifiers.iter().any(|id| id == identifier) { + self.identifiers.push(identifier.to_string()); + } } + + /// Returns an iterator over the identifiers. + pub fn iter(&self) -> std::slice::Iter<'_, String> { + self.identifiers.iter() + } +} + +// ---------------------------------------------------------------------------- +// Functions +// ---------------------------------------------------------------------------- + +/// Retain URL-map entries based on their owning page URL. +fn retain_url_map( + map: &mut HashMap>, predicate: &impl Fn(&str) -> bool, +) { + map.retain(|_, urls| { + urls.retain(|url| predicate(page_url_from_autoref(url))); + !urls.is_empty() + }); +} + +/// Return the page URL portion of an autoref URL. +fn page_url_from_autoref(url: &str) -> &str { + url.split_once('#').map_or(url, |(page, _)| page) } #[cfg(test)] @@ -660,24 +705,26 @@ mod tests { } #[test] - fn autoref_resolutions_are_collected_while_replacing() { + fn unresolved_autorefs_are_collected_while_replacing() { let mut autorefs = Autorefs::new(); autorefs .primary .insert("known".to_string(), vec!["reference/#known".to_string()]); - let (output, resolutions) = autorefs.replace_in( + let (output, unresolved) = autorefs.replace_in( concat!( "Known", "Missing", - ) - .to_string(), + "Missing", + "Skipped", + ), "guide/", ); - assert!(resolutions.is_resolved("known")); - assert!(!resolutions.is_resolved("missing")); + let unresolved = unresolved.iter().collect::>(); + assert_eq!(unresolved, ["missing"]); assert!(output.contains("href=\"../reference/#known\"")); assert!(output.contains("[Missing][missing]")); + assert!(output.contains("Skipped")); } } diff --git a/crates/zensical/src/structure/nav.rs b/crates/zensical/src/structure/nav.rs index 8e8dda5..04d01c5 100644 --- a/crates/zensical/src/structure/nav.rs +++ b/crates/zensical/src/structure/nav.rs @@ -28,9 +28,9 @@ use std::fs; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; -use ahash::HashMap; +use ahash::{HashMap, HashSet}; use pyo3::types::{PyAny, PyAnyMethods}; use pyo3::{Bound, FromPyObject, PyResult, Python}; use serde::Serialize; @@ -47,6 +47,13 @@ mod iter; pub use item::NavigationItem; use iter::Iter; +// ---------------------------------------------------------------------------- +// Constants +// ---------------------------------------------------------------------------- + +/// Lock serializing collection and caching of global autorefs data. +static AUTOREFS_CACHE_LOCK: Mutex<()> = Mutex::new(()); + // ---------------------------------------------------------------------------- // Structs // ---------------------------------------------------------------------------- @@ -64,11 +71,15 @@ pub struct Navigation { pub items: Arc>, /// Homepage, if defined. pub homepage: Option, - /// Autorefs (mkdocstrings). + /// Autorefs (mkdocstrings), kept internal to the rendering pipeline. #[pyo3(from_py_with = extract_shared_autorefs)] + #[serde(skip)] pub autorefs: Arc, - /// Precomputed hash. + /// Precomputed navigation-structure hash. pub hash: u64, + /// Site snapshot generation this navigation was created from. + #[serde(skip)] + pub generation: u64, } // ---------------------------------------------------------------------------- @@ -81,8 +92,13 @@ impl Navigation { cache_dir: PathBuf, mut items: Vec, pages: Vec<(Key, Page)>, ) -> Self { + let page_urls = pages + .iter() + .map(|(_, page)| page.url.clone()) + .collect::>(); + // Fetch and cache autorefs once, used by both branches below - let autorefs = get_autorefs_cached(&cache_dir); + let autorefs = get_autorefs_cached(&cache_dir, &page_urls); if items.is_empty() { let mut nav = Self::from(pages); @@ -163,11 +179,7 @@ impl Navigation { } // Precompute hash - let hash = { - let mut hasher = DefaultHasher::default(); - items.hash(&mut hasher); - hasher.finish() - }; + let hash = navigation_hash(&items); // Return navigation Self { @@ -175,6 +187,7 @@ impl Navigation { homepage, autorefs: Arc::new(autorefs), hash, + generation: 0, } } @@ -212,6 +225,7 @@ impl Navigation { homepage: self.homepage, autorefs: self.autorefs, hash: self.hash, + generation: self.generation, } } @@ -368,16 +382,14 @@ impl From, Page)>> for Navigation { }); } - // Precompute hash - let hash = { - let mut hasher = DefaultHasher::default(); - items.hash(&mut hasher); - hasher.finish() - }; - - // Fetch without caching — Navigation::new() will override this with + // Start from empty autorefs — Navigation::new() overrides this with // the cached+merged result when called through the normal build path. - let autorefs = get_autorefs(); + // Fetching from Python here would consume the updated-pages tracking + // outside of the cache lock, silently losing update flags. + let autorefs = Autorefs::new(); + + // Precompute hash + let hash = navigation_hash(&items); // Determine homepage and return navigation Self { @@ -385,25 +397,13 @@ impl From, Page)>> for Navigation { autorefs: Arc::new(autorefs), items: Arc::new(items), hash, + generation: 0, } } } // ---------------------------------------------------------------------------- -impl Hash for Navigation { - /// Hashes the navigation. - #[inline] - fn hash(&self, state: &mut H) - where - H: Hasher, - { - state.write_u64(self.hash); - } -} - -// ---------------------------------------------------------------------------- - impl<'a> IntoIterator for &'a Navigation { type Item = &'a NavigationItem; type IntoIter = Iter<'a>; @@ -439,6 +439,13 @@ fn is_index(component: &str) -> bool { component == "index.md" || component == "README.md" } +/// Hash the navigation structure that can affect page templates. +fn navigation_hash(items: &[NavigationItem]) -> u64 { + let mut hasher = DefaultHasher::default(); + items.hash(&mut hasher); + hasher.finish() +} + /// Computes a page title from a file name, replicating MkDocs' behavior. pub(crate) fn to_title(component: &str) -> String { let title = component.trim_end_matches(".md").replace(['-', '_'], " "); @@ -465,7 +472,10 @@ fn extract_shared_items( value.extract::>().map(Arc::new) } -fn get_autorefs_cached(cache_dir: &Path) -> Autorefs { +fn get_autorefs_cached( + cache_dir: &Path, page_urls: &HashSet, +) -> Autorefs { + let _guard = AUTOREFS_CACHE_LOCK.lock().expect("invariant"); let path = cache_dir.join("autorefs.json"); // Load previously cached autorefs, falling back to empty if unavailable @@ -474,10 +484,18 @@ fn get_autorefs_cached(cache_dir: &Path) -> Autorefs { .and_then(|data| serde_json::from_slice::(&data).ok()) .unwrap_or_default(); - // Fetch fresh data from the Python process and merge, giving it precedence - // over cached entries — pages re-processed in this build are authoritative - // while identifiers from unchanged (cached) pages are preserved from cache - autorefs.merge(get_autorefs()); + // Fetch fresh data from the Python process. Remove registrations for pages + // that were reprocessed before merging, so removed anchors don't survive + // in the cache. Fresh data takes precedence, while identifiers from pages + // that stayed cached are preserved. + let fresh = get_autorefs(); + let updated_pages = + fresh.updated_pages.iter().cloned().collect::>(); + autorefs.remove_pages(&updated_pages); + autorefs.merge(fresh); + + // Drop registrations for pages that no longer exist. + autorefs.retain_pages(page_urls); // Write merged autorefs back to cache if let Ok(data) = serde_json::to_string_pretty(&autorefs) { @@ -515,6 +533,7 @@ mod tests { homepage: None, autorefs: Arc::new(Autorefs::new()), hash: 0, + generation: 0, }; let clone = nav.clone(); @@ -523,6 +542,26 @@ mod tests { assert!(Arc::ptr_eq(&nav.items, &clone.items)); } + #[test] + fn serialization_omits_internal_autorefs_state() { + let mut autorefs = Autorefs::new(); + autorefs + .primary + .insert("item".to_string(), vec!["reference/#item".to_string()]); + let nav = Navigation { + items: Arc::new(Vec::new()), + homepage: None, + hash: navigation_hash(&[]), + autorefs: Arc::new(autorefs), + generation: 0, + }; + + let value = serde_json::to_value(nav).expect("invariant"); + assert!(value.get("autorefs").is_none()); + assert!(value.get("generation").is_none()); + assert!(value.get("hash").is_some()); + } + /// https://github.com/zensical/zensical/issues/66 #[test] fn test_to_title() { diff --git a/crates/zensical/src/structure/page.rs b/crates/zensical/src/structure/page.rs index e575aab..2cf0ecf 100644 --- a/crates/zensical/src/structure/page.rs +++ b/crates/zensical/src/structure/page.rs @@ -39,7 +39,7 @@ use crate::config::Config; use crate::template::{Output, Template, GENERATOR}; use super::dynamic::Dynamic; -use super::markdown::{AutorefResolutions, Markdown}; +use super::markdown::Markdown; use super::nav::{Navigation, NavigationItem}; use super::search::SearchItem; use super::tag::Tag; @@ -204,14 +204,14 @@ impl Page { } } - /// Renders the page. + /// Renders the page template, leaving autorefs unresolved. #[cfg_attr( feature = "tracing", tracing::instrument(skip_all, fields(url = %self.url)) )] - pub fn render( + pub fn render_template( &mut self, config: &Config, nav: Navigation, - ) -> Result<(Output, AutorefResolutions), Error> { + ) -> Result { let name = self.meta.get("template").map(ToString::to_string); let template = Template::new( name.unwrap_or(String::from("main.html")), @@ -237,9 +237,7 @@ impl Page { page => self, })?; - // Replace autorefs and retain their resolution results for validation - let (output, autorefs) = nav.autorefs.replace_in(output, &self.url); - Ok((Output::from(output), autorefs)) + Ok(Output::from(output)) } /// Returns the tags of the page. diff --git a/crates/zensical/src/workflow.rs b/crates/zensical/src/workflow.rs index 85f20c8..301f83a 100644 --- a/crates/zensical/src/workflow.rs +++ b/crates/zensical/src/workflow.rs @@ -25,14 +25,13 @@ //! Workflow definitions -use ahash::{HashMap, HashSet}; +use ahash::HashMap; use pyo3::types::PyAnyMethods; use pyo3::Python; use regex::Regex; use std::hash::{DefaultHasher, Hash, Hasher}; use std::path::{Path, PathBuf}; use std::str::FromStr; -use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, LazyLock, Mutex}; use std::{fs, io}; use zrx::id::matcher::Matcher; @@ -42,16 +41,18 @@ use zrx::scheduler::Key; use zrx::stream::{Barrier, Stream, Workflow}; use super::config::Config; -use super::python::{Anchors, Issues, References}; -use super::structure::markdown::{AutorefResolutions, Markdown}; +use super::python::{Anchors, Issues, References, SharedReferences}; +use super::structure::markdown::{Markdown, UnresolvedAutorefs}; use super::structure::nav::Navigation; use super::structure::page::Page; use super::structure::search::SearchIndex; use super::template::Template; use super::watcher::Source; +mod aggregate; mod cached; +use aggregate::aggregate; use cached::cached; // ---------------------------------------------------------------------------- @@ -82,23 +83,11 @@ pub struct Main { strict: bool, } -/// Page-content generation used to align validation inputs. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub(crate) struct PageGeneration(u64); - -impl zrx::scheduler::Value for PageGeneration {} - -/// Render generation used to align validation with rendered autoreferences. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub(crate) struct ValidationGeneration(u64, u64); - -impl zrx::scheduler::Value for ValidationGeneration {} - -/// Navigation generation used to construct validation sites. +/// Site snapshot generation a rendered page belongs to. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct ValidationNavigation(u64); +pub(crate) struct SiteGeneration(u64); -impl zrx::scheduler::Value for ValidationNavigation {} +impl zrx::scheduler::Value for SiteGeneration {} /// Site-wide validation pages shared across workflow products. #[derive(Clone, Debug, PartialEq, Eq)] @@ -106,37 +95,19 @@ struct SharedValidationPages(Arc); impl zrx::scheduler::Value for SharedValidationPages {} -/// Site-wide validation inputs shared across rendered pages. -#[derive(Clone, Debug, PartialEq, Eq)] -struct ValidationSite { - /// Monotonically increasing site snapshot sequence. - sequence: u64, - /// Navigation hash used when pages were rendered. - nav_hash: u64, - /// References, anchors, and generations for all pages. - pages: Arc, -} - -impl zrx::scheduler::Value for ValidationSite {} - /// Validation state collected while pages are rendered. #[derive(Debug, Default)] struct ValidationState { - /// Current site snapshot sequence. - sequence: u64, - /// Whether issues for the current site snapshot were printed. - reported: bool, - /// Pages expected in the current site snapshot. - expected: HashSet<(Id, ValidationGeneration)>, - /// Pages that have not rendered for the current site snapshot. - missing: HashSet<(Id, ValidationGeneration)>, - /// Current site snapshot, if validation inputs are ready. - site: Option, - /// Autoref resolutions indexed by source page and generation. - resolutions: HashMap<(Id, ValidationGeneration), AutorefResolutions>, + /// Last site generation for which issues were printed. + printed: u64, + /// Site snapshots by generation. + sites: HashMap>, + /// Unresolved autorefs by generation and source page. + unresolved: HashMap>, } -type ValidationPages = Vec<(Key, (References, (Anchors, PageGeneration)))>; +type ValidationPages = Vec<(Key, (SharedReferences, Anchors))>; +type SitePages = Vec<(Key, ((Page, Anchors), SharedReferences))>; // ---------------------------------------------------------------------------- // Implementations @@ -156,7 +127,41 @@ impl Module for Main { // Generate pages, and use the barrier to ensure that all pages have been // processed, in order to create the navigation and search index let page = generate_page(&self.config, &markdown); - let pages = page.select([wait_for_markdown(&self.config)]); + let (pages, validation_pages) = + if self.config.project.validation.is_enabled() { + // Parse anchors per page, so that serve rebuilds only redo + // this for pages that changed + let pages_and_anchors = page.map(|page: Page| { + let anchors: Anchors = page.content.parse()?; + Ok::<_, anyhow::Error>((page, anchors)) + }); + + // Join in references and aggregate everything into site-wide + // snapshots, shared by navigation, search and validation + let references = collect_references(&self.config, &files); + let site = aggregate( + &pages_and_anchors.join(&references), + wait_for_markdown(&self.config), + ); + let pages = site.map(|pages: SitePages| { + pages + .into_iter() + .map(|(key, ((page, _), _))| (key, page)) + .collect::>() + }); + let validation = site.map(|pages: SitePages| { + let pages = pages + .into_iter() + .map(|(key, ((_, anchors), references))| { + (key, (references, anchors)) + }) + .collect::(); + SharedValidationPages(Arc::new(pages)) + }); + (pages, Some(validation)) + } else { + (aggregate(&page, wait_for_markdown(&self.config)), None) + }; // Generate navigation and search index let nav = generate_nav(&self.config, &pages); @@ -167,20 +172,11 @@ impl Module for Main { // Render static and extra templates, as well as pages. render_templates(&self.config, &files, &nav); - let autorefs = render_pages(&self.config, &page, &nav); + let unresolved = render_pages(&self.config, &page, &nav); - // Validate and print issues after all page autorefs have resolved. - if self.config.project.validation.is_enabled() { - let references = collect_references(&self.config, &files); - let anchors = collect_anchors(&page); - validate( - &self.config, - self.strict, - references, - anchors, - autorefs, - &nav, - ); + // Validate and print issues after all pages have rendered. + if let Some(validation_pages) = validation_pages { + validate(&self.config, self.strict, validation_pages, unresolved); } Ok(()) } @@ -214,7 +210,7 @@ pub fn wait_for_markdown(config: &Config) -> (Key, Barrier) { /// Create a stream to collect references from all Markdown files. pub fn collect_references( config: &Config, files: &Stream, -) -> Stream { +) -> Stream { let matcher = Arc::new( Matcher::from_str(&format!( "zrs::::{}:**/*.md:", @@ -226,51 +222,38 @@ pub fn collect_references( // Create pipeline to collect references files .filter(move |id: &Id| matcher.is_match(id).expect("invariant")) - .map(|Source { path }| fs::read_to_string(&*path)?.parse()) + .map(|Source { path }| { + let references: References = fs::read_to_string(&*path)?.parse()?; + Ok::<_, anyhow::Error>(SharedReferences::from(references)) + }) } -/// Create a stream to collect anchors and generations from pages. -pub fn collect_anchors( - pages: &Stream, -) -> Stream { - pages.map(move |page: Page| { - let generation = page_generation(&page); - Ok::<_, anyhow::Error>((page.content.parse()?, generation)) - }) -} - -/// Create a stream to validate references against anchors. -pub fn validate( - config: &Config, strict: bool, refs: Stream, - anchors: Stream, - autorefs: Stream, - nav: &Stream, +/// Create a stream to validate references and autorefs against anchors. +/// +/// Source-based issues are derived from the site snapshot, while unresolved +/// autorefs are collected from rendered pages. The snapshot generation ties +/// both together, so one combined report is printed per settled build. +fn validate( + config: &Config, strict: bool, pages: Stream, + unresolved: Stream, ) { let validation = config.project.validation.clone(); - let pages = refs - .join(&anchors) - .select([wait_for_markdown(config)]) - .map(|pages: ValidationPages| SharedValidationPages(Arc::new(pages))); - let nav = nav.map(|nav: Navigation| ValidationNavigation(nav.hash)); - let sequence = Arc::new(AtomicU64::new(0)); - let pages = pages.join(&nav).map( - move |pages: SharedValidationPages, nav: ValidationNavigation| { - ValidationSite { - sequence: sequence.fetch_add(1, Ordering::Relaxed) + 1, - nav_hash: nav.0, - pages: pages.0, - } - }, - ); let state = Arc::new(Mutex::new(ValidationState::default())); let site_state = Arc::clone(&state); let site_validation = validation.clone(); - pages.map(move |site: ValidationSite| { + pages.map(move |id: &Id, pages: SharedValidationPages| { + let generation = id + .fragment() + .expect("invariant") + .parse() + .expect("invariant"); let issues = { let mut state = site_state.lock().expect("invariant"); - update_validation_site(&mut state, site); - validation_issues(&mut state) + if generation > state.printed { + state.sites.insert(generation, pages.0); + } + validation_issues(&mut state, generation) }; if let Some(issues) = issues { issues.print(&site_validation, strict)?; @@ -278,18 +261,20 @@ pub fn validate( Ok::<_, anyhow::Error>(()) }); - autorefs.map( + unresolved.map( move |source: Id, - generation: ValidationGeneration, - autorefs: AutorefResolutions| { + SiteGeneration(generation): SiteGeneration, + unresolved: UnresolvedAutorefs| { let issues = { let mut state = state.lock().expect("invariant"); - let current = (source, generation); - state.resolutions.insert(current.clone(), autorefs); - if state.expected.contains(¤t) { - state.missing.remove(¤t); + if generation > state.printed { + state + .unresolved + .entry(generation) + .or_default() + .insert(source, unresolved); } - validation_issues(&mut state) + validation_issues(&mut state, generation) }; if let Some(issues) = issues { issues.print(&validation, strict)?; @@ -299,65 +284,50 @@ pub fn validate( ); } -/// Update validation state with a new site snapshot. -fn update_validation_site(state: &mut ValidationState, site: ValidationSite) { - if site.sequence <= state.sequence { - return; - } - - let expected = site - .pages - .iter() - .map(|(key, (_, (_, page_generation)))| { - let id = key.try_as_id().expect("invariant"); - let generation = - ValidationGeneration(page_generation.0, site.nav_hash); - (id.clone(), generation) - }) - .collect::>(); - state.resolutions.retain(|key, _| expected.contains(key)); - state.missing = expected - .iter() - .filter(|key| !state.resolutions.contains_key(*key)) - .cloned() - .collect(); - state.sequence = site.sequence; - state.reported = false; - state.expected = expected; - state.site = Some(site); -} - -/// Create issues once every page in the current snapshot has rendered. -fn validation_issues(state: &mut ValidationState) -> Option { - if state.reported || !state.missing.is_empty() { +/// Create issues once every page in a site snapshot has rendered. +fn validation_issues( + state: &mut ValidationState, generation: u64, +) -> Option { + if generation <= state.printed { return None; } - let site = state.site.as_ref()?; - let iter = site.pages.iter().cloned().map( - |(key, (references, (anchors, page_generation)))| { + // Check that every page in the snapshot has rendered for this generation + let complete = { + let site = state.sites.get(&generation)?; + let unresolved = state.unresolved.get(&generation); + site.iter().all(|(key, _)| { let id = key.try_as_id().expect("invariant"); - let generation = - ValidationGeneration(page_generation.0, site.nav_hash); - let autorefs = state - .resolutions - .get(&(id.clone(), generation)) - .cloned() - .expect("invariant"); - (key, (references, anchors, autorefs)) - }, - ); + unresolved.is_some_and(|pages| pages.contains_key(id)) + }) + }; + if !complete { + return None; + } + + // Combine unresolved autorefs with references and anchors per page, and + // drop this and all previous generations from the validation state + let site = state.sites.remove(&generation).expect("invariant"); + let mut unresolved = + state.unresolved.remove(&generation).unwrap_or_default(); + let iter = site.iter().cloned().map(|(key, (references, anchors))| { + let id = key.try_as_id().expect("invariant"); + let autorefs = unresolved.remove(id).expect("invariant"); + (key, (references, anchors, autorefs)) + }); let issues = Issues::new(iter); - state.reported = true; + state.printed = generation; + state.sites.retain(|current, _| *current > generation); + state.unresolved.retain(|current, _| *current > generation); Some(issues) } -/// Compute a generation from page content relevant to validation. -fn page_generation(page: &Page) -> PageGeneration { +/// Compute a hash of the page content relevant to template rendering. +fn page_hash(page: &Page) -> u64 { let mut hasher = DefaultHasher::new(); page.content.hash(&mut hasher); page.meta.hash(&mut hasher); - PageGeneration(hasher.finish()) + hasher.finish() } /// Create a stream to process static assets. @@ -531,12 +501,21 @@ pub fn generate_nav( config: &Config, pages: &Stream, Page)>>, ) -> Stream { let config = config.clone(); - pages.map(move |pages: Vec<(Key, Page)>| { - Navigation::new( + pages.map(move |id: &Id, pages: Vec<(Key, Page)>| { + let mut nav = Navigation::new( config.get_cache_dir(), config.project.nav.clone(), pages, - ) + ); + + // Adopt the generation of the site snapshot, so that pages rendered + // with this navigation can be attributed to the snapshot + nav.generation = id + .fragment() + .expect("invariant") + .parse() + .expect("invariant"); + nav }) } @@ -657,32 +636,32 @@ pub fn render_templates( /// Render pages. pub fn render_pages( config: &Config, page: &Stream, nav: &Stream, -) -> Stream { +) -> Stream { let config = config.clone(); let pages = page.map(|id: &Id, page: Page| (id.clone(), page)); pages.product(nav).map( move |(source, mut page): (Id, Page), nav: Navigation| { let id = page.url.clone(); - // Compute hash of page content - let page_generation = page_generation(&page); - let generation = ValidationGeneration(page_generation.0, nav.hash); + // Cache template rendering independently of autorefs, which are + // substituted below on every pass. Deriving a cache key for the + // substitution would require the same resolution scan that the + // substitution itself performs, so caching it can't pay off. + let args = (config.hash, nav.hash, page_hash(&page)); + let rendered = + cached(&config, ("template", id), args, |(_, _, _)| { + Ok(page.render_template(&config, nav.clone())?) + })?; - // Render page if we don't have a recent cached version at our own - // disposal. Otherwise, just return if the content did not change. - let args = (config.hash, nav.hash, page_generation.0); - cached( - &config, - id, - args, - |(_, _, _)| Ok(page.render(&config, nav)?), - ) - .and_then(|(data, autorefs)| { - let path = Path::new(&page.path); - fs::create_dir_all(path.parent().expect("invariant"))?; - fs::write(path, &*data)?; - Ok((source, generation, autorefs)) - }) + // Replace autorefs and retain unresolved identifiers + let (data, unresolved) = + nav.autorefs.replace_in(&rendered, &page.url); + + let path = Path::new(&page.path); + fs::create_dir_all(path.parent().expect("invariant"))?; + fs::write(path, &data)?; + let generation = SiteGeneration(nav.generation); + Ok::<_, anyhow::Error>((source, generation, unresolved)) }, ) } diff --git a/crates/zensical/src/workflow/aggregate.rs b/crates/zensical/src/workflow/aggregate.rs new file mode 100644 index 0000000..b76c51d --- /dev/null +++ b/crates/zensical/src/workflow/aggregate.rs @@ -0,0 +1,254 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. + +// ---------------------------------------------------------------------------- + +//! Stream aggregation for rebuildable site-wide snapshots. + +use ahash::HashSet; +use std::marker::PhantomData; +use zrx::id::{id, Id}; +use zrx::scheduler::action::context::Binding; +use zrx::scheduler::action::options::{Event, Interest}; +use zrx::scheduler::action::{Action, Context, Options}; +use zrx::scheduler::schedule::Subscriber; +use zrx::scheduler::step::{IntoSteps, Scope}; +use zrx::scheduler::{Key, Value}; +use zrx::stream::operator::Operator; +use zrx::stream::{Barrier, Stream}; + +// ---------------------------------------------------------------------------- +// Structs +// ---------------------------------------------------------------------------- + +/// Aggregate all values matching a barrier into one site-wide snapshot. +struct Aggregate { + /// Base output scope. + output: Key, + /// Barrier selecting source scopes. + barrier: Barrier, + /// Source scopes that have not reached this stream yet. + pending: HashSet>, + /// Current output scope. + current: Option>, + /// Output generation. + generation: u64, + /// Capture value type. + marker: PhantomData, +} + +// ---------------------------------------------------------------------------- +// Implementations +// ---------------------------------------------------------------------------- + +impl Aggregate { + /// Create an aggregate for the given output scope and barrier. + fn new(output: Key, barrier: Barrier) -> Self { + Self { + output, + barrier, + pending: HashSet::default(), + current: None, + generation: 0, + marker: PhantomData, + } + } +} + +// ---------------------------------------------------------------------------- +// Trait implementations +// ---------------------------------------------------------------------------- + +impl Action for Aggregate +where + T: Value + Clone, +{ + type Inputs = (T,); + type Output<'a> = Vec<(Key, T)>; + + fn execute(&mut self, ctx: Context) -> impl IntoSteps { + let Binding { + events, + scopes, + inputs, + mut output, + .. + } = ctx.bind(); + // Track every submitted source scope, including repeated submissions + // of an existing scope during serve rebuilds. + for event in events { + match event { + Event::Insert(scope) if self.barrier.contains(&scope) => { + self.pending.insert(scope); + } + Event::Remove(scope) => { + self.pending.remove(&scope); + } + Event::Insert(_) => {} + } + } + + // A scope reaching this action is complete for this stream. Repeated + // scopes must still advance the aggregate, even if they were already + // present during a previous build. + let mut advanced = false; + for scope in scopes { + if self.barrier.contains(scope.key()) { + self.pending.remove(scope.key()); + advanced = true; + } + } + + let complete = advanced && self.pending.is_empty(); + let mut steps = Vec::new(); + if complete { + let mut values = inputs + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect::>(); + values.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + + // A source submission can reach the aggregate before an updated + // asynchronous value does. Ignore that lifecycle-only advance; + // the changed value will arrive in a subsequent scope. + if self.current.as_ref().and_then(|key| output.get(key)) + == Some(&values) + { + return steps.into_iter(); + } + + // Repeated synthetic scopes are not propagated by the current + // runtime. Rotate the aggregate scope on every snapshot, removing + // the previous generation so downstream stores stay bounded. + self.generation = + self.generation.checked_add(1).expect("invariant"); + let id = id!( + self.output.try_as_id().expect("invariant"); + fragment = self.generation.to_string() + ) + .expect("invariant"); + let key = Key::from(id); + + if let Some(current) = self.current.replace(key.clone()) { + output.remove(¤t); + steps.push(Scope::from(current).done()); + } + output.insert(key.clone(), values); + steps.push(Scope::from(key).done()); + } + steps.into_iter() + } +} + +// ---------------------------------------------------------------------------- +// Functions +// ---------------------------------------------------------------------------- + +/// Aggregate a stream whenever all matching source scopes have completed. +pub fn aggregate( + stream: &Stream, (output, barrier): (Key, Barrier), +) -> Stream, T)>> +where + T: Value + Clone, +{ + let options = Options::default() + .interest(Interest::Enter) + .interest(Interest::Leave); + stream.subscribe( + Subscriber::new(Aggregate::new(output, barrier)).with_options(options), + ) +} + +// ---------------------------------------------------------------------------- +// Tests +// ---------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + use zrx::module::Context as ModuleContext; + use zrx::scheduler::Scheduler; + use zrx::stream::Workflow; + + #[derive(Clone, Debug, PartialEq, Eq)] + struct Number(u8); + + impl Value for Number {} + + fn tick_until( + scheduler: &mut Scheduler, condition: impl Fn() -> bool, + stage: &str, + ) { + for _ in 0..100 { + if condition() { + return; + } + scheduler.tick_timeout(Duration::from_millis(10)).unwrap(); + } + panic!("scheduler did not produce output after {stage}"); + } + + #[test] + fn test_aggregate_reemits_changed_repeated_scope() { + let context = ModuleContext::default(); + let input = context.add::(); + let root = Key::from( + id!(provider = "file", context = ".", location = ".").unwrap(), + ); + let barrier = + Barrier::new(|key: &Key| key[0].location().as_ref() == "item"); + let snapshots = aggregate(&input, (root, barrier)); + let seen = Arc::new(Mutex::new(Vec::new())); + let seen_by_stream = Arc::clone(&seen); + snapshots.map(move |values: Vec<(Key, Number)>| { + let (_, Number(value)) = values.first().expect("invariant"); + seen_by_stream.lock().expect("invariant").push(*value); + }); + drop(snapshots); + drop(input); + + let workflow: Workflow = context.into(); + let mut scheduler = Scheduler::::default(); + scheduler.attach(workflow); + let session = scheduler.session::(); + let item = + id!(provider = "file", context = ".", location = "item").unwrap(); + + session.insert(item.clone(), Number(1)).unwrap(); + tick_until( + &mut scheduler, + || seen.lock().expect("invariant").last() == Some(&1), + "first insertion", + ); + session.insert(item, Number(2)).unwrap(); + tick_until( + &mut scheduler, + || seen.lock().expect("invariant").last() == Some(&2), + "changed insertion", + ); + + assert_eq!(*seen.lock().expect("invariant"), [1, 2]); + } +} diff --git a/python/tests/integration/test_validation.py b/python/tests/integration/test_validation.py index 0b49612..15da309 100644 --- a/python/tests/integration/test_validation.py +++ b/python/tests/integration/test_validation.py @@ -23,20 +23,51 @@ from __future__ import annotations +import subprocess +import sys +import threading +import time from typing import TYPE_CHECKING import zensical if TYPE_CHECKING: + from collections.abc import Callable + from io import TextIOWrapper from pathlib import Path import pytest +def _wait_for( + condition: Callable[[], bool], + process: subprocess.Popen[str], + output: Callable[[], str], + *, + timeout: float = 10.0, +) -> None: + """Wait for a serve-process condition or report its captured output.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if condition(): + return + if process.poll() is not None: + raise AssertionError( + f"Serve process exited with {process.returncode}:\n{output()}" + ) + time.sleep(0.02) + raise AssertionError(f"Serve process timed out:\n{output()}") + + +def _collect_output(stream: TextIOWrapper, lines: list[str]) -> None: + """Collect process output without blocking its pipes.""" + lines.extend(stream) + + def test_validation_reports_issues_after_rendering( tmp_path: Path, capfd: pytest.CaptureFixture[str] ) -> None: - """Validation reports source and autoref issues after page rendering.""" + """Validation reports source and link issues after page rendering.""" docs = tmp_path / "docs" docs.mkdir() (docs / "index.md").write_text( @@ -74,3 +105,294 @@ unresolved_references = true captured = capfd.readouterr() assert "No issues found" in captured.err assert "2 issues found" not in captured.err + + +def test_validation_reports_unresolved_autorefs( + tmp_path: Path, capfd: pytest.CaptureFixture[str] +) -> None: + """Autorefs that fail to resolve are reported as invalid links.""" + docs = tmp_path / "docs" + docs.mkdir() + (docs / "index.md").write_text( + "# Hello\n\n[normal](missing.md)\n\n[autoref][missing-id]\n", + encoding="utf-8", + ) + (docs / "other.md").write_text("# Other\n", encoding="utf-8") + config = tmp_path / "zensical.toml" + config.write_text( + """ +[project] +site_name = "Test" + +[project.plugins.autorefs] +""".lstrip(), + encoding="utf-8", + ) + + zensical.build(str(config), {"clean": True, "strict": False}) + + captured = capfd.readouterr() + assert "page does not exist" in captured.err + assert "unresolved autoref" in captured.err + assert "2 issues found" in captured.err + assert captured.err.count("2 issues found") == 1 + + (docs / "index.md").write_text("# Hello\n", encoding="utf-8") + zensical.build(str(config), {"clean": True, "strict": False}) + + captured = capfd.readouterr() + assert "No issues found" in captured.err + assert "2 issues found" not in captured.err + + +def test_validation_reports_references_resolved_by_autorefs( + tmp_path: Path, capfd: pytest.CaptureFixture[str] +) -> None: + """The deprecated unresolved_references setting keeps its original + behavior, reporting references even when autorefs resolve them.""" + docs = tmp_path / "docs" + docs.mkdir() + (docs / "index.md").write_text( + "# Home\n\n[Target][target-heading]\n", encoding="utf-8" + ) + (docs / "other.md").write_text( + "# Other\n\n## target-heading\n", encoding="utf-8" + ) + config = tmp_path / "zensical.toml" + config.write_text( + """ +[project] +site_name = "Test" + +[project.plugins.autorefs] + +[project.validation] +unresolved_references = true +""".lstrip(), + encoding="utf-8", + ) + + zensical.build(str(config), {"clean": True, "strict": False}) + + captured = capfd.readouterr() + assert "unresolved link reference" in captured.err + assert "unresolved autoref" not in captured.err + assert "1 issue found" in captured.err + output = (tmp_path / "site" / "index.html").read_text(encoding="utf-8") + assert 'href="other/#target-heading"' in output + + +def test_validation_refreshes_cached_autoref_resolutions( + tmp_path: Path, capfd: pytest.CaptureFixture[str] +) -> None: + """Validation refreshes autorefs when targets are added and removed.""" + docs = tmp_path / "docs" + docs.mkdir() + identifier = "validation-target-only-available-after-rebuild" + (docs / "index.md").write_text( + f"# Home\n\n[Target][{identifier}]\n", encoding="utf-8" + ) + target = docs / "other.md" + target.write_text("# Other\n", encoding="utf-8") + stable_identifier = "validation-stable-target" + stable_marker = "validation-stable-cache-sentinel" + (docs / "stable.md").write_text( + ( + f"# Stable\n\n[{stable_marker}][{stable_identifier}]\n\n" + f"## {stable_identifier}\n" + ), + encoding="utf-8", + ) + config = tmp_path / "zensical.toml" + config.write_text( + """ +[project] +site_name = "Test" + +[project.plugins.autorefs] +""".lstrip(), + encoding="utf-8", + ) + + zensical.build(str(config), {"clean": True, "strict": False}) + + captured = capfd.readouterr() + assert "unresolved autoref" in captured.err + marker = stable_marker.encode() + + def stable_cache_entries() -> dict[str, bytes]: + cache_dir = tmp_path / ".cache" + return { + path.name: data + for path in cache_dir.iterdir() + if path.is_file() and marker in (data := path.read_bytes()) + } + + stable_cache = stable_cache_entries() + assert stable_cache + + target.write_text(f"# Other\n\n## {identifier}\n", encoding="utf-8") + zensical.build(str(config), {"clean": False, "strict": False}) + + captured = capfd.readouterr() + assert "No issues found" in captured.err + assert "unresolved autoref" not in captured.err + output = (tmp_path / "site" / "index.html").read_text(encoding="utf-8") + assert f'href="other/#{identifier}"' in output + assert stable_cache_entries() == stable_cache + + target.write_text("# Other\n", encoding="utf-8") + zensical.build(str(config), {"clean": False, "strict": False}) + + captured = capfd.readouterr() + assert "unresolved autoref" in captured.err + output = (tmp_path / "site" / "index.html").read_text(encoding="utf-8") + assert f'href="other/#{identifier}"' not in output + assert stable_cache_entries() == stable_cache + + +def test_serve_refreshes_autorefs_before_validation(tmp_path: Path) -> None: + """Serving resolves a fixed autoref without reporting a stale snapshot.""" + docs = tmp_path / "docs" + docs.mkdir() + identifier = "serve-target-only-available-after-rebuild" + (docs / "index.md").write_text( + f"# Home\n\n[Target][{identifier}]\n", encoding="utf-8" + ) + target = docs / "other.md" + target.write_text("# Other\n", encoding="utf-8") + config = tmp_path / "zensical.toml" + config.write_text( + """ +[project] +site_name = "Test" +dev_addr = "127.0.0.1:0" + +[project.plugins.autorefs] +""".lstrip(), + encoding="utf-8", + ) + + process = subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-m", + "zensical", + "serve", + "--config-file", + str(config), + ], + cwd=tmp_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + ) + assert process.stdout is not None + assert process.stderr is not None + stdout: list[str] = [] + stderr: list[str] = [] + threads = [ + threading.Thread( + target=_collect_output, + args=(process.stdout, stdout), + daemon=True, + ), + threading.Thread( + target=_collect_output, + args=(process.stderr, stderr), + daemon=True, + ), + ] + for thread in threads: + thread.start() + + def output() -> str: + return "".join(stdout + stderr) + + try: + _wait_for(lambda: "1 issue found" in "".join(stderr), process, output) + offset = len(stderr) + + target.write_text(f"# Other\n\n## {identifier}\n", encoding="utf-8") + _wait_for( + lambda: "No issues found" in "".join(stderr[offset:]), + process, + output, + ) + + rebuild_output = "".join(stderr[offset:]) + assert "unresolved autoref" not in rebuild_output + rendered = tmp_path / "site" / "index.html" + _wait_for( + lambda: ( + rendered.exists() + and f'href="other/#{identifier}"' + in rendered.read_text(encoding="utf-8") + ), + process, + output, + ) + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + for thread in threads: + thread.join(timeout=1) + + +def test_cached_template_autorefs_refresh_with_targets( + tmp_path: Path, capfd: pytest.CaptureFixture[str] +) -> None: + """Autorefs introduced by cached templates use current target data.""" + docs = tmp_path / "docs" + docs.mkdir() + (docs / "index.md").write_text("# Home\n", encoding="utf-8") + target = docs / "other.md" + target.write_text("# Other\n", encoding="utf-8") + + identifier = "template-autoref-target" + overrides = tmp_path / "overrides" + overrides.mkdir() + (overrides / "main.html").write_text( + ( + "
{{ page.content }}
" + f'Target' + ), + encoding="utf-8", + ) + config = tmp_path / "zensical.toml" + config.write_text( + """ +[project] +site_name = "Test" + +[project.theme] +custom_dir = "overrides" + +[project.plugins.autorefs] +""".lstrip(), + encoding="utf-8", + ) + + zensical.build(str(config), {"clean": True, "strict": False}) + output = (tmp_path / "site" / "index.html").read_text(encoding="utf-8") + assert f'href="other/#{identifier}"' not in output + + # Template autorefs cannot be located in the Markdown source, so they + # are reported as page-level issues without a source location + captured = capfd.readouterr() + assert f"unresolved autoref `{identifier}` in index.md" in captured.err + assert f"unresolved autoref `{identifier}` in other.md" in captured.err + assert "2 issues found" in captured.err + + target.write_text(f"# Other\n\n## {identifier}\n", encoding="utf-8") + zensical.build(str(config), {"clean": False, "strict": False}) + output = (tmp_path / "site" / "index.html").read_text(encoding="utf-8") + assert f'href="other/#{identifier}"' in output + + captured = capfd.readouterr() + assert "No issues found" in captured.err diff --git a/python/zensical/extensions/autorefs.py b/python/zensical/extensions/autorefs.py index 07b545d..11d28d5 100644 --- a/python/zensical/extensions/autorefs.py +++ b/python/zensical/extensions/autorefs.py @@ -88,6 +88,24 @@ class AutorefsStore: self._secondary_url_map: dict[str, list[str]] = {} self._abs_url_map: dict[str, str] = {} self._title_map: dict[str, str] = {} + self._updated_pages: set[str] = set() + self._page_registrations: dict[str, set[tuple[bool, str, str]]] = {} + + def set_page(self, page: Page) -> None: + """Set the current page and discard its previous registrations.""" + self.current_page = page + self._updated_pages.add(page.url) + for primary, identifier, url in self._page_registrations.pop( + page.url, set() + ): + url_map = ( + self._primary_url_map if primary else self._secondary_url_map + ) + urls = url_map[identifier] + urls.remove(url) + if not urls: + del url_map[identifier] + self._title_map.pop(url, None) def register_anchor( self, @@ -105,6 +123,9 @@ class AutorefsStore: url_map[identifier].append(url) else: url_map[identifier] = [url] + self._page_registrations.setdefault(page.url, set()).add( + (primary, identifier, url) + ) if title and url not in self._title_map: self._title_map[url] = title @@ -424,11 +445,14 @@ def get_autorefs_data() -> dict[str, Any]: mkdocstrings extension (for automatic cross-references). """ if AUTOREFS: + updated_pages = list(AUTOREFS._updated_pages) + AUTOREFS._updated_pages.clear() return { "primary": AUTOREFS._primary_url_map, "secondary": AUTOREFS._secondary_url_map, "inventory": AUTOREFS._abs_url_map, "titles": AUTOREFS._title_map, + "updated_pages": updated_pages, } return {} @@ -436,7 +460,7 @@ def get_autorefs_data() -> dict[str, Any]: def set_autorefs_page(page: Page) -> None: """Set autorefs current page.""" store = get_autorefs_store() - store.current_page = page + store.set_page(page) def reset() -> None: