diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d2f4c7..fe5b2d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -288,7 +288,7 @@ jobs: buildkitd-flags: --debug - name: Build image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . load: true diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 21c3e24..ce18577 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -38,7 +38,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 with: - ref: ${{ github.event.pull_request.base.sha }} + ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - name: Set up mono diff --git a/Cargo.lock b/Cargo.lock index 3eb5c9b..5ad9088 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1186,9 +1186,9 @@ dependencies = [ [[package]] name = "webbrowser" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe985f41e291eecef5e5c0770a18d28390addb03331c043964d9e916453d6f16" +checksum = "0fc95580916af1e68ff6a7be07446fc5db73ebf71cf092de939bbf5f7e189f72" dependencies = [ "core-foundation", "jni", @@ -1472,16 +1472,18 @@ checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65" [[package]] name = "zrx" -version = "0.0.12" +version = "0.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83047c9af3125774463cbcbbcf856cc89697b302f0fc49c2582a59ec66d4c026" +checksum = "a751c426c2581f40dbc1131a3388c722a01d131c1c8313ecb14f3b1f344251af" dependencies = [ "zrx-diagnostic", "zrx-executor", "zrx-graph", "zrx-id", + "zrx-module", "zrx-path", "zrx-scheduler", + "zrx-storage", "zrx-store", "zrx-stream", ] @@ -1494,9 +1496,9 @@ checksum = "1cf90c723631486819bb50d9871072595f4f17b2f6c596534d9cbcf4b6c2ae54" [[package]] name = "zrx-executor" -version = "0.0.1" +version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c0ace038e234e564aa9a352419b7eb8ea64996351affbfb8f2b1a593c3b24d4" +checksum = "51151cff60737328cc2512f6b21a781eb0bc4130a97f6628d261caea8c44f848" dependencies = [ "crossbeam", "thiserror", @@ -1504,9 +1506,9 @@ dependencies = [ [[package]] name = "zrx-graph" -version = "0.0.7" +version = "0.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c40e047c9ddb9468f63046cf57a9952cfd9cc8607f60762cd140b7c6c940f374" +checksum = "03f81542d95e53dc86ad2785074b480ff71b391356a2dba8d3413c0ebc6db7cf" dependencies = [ "ahash", "thiserror", @@ -1514,15 +1516,28 @@ dependencies = [ [[package]] name = "zrx-id" -version = "0.0.8" +version = "0.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd509179c6edbfa9eb0a642c274b7241bacfc09f1a840ede358ed241962c0822" +checksum = "c79bb424eaeb299ebe815e9340ebff062b30c84fdb0f3abeb81402dd7f05bf78" dependencies = [ + "ahash", "globset", "percent-encoding", "slab", "thiserror", "zrx-path", + "zrx-scheduler", +] + +[[package]] +name = "zrx-module" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db8c41040c9d474934d3809806d890a6d7f8e4a7aa518003b42f2e56d6d287" +dependencies = [ + "thiserror", + "zrx-id", + "zrx-stream", ] [[package]] @@ -1533,9 +1548,9 @@ checksum = "d578267e852d4f325ce124ecbffe0530d9f9013d58a3cfacec75daf04ac40d0b" [[package]] name = "zrx-scheduler" -version = "0.0.9" +version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6b5d00390d0277ebbcfcab156896b9ac8869512386b34245e8989a433d5c9b7" +checksum = "8142553f537ce3c5e0c82ae7bb4a84ae54942bce361c25f39f9cd656e08cc8fc" dependencies = [ "ahash", "crossbeam", @@ -1545,15 +1560,27 @@ dependencies = [ "zrx-diagnostic", "zrx-executor", "zrx-graph", - "zrx-id", + "zrx-storage", + "zrx-store", +] + +[[package]] +name = "zrx-storage" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ac271e365aa03e537b5cedabfa8f945c3f65b2302ca92cc1ba2c8011eb56935" +dependencies = [ + "ahash", + "slab", + "thiserror", "zrx-store", ] [[package]] name = "zrx-store" -version = "0.0.5" +version = "0.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a496e11596686700ce05773f46a570ed617abb5f0f6923e50785d681caff696" +checksum = "c60a5745e6a4b9afaf183be1bc32f35f74c93ce6cca84e0ad801f53f502a951b" dependencies = [ "ahash", "slab", @@ -1561,14 +1588,13 @@ dependencies = [ [[package]] name = "zrx-stream" -version = "0.0.9" +version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cde3ce73eac8413efa37e026df2a1b41aa46cb6d274559272788e67f2c8396a" +checksum = "32839b219080c3c9581432baa63e00dcc3a68bae5cf0bc18a45f9d55bf3d0fed" dependencies = [ - "ahash", "thiserror", "tracing", - "zrx-id", "zrx-scheduler", + "zrx-storage", "zrx-store", ] diff --git a/Cargo.toml b/Cargo.toml index 82e966f..5cda698 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,8 +68,8 @@ tracing = { version = "0.1" } tracing-chrome = "0.7" tracing-subscriber = "0.3.23" walkdir = "2.5" -webbrowser = "1.2.0" -zrx = "0.0.12" +webbrowser = "1.2.1" +zrx = "0.0.21" [workspace.dependencies.pyo3] version = "0.28.3" diff --git a/crates/zensical-watch/src/agent.rs b/crates/zensical-watch/src/agent.rs index b4f001f..ee0816f 100644 --- a/crates/zensical-watch/src/agent.rs +++ b/crates/zensical-watch/src/agent.rs @@ -39,7 +39,7 @@ mod monitor; pub use error::{Error, Result}; pub use event::Event; -pub use handler::Handler; +pub use handler::{Handler, Mode}; pub use manager::Manager; pub use monitor::{Kind, Monitor}; @@ -82,7 +82,7 @@ impl Agent { /// # Panics /// /// Panics if thread creation fails. - pub fn new(timeout: Duration, f: F) -> Self + pub fn new(timeout: Duration, mode: bool, f: F) -> Self where F: FnMut(Result) -> Result + Send + 'static, { @@ -97,7 +97,10 @@ impl Agent { // Start event loop, which will automatically exit when the file // agent is dropped, since the sender disconnects the receiver loop { - handler.handle(timeout)?; + handler.handle( + if mode { Mode::Serve } else { Mode::Build }, + timeout, + )?; } }; diff --git a/crates/zensical-watch/src/agent/handler.rs b/crates/zensical-watch/src/agent/handler.rs index c68eaf1..39fa01f 100644 --- a/crates/zensical-watch/src/agent/handler.rs +++ b/crates/zensical-watch/src/agent/handler.rs @@ -31,7 +31,7 @@ use std::mem; use std::path::PathBuf; use std::time::Duration; -use super::error::Result; +use super::error::{Error, Result}; use super::event::Event; use super::manager::Manager; use super::monitor::Monitor; @@ -41,6 +41,19 @@ mod builder; use builder::Builder; +// ---------------------------------------------------------------------------- +// Enums +// ---------------------------------------------------------------------------- + +/// File mode. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Mode { + /// Build mode. + Build, + /// Serve mode. + Serve, +} + // ---------------------------------------------------------------------------- // Structs // ---------------------------------------------------------------------------- @@ -83,7 +96,7 @@ impl Handler { /// Handles messages from the file agent and the file monitor. #[cfg_attr(feature = "tracing", tracing::instrument(skip_all))] - pub fn handle(&mut self, timeout: Duration) -> Result { + pub fn handle(&mut self, mode: Mode, timeout: Duration) -> Result { // When receiving events from the file system, we debounce processing // by the given timeout, as we need to give the file system some time // to settle down. This ensures, that we can correctly handle renames, @@ -121,6 +134,13 @@ impl Handler { } } + // Hack: we need to rebuild build and serve mode + recv(after(Duration::from_millis(100))) -> _ => { + if self.queue.is_empty() && mode == Mode::Build { + return Err(Error::Disconnected); + } + } + // Handle messages from the file monitor, which are sent whenever // a file system event is detected on a watched path recv(self.monitor.as_receiver()) -> message => { diff --git a/crates/zensical/src/lib.rs b/crates/zensical/src/lib.rs index e2b2366..26cfe49 100644 --- a/crates/zensical/src/lib.rs +++ b/crates/zensical/src/lib.rs @@ -37,7 +37,7 @@ use std::path::{Path, PathBuf}; use std::process; use std::time::{Duration, Instant}; use std::{fs, io, thread}; -use zrx::scheduler::action::Report; +use zrx::id::Id; use zrx::scheduler::Scheduler; mod config; @@ -50,7 +50,7 @@ mod workflow; use config::Config; use server::{create_server, ServeOptions}; use watcher::Watcher; -use workflow::create_workspace; +use workflow::create_workflow; // ---------------------------------------------------------------------------- // Enums @@ -86,13 +86,6 @@ fn setup_tracing() -> tracing_chrome::FlushGuard { guard } -/// Handle report from the scheduler. -fn handle(report: Report) { - for diagnostic in &report { - println!("[{:?}] {}", diagnostic.severity, diagnostic.message); - } -} - /// Wait until the file at the given path is touched. /// /// During the wait we also poll for Python signal handling so a keyboard @@ -154,8 +147,9 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult { } // Create workspace and scheduler - let workspace = create_workspace(&config); - let mut scheduler = Scheduler::new(workspace.into_builder().build()); + let workflow = create_workflow(&config); + let mut scheduler = Scheduler::::default(); + scheduler.attach(workflow); // Create channel for reload notifications let (sender, receiver) = unbounded(); @@ -164,7 +158,7 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult { // assign the agent to a variable right now, or it is dropped, and will // automatically terminate. This is a temporary workaround until we could // better integrate the scheduler with the agent. - let session = scheduler.session().expect("invariant"); + let session = scheduler.session(); // If site should be served, create HTTP server - note that we must assign // the agent to a variable right now or it's dropped and will automatically @@ -188,7 +182,9 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult { Some(create_server(&config, receiver, options.clone())) } }; - let watcher = Watcher::new(&config, session, sender, waker.clone())?; + + let serve = matches!(mode, Mode::Serve(_, _)); + let watcher = Watcher::new(&config, serve, session, sender, waker.clone())?; // Hack: the scheduler and file agent are currently not synchronized, which // can lead to cases where the file agent is still busy reading the contents @@ -208,7 +204,7 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult { match mode { // Build mode - just exit when we're done Mode::Build(..) => { - handle(scheduler.tick_timeout(Duration::from_millis(100))); + scheduler.tick_timeout(Duration::from_millis(100)); if scheduler.is_empty() { let elapsed = time.elapsed().as_secs_f32(); println!("Build finished in {elapsed:.2}s"); @@ -220,7 +216,7 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult { // the scheduler with the agent, we can remove this temporary hack // and have immediate reloading. Mode::Serve(..) => { - handle(scheduler.tick_timeout(Duration::from_millis(100))); + scheduler.tick_timeout(Duration::from_millis(100)); if watcher.is_terminated() { // Wake the server if let Some(waker) = &waker { diff --git a/crates/zensical/src/structure/markdown.rs b/crates/zensical/src/structure/markdown.rs index fae0dfa..ef4d085 100644 --- a/crates/zensical/src/structure/markdown.rs +++ b/crates/zensical/src/structure/markdown.rs @@ -30,9 +30,8 @@ use pyo3::{FromPyObject, PyErr, Python}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use zrx::id::Id; -use zrx::scheduler::action::report::IntoReport; -use zrx::scheduler::action::Error; -use zrx::scheduler::Value; +use zrx::scheduler::step::{Error, Result}; +use zrx::stream::Value; use crate::structure::dynamic::Dynamic; use crate::structure::nav::to_title; @@ -70,9 +69,7 @@ pub struct Markdown { impl Markdown { /// Renders Markdown using Python Markdown. #[cfg_attr(feature = "tracing", tracing::instrument(skip_all))] - pub fn new( - id: &Id, url: String, content: String, - ) -> impl IntoReport { + pub fn new(id: &Id, url: String, content: String) -> Result { let id = id.clone(); Python::attach(|py| { let module = py.import("zensical.markdown")?; diff --git a/crates/zensical/src/structure/nav.rs b/crates/zensical/src/structure/nav.rs index 0c6de0e..3979147 100644 --- a/crates/zensical/src/structure/nav.rs +++ b/crates/zensical/src/structure/nav.rs @@ -32,8 +32,7 @@ use pyo3::types::PyAnyMethods; use pyo3::{FromPyObject, Python}; use serde::Serialize; use zrx::id::Id; -use zrx::scheduler::Value; -use zrx::stream::value::Chunk; +use zrx::scheduler::{Scope, Value}; use crate::structure::markdown::Autorefs; @@ -73,7 +72,9 @@ pub struct Navigation { impl Navigation { /// Creates a navigation from the given items. - pub fn new(mut items: Vec, pages: Chunk) -> Self { + pub fn new( + mut items: Vec, pages: Vec<(Scope, Page)>, + ) -> Self { if items.is_empty() { return Self::from(pages); } @@ -82,9 +83,9 @@ impl Navigation { // icons from the file location of the respective page. let pages = pages .into_iter() - .map(|item| { - let id = item.id.location().to_string(); - (id, item.data) + .map(|(id, page)| { + let id = id[0].location().to_string(); + (id, page) }) .collect::>(); @@ -129,7 +130,26 @@ impl Navigation { // Determine homepage - here, we mirror MkDocs behavior, which only // considers index pages at the root level as potential homepages - let homepage = items.iter().find(|item| item.is_index).cloned(); + let mut homepage = items.iter().find(|item| item.is_index).cloned(); + if homepage.is_none() { + // However, if we couldn't find anything, but there's still an index + // page, we check if it's out of navigation, and if so, use it + if let Some(page) = pages.get("index.md") { + if !Iter::new(&items) + .any(|item| item.url.as_deref() == Some(&page.url)) + { + homepage = Some(NavigationItem { + title: Some(page.title.clone()), + url: Some(page.url.clone()), + canonical_url: page.canonical_url.clone(), + meta: Some(page.meta.clone()), + children: Vec::new(), + is_index: true, + active: false, + }); + } + } + } // Precompute hash let hash = { @@ -267,25 +287,25 @@ impl Value for Navigation {} // ---------------------------------------------------------------------------- -impl From> for Navigation { +impl From, Page)>> for Navigation { /// Creates a navigation from pages. /// /// This mirrors the functionality of auto-populated navigation that MkDocs /// provides. In the future, we intend to refactor this into a more flexible /// system that allows for custom and modular navigation structures, but for /// now, compatibility is key. - fn from(pages: Chunk) -> Self { + fn from(pages: Vec<(Scope, Page)>) -> Self { let mut items: Vec = Vec::new(); // Convert chunk into a vector for easier processing, and sort pages by // the exact same method that MkDocs uses let mut pages = Vec::from_iter(pages); - pages.sort_by_key(|item| file_sort_key(&item.id)); + pages.sort_by_key(|(id, _)| file_sort_key(&id[0])); // There can only be pages, no URLs, since we're auto-populating the // navigation from the files in the docs directory - for page in pages { - let location = page.id.location(); + for (id, page) in pages { + let location = id[0].location(); // Split location into components at slashes let mut components = location @@ -328,10 +348,10 @@ impl From> for Navigation { // Insert page into the section section.push(NavigationItem { - title: Some(page.data.title), - url: Some(page.data.url), - canonical_url: page.data.canonical_url, - meta: Some(page.data.meta.clone()), + title: Some(page.title), + url: Some(page.url), + canonical_url: page.canonical_url, + meta: Some(page.meta.clone()), children: Vec::new(), is_index: is_index(&file), active: false, diff --git a/crates/zensical/src/structure/page.rs b/crates/zensical/src/structure/page.rs index 01bb60c..047132b 100644 --- a/crates/zensical/src/structure/page.rs +++ b/crates/zensical/src/structure/page.rs @@ -35,7 +35,7 @@ use zrx::id::Id; use zrx::scheduler::Value; use crate::config::Config; -use crate::template::{Template, GENERATOR}; +use crate::template::{Output, Template, GENERATOR}; use super::dynamic::Dynamic; use super::markdown::Markdown; @@ -108,7 +108,7 @@ impl Page { // Create identifier builder, as we need to change the context in order // to copy the file over to the site directory - let builder = id.to_builder().with_context(&site_dir); + let builder = id.to_builder().context(&site_dir); let id = builder.clone().build().expect("invariant"); // Next, obtain the path, and check whether it is an index file, which @@ -136,7 +136,7 @@ impl Page { // more convenience function to the id crate, we can make this shorter let path = path.to_string_lossy().into_owned(); let id = builder - .with_location(path.replace('\\', "/")) + .location(path.replace('\\', "/")) .build() .expect("invariant"); @@ -197,7 +197,7 @@ impl Page { )] pub fn render( &mut self, config: &Config, nav: Navigation, - ) -> Result { + ) -> Result { let name = self.meta.get("template").map(ToString::to_string); let template = Template::new( name.unwrap_or(String::from("main.html")), @@ -224,7 +224,7 @@ impl Page { })?; // Replace autorefs, if any - Ok(nav.autorefs.replace_in(output, &self.url)) + Ok(Output::from(nav.autorefs.replace_in(output, &self.url))) } /// Returns the tags of the page. diff --git a/crates/zensical/src/structure/search.rs b/crates/zensical/src/structure/search.rs index 286b332..e33e143 100644 --- a/crates/zensical/src/structure/search.rs +++ b/crates/zensical/src/structure/search.rs @@ -28,8 +28,7 @@ use pyo3::FromPyObject; use serde::Serialize; use zrx::id::Id; -use zrx::scheduler::Value; -use zrx::stream::value::Chunk; +use zrx::scheduler::{Scope, Value}; use crate::config::plugins::SearchPluginConfig; @@ -71,44 +70,45 @@ impl SearchIndex { /// Creates a search index from pages. #[allow(clippy::assigning_clones)] pub fn new( - pages: Chunk, nav: &Navigation, config: SearchPluginConfig, + pages: Vec<(Scope, Page)>, nav: &Navigation, + config: SearchPluginConfig, ) -> Self { let mut items: Vec = Vec::new(); // Convert chunk into a vector for easier processing, and sort pages by // the exact same method that MkDocs uses let mut pages = Vec::from_iter(pages); - pages.sort_by_key(|item| file_sort_key(&item.id)); + pages.sort_by_key(|(id, _)| file_sort_key(&id[0])); // Assemble search index, combining all items from all pages into a // single, flat list, adjusting the location to include the page URL - for page in pages { - let iter = nav.ancestors(&page.data).into_iter().rev(); + for (_id, page) in pages { + let iter = nav.ancestors(&page).into_iter().rev(); let mut path = iter .map(|item| item.title.expect("invariant")) .collect::>(); // Add page title to path if not already present - this might be // the true in case of index pages - if path.last() != Some(&page.data.title) { - path.push(page.data.title.clone()); + if path.last() != Some(&page.title) { + path.push(page.title.clone()); } // Extract page tags, if any let tags: Vec = - page.data.tags().into_iter().map(|tag| tag.name).collect(); + page.tags().into_iter().map(|tag| tag.name).collect(); // For each page, adjust the location of each item and add it to // the overall list - for mut item in page.data.search { + for mut item in page.search { let location = match item.location { - Some(id) => format!("{}#{}", page.data.url, id), - _ => page.data.url.clone(), + Some(id) => format!("{}#{}", page.url, id), + _ => page.url.clone(), }; // Fall back to page title, if item title is empty if item.title.is_empty() { - item.title = page.data.title.clone(); + item.title = page.title.clone(); } // Update location and path and add item diff --git a/crates/zensical/src/template.rs b/crates/zensical/src/template.rs index 3a1841f..0f47548 100644 --- a/crates/zensical/src/template.rs +++ b/crates/zensical/src/template.rs @@ -35,9 +35,11 @@ use super::structure::nav::Navigation; mod filter; mod loader; +mod output; use filter::{script_tag_filter, url_filter}; use loader::Loader; +pub use output::Output; // ---------------------------------------------------------------------------- // Structs diff --git a/crates/zensical/src/template/output.rs b/crates/zensical/src/template/output.rs new file mode 100644 index 0000000..7c76c1e --- /dev/null +++ b/crates/zensical/src/template/output.rs @@ -0,0 +1,67 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. + +// ---------------------------------------------------------------------------- + +//! MiniJinja template output. + +use std::ops::Deref; + +use serde::{Deserialize, Serialize}; +use zrx::stream::Value; + +// ---------------------------------------------------------------------------- +// Structs +// ---------------------------------------------------------------------------- + +/// Rendered template output. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Output(String); + +// ---------------------------------------------------------------------------- +// Trait implementations +// ---------------------------------------------------------------------------- + +impl Value for Output {} + +// ---------------------------------------------------------------------------- + +impl From for Output { + /// Creates an output from a string. + #[inline] + fn from(value: String) -> Self { + Self(value) + } +} + +// ---------------------------------------------------------------------------- + +impl Deref for Output { + type Target = String; + + /// Dereferences the output to a string. + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/crates/zensical/src/watcher.rs b/crates/zensical/src/watcher.rs index f235d17..991f1d4 100644 --- a/crates/zensical/src/watcher.rs +++ b/crates/zensical/src/watcher.rs @@ -39,6 +39,10 @@ use zrx::scheduler::Session; use super::config::Config; +mod source; + +pub use source::Source; + // ---------------------------------------------------------------------------- // Structs // ---------------------------------------------------------------------------- @@ -60,8 +64,8 @@ impl Watcher { /// Creates a file watcher. #[allow(clippy::too_many_lines)] pub fn new( - config: &Config, session: Session, reload: Sender, - waker: Option>, + config: &Config, serve: bool, session: Session, + reload: Sender, waker: Option>, ) -> Result { let mut sources = Vec::default(); @@ -106,7 +110,7 @@ impl Watcher { // Initialize file agent - we use a debounce interval of 20ms, which // should be sufficient to correctly determine rename events - let agent = Agent::new(Duration::from_millis(20), { + let agent = Agent::new(Duration::from_millis(20), serve, { let config = config.clone(); move |res| { // For now, we just swallow the event, as the file agent should @@ -200,14 +204,15 @@ impl Watcher { Event::Create { path, .. } | Event::Modify { path, .. } => { let data = path.to_string_lossy().into_owned(); - session.insert(to_id(path, &sources), data)?; + session + .insert(to_id(path, &sources), data.into())?; } // File was renamed Event::Rename { from, to, .. } => { let data = to.to_string_lossy().into_owned(); session.remove(to_id(from, &sources))?; - session.insert(to_id(to, &sources), data)?; + session.insert(to_id(to, &sources), data.into())?; } // File was removed @@ -252,7 +257,6 @@ impl Watcher { } } - // ---------------------------------------------------------------------------- // Functions // ---------------------------------------------------------------------------- @@ -267,9 +271,9 @@ fn to_id(path: Arc, sources: &[(PathBuf, String)]) -> Id { let location = suffix.to_str().unwrap_or(""); Some( Id::builder() - .with_provider("file") - .with_context(context.replace('\\', "/")) - .with_location(location.replace('\\', "/")) + .provider("file") + .context(context.replace('\\', "/")) + .location(location.replace('\\', "/")) .build() .expect("invariant"), ) diff --git a/crates/zensical/src/watcher/source.rs b/crates/zensical/src/watcher/source.rs new file mode 100644 index 0000000..d59e727 --- /dev/null +++ b/crates/zensical/src/watcher/source.rs @@ -0,0 +1,72 @@ +// Copyright (c) 2025-2026 Zensical and contributors + +// SPDX-License-Identifier: MIT +// All contributions are certified under the DCO + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. + +// ---------------------------------------------------------------------------- + +//! Source. + +use std::ops::Deref; + +use zrx::stream::Value; + +// ---------------------------------------------------------------------------- +// Structs +// ---------------------------------------------------------------------------- + +/// Source. +/// +/// From ZRX 0.0.17 on, all ata emitted in streams must explicitly implement the +/// `Value` trait. Right now, we just emit `String` representations of paths, +/// but as we develop the provider architecture, we'll switch to a structured +/// representation that includes the path as well as metadata. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Source { + /// Path as string. + pub path: String, +} + +// ---------------------------------------------------------------------------- +// Trait implementations +// ---------------------------------------------------------------------------- + +impl Value for Source {} + +// ---------------------------------------------------------------------------- + +impl From for Source { + /// Creates a source from a string. + #[inline] + fn from(path: String) -> Self { + Self { path } + } +} + +impl Deref for Source { + type Target = String; + + /// Dereferences the source to a string. + #[inline] + fn deref(&self) -> &Self::Target { + &self.path + } +} diff --git a/crates/zensical/src/workflow.rs b/crates/zensical/src/workflow.rs index 1182ec5..68b6557 100644 --- a/crates/zensical/src/workflow.rs +++ b/crates/zensical/src/workflow.rs @@ -34,13 +34,10 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::{Arc, LazyLock}; use std::{fs, io}; -use zrx::id::{Id, Matcher}; -use zrx::scheduler::action::report::IntoReport; -use zrx::stream::barrier::Condition; -use zrx::stream::function::{with_id, with_splat}; -use zrx::stream::value::{Chunk, Delta}; -use zrx::stream::workspace::Workspace; -use zrx::stream::Stream; +use zrx::id::{id, Id, Matcher}; +use zrx::module::{self, Context, Module}; +use zrx::scheduler::Scope; +use zrx::stream::{Barrier, Stream, Workflow}; use super::config::Config; use super::structure::markdown::Markdown; @@ -48,20 +45,94 @@ use super::structure::nav::Navigation; use super::structure::page::Page; use super::structure::search::SearchIndex; use super::template::Template; +use super::watcher::Source; mod cached; use cached::cached; +// ---------------------------------------------------------------------------- +// Constants +// ---------------------------------------------------------------------------- + +/// Regular expression to detect use of snippets static SNIPPET_RE: LazyLock = LazyLock::new(|| Regex::new(r"^[ \t]*-+8<-+").expect("invariant")); +// ---------------------------------------------------------------------------- +// Structs +// ---------------------------------------------------------------------------- + +/// Main module. +/// +/// With the advent of the module system at the beginning of April 2026, we can +/// start our journey to migrate all logic into modules. We now move the entire +/// build process into a single module, and then factor out functionality into +/// smaller, logically self-contained units. This approach ensures that we can +/// ship the module system as fast as possible, allowing us to work on feature +/// parity, while testing the module system in a real-world codebase. +#[derive(Debug)] +pub struct Main { + /// Configuration. + config: Config, +} + +// ---------------------------------------------------------------------------- +// Implementations +// ---------------------------------------------------------------------------- + +impl Module for Main { + /// Initializes the module. + fn setup(&self, ctx: &mut Context) -> module::Result { + let files = ctx.add::(); + + // Set up workflow to process static assets, as well as Markdown files, and + // create a barrier to wait for the completion of all Markdown files + process_theme_assets(&self.config, &files); + process_assets(&self.config, &files); + let markdown = process_markdown(&self.config, &files); + + // Return condition waiting for all Markdown files + let docs_dir = self.config.project.docs_dir.clone(); + let matcher = Matcher::from_str(&format!("zrs::::{docs_dir}:**/*.md:")) + .expect("invariant"); + let barrier = Barrier::new(move |id: &Scope| { + matcher.is_match(&id[0]).expect("invariant") + }); + + // Generate pages, and use the barrier to ensure that all pages have been + // processed, in order to create the navigation and search index + let page = generate_page(&self.config, &markdown); + let pages = page.select([( + Scope::from_iter([id!( + provider = "file", + context = ".", + location = "." + ) + .unwrap()]), + barrier, + )]); + + // Generate navigation and search index + let nav = generate_nav(&self.config, &pages); + generate_search_index(&self.config, &nav, &pages); + + // Generate object inventory + generate_object_inventory(&self.config, &pages); + + // // Render static and extra templates, as well as pages + render_templates(&self.config, &files, &nav); + render_pages(&self.config, &page, &nav); + Ok(()) + } +} + // ---------------------------------------------------------------------------- // Functions // ---------------------------------------------------------------------------- /// Create a stream to process static assets. -pub fn process_assets(config: &Config, files: &Stream) { +pub fn process_assets(config: &Config, files: &Stream) { let extra_templates = config.project.extra_templates.clone(); let docs_dir = config.project.docs_dir.clone(); let matcher = Arc::new( @@ -71,7 +142,7 @@ pub fn process_assets(config: &Config, files: &Stream) { // Create pipeline to copy static assets let site_dir = config.project.site_dir.clone(); let root_dir = config.get_root_dir(); - files.map(with_id(move |id: &Id, from: String| { + files.map(move |id: &Id, from: Source| { if !matcher.is_match(id).expect("invariant") { return Ok(()); } @@ -88,25 +159,27 @@ pub fn process_assets(config: &Config, files: &Stream) { // Create identifier builder, as we need to change the context in order // to copy the file over to the site directory - let builder = id.to_builder().with_context(&site_dir); + let builder = id.to_builder().context(&site_dir); let id = builder.build().expect("invariant"); // Compute parent path, create intermediate directories and copy files let to = root_dir.join(id.to_path()); - fs::create_dir_all(to.parent().expect("invariant"))?; - copy_file(from, to) - })); + fs::create_dir_all(to.parent().expect("invariant")) + .map_err(|err| Box::new(err) as Box<_>)?; + copy_file(&*from, to).map_err(|err| Box::new(err) as Box<_>)?; + Ok(()) + }); } /// Create a stream to process static assets in theme. -pub fn process_theme_assets(config: &Config, files: &Stream) { +pub fn process_theme_assets(config: &Config, files: &Stream) { let matcher = Arc::new(Matcher::from_str("zrs::::templates/*::").expect("invariant")); // Create pipeline to copy static assets let site_dir = config.project.site_dir.clone(); let root_dir = config.get_root_dir(); - files.map(with_id(move |id: &Id, from: String| { + files.map(move |id: &Id, from: Source| { if !matcher.is_match(id).expect("invariant") { return Ok(()); } @@ -118,14 +191,16 @@ pub fn process_theme_assets(config: &Config, files: &Stream) { // Create identifier builder, as we need to change the context in order // to copy the file over to the site directory - let builder = id.to_builder().with_context(&site_dir); + let builder = id.to_builder().context(&site_dir); let id = builder.build().expect("invariant"); // Compute parent path, create intermediate directories and copy files let to = root_dir.join(id.to_path()); - fs::create_dir_all(to.parent().expect("invariant"))?; - copy_file(from, to) - })); + fs::create_dir_all(to.parent().expect("invariant")) + .map_err(|err| Box::new(err) as Box<_>)?; + copy_file(&*from, to).map_err(|err| Box::new(err) as Box<_>)?; + Ok(()) + }); } /// Copy a file to a new location, without copying its permissions. @@ -139,7 +214,7 @@ fn copy_file( /// Create a stream to process Markdown files. pub fn process_markdown( - config: &Config, files: &Stream, + config: &Config, files: &Stream, ) -> Stream { let matcher = Arc::new( Matcher::from_str(&format!( @@ -152,94 +227,67 @@ pub fn process_markdown( // Create pipeline to render Markdown files let config = config.clone(); files - .filter(with_id(move |id: &Id, _: &_| { - matcher.is_match(id).expect("invariant") - })) + .filter(move |id: &Id, _: &_| { + Ok(matcher.is_match(id).expect("invariant")) + }) // Render Markdown if we don't have a recent cached version at our own // disposal. Otherwise, just return that if the content did not change. // Note that we need to limit concurrency here, or we'll overwhelm the // Python interpreter with all tasks competing for the GIL. - .map_concurrency( - with_id(move |id: &Id, path: String| { - let data = fs::read_to_string(path)?; + .map(move |id: &Id, path: Source| { + let data = fs::read_to_string(&*path) + .map_err(|err| Box::new(err) as Box<_>)?; - // Compute URL using same logic as Page::new() - let site_dir = config.project.site_dir.clone(); - let use_directory_urls = config.project.use_directory_urls; + // Compute URL using same logic as Page::new() + let site_dir = config.project.site_dir.clone(); + let use_directory_urls = config.project.use_directory_urls; - let builder = id.to_builder().with_context(&site_dir); - let url_id = builder.clone().build().expect("invariant"); + let builder = id.to_builder().context(&site_dir); + let url_id = builder.clone().build().expect("invariant"); - let mut url_path: PathBuf = - url_id.location().to_string().into(); - let is_index = url_path.ends_with("index.md") - || url_path.ends_with("README.md"); + let mut url_path: PathBuf = url_id.location().to_string().into(); + let is_index = url_path.ends_with("index.md") + || url_path.ends_with("README.md"); - if url_path.ends_with("README.md") { - url_path.pop(); - url_path = url_path.join("index.md"); - } + if url_path.ends_with("README.md") { + url_path.pop(); + url_path = url_path.join("index.md"); + } - if !use_directory_urls || is_index { - url_path.set_extension("html"); - } else { - url_path.set_extension(""); - url_path.push("index.html"); - } + if !use_directory_urls || is_index { + url_path.set_extension("html"); + } else { + url_path.set_extension(""); + url_path.push("index.html"); + } - let url_path = url_path.to_string_lossy().into_owned(); - let url_id = builder - .with_location(url_path.replace('\\', "/")) - .build() - .expect("invariant"); + let url_path = url_path.to_string_lossy().into_owned(); + let url_id = builder + .location(url_path.replace('\\', "/")) + .build() + .expect("invariant"); - let url = url_id.as_uri().to_string(); - let url = if use_directory_urls { - url.trim_end_matches("index.html").to_string() - } else { - url - }; + let url = url_id.as_uri().to_string(); + let url = if use_directory_urls { + url.trim_end_matches("index.html").to_string() + } else { + url + }; - // Don't cache page if it inserts (pymdownx) snippets. - // This is a hack while waiting for CommonMark (AST) and components, - // as well as topic-based authoring functionality. - if SNIPPET_RE.is_match(&data) { - Markdown::new(id, url, data).into_report() - } else { - cached( - &config, - id, - (config.hash, data.clone(), url.clone()), - |(_, data, url)| Markdown::new(id, url, data), - ) - .into_report() - } - }), - 1, - ) -} - -/// Create a stream to wait for all Markdown files to be rendered. -pub fn wait_for_markdown( - config: &Config, files: &Stream, -) -> Stream> { - let name = config.path.file_name().expect("invariant"); - let matcher = Arc::new( - Matcher::from_str(&format!("zrs:::::{}:", name.to_string_lossy())) - .expect("invariant"), - ); - - // Set up matcher to filter for the configuration file, and return a new - // stream that emits a condition in order to implement barriers - files.filter_map(with_id(move |id: &Id, _: _| { - matcher.is_match(id).expect("invariant").then(|| { - let matcher = - Matcher::from_str("zrs:::::**/*.md:").expect("invariant"); - - // Return condition waiting for all Markdown files - Condition::new(matcher) + // Don't cache page if it inserts (pymdownx) snippets. + // This is a hack while waiting for CommonMark (AST) and components, + // as well as topic-based authoring functionality. + if SNIPPET_RE.is_match(&data) { + Markdown::new(id, url, data) + } else { + cached( + &config, + id.as_str(), + (config.hash, data.clone(), url.clone()), + |(_, data, url)| Markdown::new(id, url, data), + ) + } }) - })) } /// Generate pages from Markdown files. @@ -247,24 +295,22 @@ pub fn generate_page( config: &Config, markdown: &Stream, ) -> Stream { let config = config.clone(); - markdown.map(with_id(move |id: &Id, markdown| { - Page::new(&config, id, markdown) - })) + markdown.map(move |id: &Id, markdown| Ok(Page::new(&config, id, markdown))) } /// Generate navigation from all pages. pub fn generate_nav( - config: &Config, pages: &Stream>, + config: &Config, pages: &Stream, Page)>>, ) -> Stream { let config = config.clone(); - pages.map(move |pages: Chunk| { - Navigation::new(config.project.nav.clone(), pages) + pages.map(move |pages: Vec<(Scope, Page)>| { + Ok(Navigation::new(config.project.nav.clone(), pages)) }) } /// Generate object inventory pub fn generate_object_inventory( - config: &Config, pages: &Stream>, + config: &Config, pages: &Stream, Page)>>, ) { // Retrieve inventory from Python interpreter using pyo3 let config = config.clone(); @@ -281,16 +327,17 @@ pub fn generate_object_inventory( let _ = fs::create_dir_all(path.parent().expect("invariant")); let _ = fs::write(path, &data); } + Ok(()) }); } /// Generate search index pub fn generate_search_index( config: &Config, nav: &Stream, - pages: &Stream>, + pages: &Stream, Page)>>, ) { let config = config.clone(); - pages.product(nav).delta_map(with_splat(move |pages, nav| { + pages.product(nav).map(move |pages, nav| { let plugin = config.project.plugins.search.config.clone(); let search = SearchIndex::new(pages, &nav, plugin); @@ -300,25 +347,28 @@ pub fn generate_search_index( // Write search index to disk let path = site_dir.join("search.json"); - fs::create_dir_all(path.parent().expect("invariant"))?; - fs::write(path, &data)?; + fs::create_dir_all(path.parent().expect("invariant")) + .map_err(|err| Box::new(err) as Box<_>)?; + fs::write(path, &data).map_err(|err| Box::new(err) as Box<_>)?; // If offline plugin is enabled, create search.js as well if config.project.plugins.offline.config.enabled { let path = site_dir.join("search.js"); - fs::create_dir_all(path.parent().expect("invariant"))?; - fs::write(path, format!("var __index = {data};").as_str())?; + fs::create_dir_all(path.parent().expect("invariant")) + .map_err(|err| Box::new(err) as Box<_>)?; + fs::write(path, format!("var __index = {data};").as_str()) + .map_err(|err| Box::new(err) as Box<_>)?; } // All files were written successfully - Ok::<_, io::Error>(()) - })); + Ok(()) + }); } /// Render static and extra templates. pub fn render_templates( - config: &Config, files: &Stream, nav: &Stream, -) -> Stream> { + config: &Config, files: &Stream, nav: &Stream, +) -> Stream { let docs_dir = config.project.docs_dir.clone(); // Retrieve template names @@ -339,9 +389,9 @@ pub fn render_templates( // Create matcher from builder, and filter templates let matcher = Arc::new(builder.build().expect("invariant")); - let templates = files.filter(with_id(move |id: &Id, _: &String| { - matcher.is_match(id).expect("invariant") - })); + let templates = files.filter(move |id: &Id, _: &Source| { + Ok(matcher.is_match(id).expect("invariant")) + }); // Add docs directory to theme templates let mut theme_dirs = config.theme_dirs.clone(); @@ -349,35 +399,33 @@ pub fn render_templates( // Create pipeline to render templates let config = config.clone(); - templates.product(nav).delta_map(with_splat( - move |template: String, nav: Navigation| { - let name = Path::new(&template).file_name().expect("invariant"); - let site_dir = config.get_site_dir(); + templates.product(nav).map(move |template: Source, nav| { + let name = Path::new(&*template).file_name().expect("invariant"); + let site_dir = config.get_site_dir(); - // Obtain template - let template = - Template::new(name.to_string_lossy(), theme_dirs.clone()); + // Obtain template + let template = + Template::new(name.to_string_lossy(), theme_dirs.clone()); - // Render template and write to disk - template - .render(&config, &nav) - .into_report() - .and_then(|report| { - let path = site_dir.join(name); - fs::create_dir_all(path.parent().expect("invariant"))?; - fs::write(path, &report.data).map_err(Into::into) - }) - }, - )) + // Render template and write to disk + let data = template + .render(&config, &nav) + .map_err(|err| Box::new(err) as Box<_>)?; + let path = site_dir.join(name); + fs::create_dir_all(path.parent().expect("invariant")) + .map_err(|err| Box::new(err) as Box<_>)?; + fs::write(path, &data).map_err(|err| Box::new(err) as Box<_>)?; + Ok(()) + }) } /// Render pages. pub fn render_pages( config: &Config, page: &Stream, nav: &Stream, -) -> Stream> { +) -> Stream { let config = config.clone(); - page.product(nav).delta_map(with_splat( - move |mut page: Page, nav: Navigation| { + page.product(nav) + .map(move |mut page: Page, nav: Navigation| { let id = page.url.clone(); // Compute hash of page content @@ -391,57 +439,31 @@ pub fn render_pages( // Render page if we don't have a recent cached version at our own // disposal. Otherwise, just return if the content did not change. let args = (config.hash, nav.hash, hash); - cached(&config, id, args, |(_, _, _)| page.render(&config, nav)) - .into_report() - .and_then(|report| { - let path = Path::new(&page.path); - fs::create_dir_all(path.parent().expect("invariant"))?; - fs::write(path, &report.data).map_err(Into::into).inspect( - |()| { - let url = percent_decode_str(&page.url); - println!("+ /{}", url.decode_utf8_lossy()); - }, - ) - }) - }, - )) + cached(&config, id, args, |(_, _, _)| { + Ok(page + .render(&config, nav) + .map_err(|err| Box::new(err) as Box<_>)?) + }) + .and_then(|data| { + let path = Path::new(&page.path); + fs::create_dir_all(path.parent().expect("invariant")) + .map_err(|err| Box::new(err) as Box<_>)?; + fs::write(path, &*data) + .map_err(|err| Box::new(err) as Box<_>) + .map_err(Into::into) + .inspect(|()| { + let url = percent_decode_str(&page.url); + println!("+ /{}", url.decode_utf8_lossy()); + }) + }) + }) } -/// Creates a new workspace for the given config. -pub fn create_workspace(config: &Config) -> Workspace { - let workspace = Workspace::new(); - let config = config.clone(); - - // Right now, we use a single workflow for the entirety of the build. Later, - // when we work on the module system, modules will have their own workflows. - // Create a source for files, so the file agent can submit file creation, - // change and delete events to the workflow - let workflow = workspace.add_workflow(); - let files = workflow.add_source::(); - - // Set up workflow to process static assets, as well as Markdown files, and - // create a barrier to wait for the completion of all Markdown files - process_theme_assets(&config, &files); - process_assets(&config, &files); - let markdown = process_markdown(&config, &files); - let wait = wait_for_markdown(&config, &files); - - // Generate pages, and use the barrier to ensure that all pages have been - // processed, in order to create the navigation and search index - let page = generate_page(&config, &markdown); - let pages = page.select(&wait).chunks(); - - // Generate navigation and search index - let nav = generate_nav(&config, &pages); - generate_search_index(&config, &nav, &pages); - - // Generate object inventory - generate_object_inventory(&config, &pages); - - // Render static and extra templates, as well as pages - render_templates(&config, &files, &nav); - render_pages(&config, &page, &nav); - - // Return workspace - workspace +/// Creates a workflow for the given config. +pub fn create_workflow(config: &Config) -> Workflow { + let mut context = Context::default(); + Main { config: config.clone() } + .setup(&mut context) + .expect("invariant"); + context.into() } diff --git a/crates/zensical/src/workflow/cached.rs b/crates/zensical/src/workflow/cached.rs index 0f77092..c2d5cfa 100644 --- a/crates/zensical/src/workflow/cached.rs +++ b/crates/zensical/src/workflow/cached.rs @@ -28,7 +28,7 @@ use serde::{Deserialize, Serialize}; use std::fs; use std::hash::{DefaultHasher, Hash, Hasher}; -use zrx::scheduler::action::report::IntoReport; +use zrx::scheduler::step::Result; use zrx::scheduler::Value; use crate::config::Config; @@ -54,14 +54,11 @@ struct Cached { /// input arguments. Note that this is only a preliminary implementation, and /// will be replaced with a more generic caching mechanism integrated into /// the runtime. -pub fn cached( - config: &Config, id: I, args: T, f: F, -) -> impl IntoReport +pub fn cached(config: &Config, id: I, args: T, f: F) -> Result where I: Hash, T: Hash, - F: FnOnce(T) -> R, - R: IntoReport, + F: FnOnce(T) -> Result, U: Value + Serialize + for<'de> Deserialize<'de>, { // Compute hash of identifier @@ -90,15 +87,15 @@ where if let Ok(cached) = serde_json::from_slice::>(&data) { // In case content hashes match, return cached data if cached.hash == hash { - return cached.data.into_report(); + return Ok(cached.data); } } } // Compute artifact and convert into report - note that we need to properly // handle encoding and file I/O errors here as well - f(args).into_report().inspect(|report| { - serde_json::to_string_pretty(&Cached { data: &report.data, hash }) + f(args).inspect(|data| { + serde_json::to_string_pretty(&Cached { data, hash }) .map(|content| fs::write(path, content).expect("invariant")) .expect("invariant"); })