lee/zensical
1
0
Fork 0
mirror of https://github.com/zensical/zensical.git synced 2026-08-23 07:36:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: update to zrx to v0.0.21

Browse Source 
Signed-off-by: squidfunk <martin.donath@squidfunk.com>
This commit is contained in:
Martin Donath
2026-04-18 12:21:23 +02:00
committed by GitHub
parent 670834fb04
commit 50aa8a5eb8
17 changed files with 514 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yml
+1 -1
View File
@@ -288,7 +288,7 @@ jobs:
buildkitd-flags: --debug
- name: Build image
uses: docker/build-push-action@v6
uses: docker/build-push-action@v7
with:
context: .
load: true
.github/workflows/commit.yml
+1 -1
View File
@@ -38,7 +38,7 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.base.sha }}
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
- name: Set up mono
Cargo.lock
Generated
+45 -19
View File
@@ -1186,9 +1186,9 @@ dependencies = [
[[package]]
name = "webbrowser"
version = "1.2.0"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe985f41e291eecef5e5c0770a18d28390addb03331c043964d9e916453d6f16"
checksum = "0fc95580916af1e68ff6a7be07446fc5db73ebf71cf092de939bbf5f7e189f72"
dependencies = [
"core-foundation",
"jni",
@@ -1472,16 +1472,18 @@ checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65"
[[package]]
name = "zrx"
version = "0.0.12"
version = "0.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83047c9af3125774463cbcbbcf856cc89697b302f0fc49c2582a59ec66d4c026"
checksum = "a751c426c2581f40dbc1131a3388c722a01d131c1c8313ecb14f3b1f344251af"
dependencies = [
"zrx-diagnostic",
"zrx-executor",
"zrx-graph",
"zrx-id",
"zrx-module",
"zrx-path",
"zrx-scheduler",
"zrx-storage",
"zrx-store",
"zrx-stream",
]
@@ -1494,9 +1496,9 @@ checksum = "1cf90c723631486819bb50d9871072595f4f17b2f6c596534d9cbcf4b6c2ae54"
[[package]]
name = "zrx-executor"
version = "0.0.1"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c0ace038e234e564aa9a352419b7eb8ea64996351affbfb8f2b1a593c3b24d4"
checksum = "51151cff60737328cc2512f6b21a781eb0bc4130a97f6628d261caea8c44f848"
dependencies = [
"crossbeam",
"thiserror",
@@ -1504,9 +1506,9 @@ dependencies = [
[[package]]
name = "zrx-graph"
version = "0.0.7"
version = "0.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c40e047c9ddb9468f63046cf57a9952cfd9cc8607f60762cd140b7c6c940f374"
checksum = "03f81542d95e53dc86ad2785074b480ff71b391356a2dba8d3413c0ebc6db7cf"
dependencies = [
"ahash",
"thiserror",
@@ -1514,15 +1516,28 @@ dependencies = [
[[package]]
name = "zrx-id"
version = "0.0.8"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd509179c6edbfa9eb0a642c274b7241bacfc09f1a840ede358ed241962c0822"
checksum = "c79bb424eaeb299ebe815e9340ebff062b30c84fdb0f3abeb81402dd7f05bf78"
dependencies = [
"ahash",
"globset",
"percent-encoding",
"slab",
"thiserror",
"zrx-path",
"zrx-scheduler",
]
[[package]]
name = "zrx-module"
version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4db8c41040c9d474934d3809806d890a6d7f8e4a7aa518003b42f2e56d6d287"
dependencies = [
"thiserror",
"zrx-id",
"zrx-stream",
]
[[package]]
@@ -1533,9 +1548,9 @@ checksum = "d578267e852d4f325ce124ecbffe0530d9f9013d58a3cfacec75daf04ac40d0b"
[[package]]
name = "zrx-scheduler"
version = "0.0.9"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6b5d00390d0277ebbcfcab156896b9ac8869512386b34245e8989a433d5c9b7"
checksum = "8142553f537ce3c5e0c82ae7bb4a84ae54942bce361c25f39f9cd656e08cc8fc"
dependencies = [
"ahash",
"crossbeam",
@@ -1545,15 +1560,27 @@ dependencies = [
"zrx-diagnostic",
"zrx-executor",
"zrx-graph",
"zrx-id",
"zrx-storage",
"zrx-store",
]
[[package]]
name = "zrx-storage"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ac271e365aa03e537b5cedabfa8f945c3f65b2302ca92cc1ba2c8011eb56935"
dependencies = [
"ahash",
"slab",
"thiserror",
"zrx-store",
]
[[package]]
name = "zrx-store"
version = "0.0.5"
version = "0.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a496e11596686700ce05773f46a570ed617abb5f0f6923e50785d681caff696"
checksum = "c60a5745e6a4b9afaf183be1bc32f35f74c93ce6cca84e0ad801f53f502a951b"
dependencies = [
"ahash",
"slab",
@@ -1561,14 +1588,13 @@ dependencies = [
[[package]]
name = "zrx-stream"
version = "0.0.9"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cde3ce73eac8413efa37e026df2a1b41aa46cb6d274559272788e67f2c8396a"
checksum = "32839b219080c3c9581432baa63e00dcc3a68bae5cf0bc18a45f9d55bf3d0fed"
dependencies = [
"ahash",
"thiserror",
"tracing",
"zrx-id",
"zrx-scheduler",
"zrx-storage",
"zrx-store",
]
Cargo.toml
+2 -2
View File
@@ -68,8 +68,8 @@ tracing = { version = "0.1" }
tracing-chrome = "0.7"
tracing-subscriber = "0.3.23"
walkdir = "2.5"
webbrowser = "1.2.0"
zrx = "0.0.12"
webbrowser = "1.2.1"
zrx = "0.0.21"
[workspace.dependencies.pyo3]
version = "0.28.3"
crates/zensical-watch/src/agent.rs
+6 -3
View File
@@ -39,7 +39,7 @@ mod monitor;
pub use error::{Error, Result};
pub use event::Event;
pub use handler::Handler;
pub use handler::{Handler, Mode};
pub use manager::Manager;
pub use monitor::{Kind, Monitor};
@@ -82,7 +82,7 @@ impl Agent {
/// # Panics
///
/// Panics if thread creation fails.
pub fn new<F>(timeout: Duration, f: F) -> Self
pub fn new<F>(timeout: Duration, mode: bool, f: F) -> Self
where
F: FnMut(Result<Event>) -> Result + Send + 'static,
{
@@ -97,7 +97,10 @@ impl Agent {
// Start event loop, which will automatically exit when the file
// agent is dropped, since the sender disconnects the receiver
loop {
handler.handle(timeout)?;
handler.handle(
if mode { Mode::Serve } else { Mode::Build },
timeout,
)?;
}
};
crates/zensical-watch/src/agent/handler.rs
+22 -2
View File
@@ -31,7 +31,7 @@ use std::mem;
use std::path::PathBuf;
use std::time::Duration;
use super::error::Result;
use super::error::{Error, Result};
use super::event::Event;
use super::manager::Manager;
use super::monitor::Monitor;
@@ -41,6 +41,19 @@ mod builder;
use builder::Builder;
// ----------------------------------------------------------------------------
// Enums
// ----------------------------------------------------------------------------
/// File mode.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Mode {
/// Build mode.
Build,
/// Serve mode.
Serve,
}
// ----------------------------------------------------------------------------
// Structs
// ----------------------------------------------------------------------------
@@ -83,7 +96,7 @@ impl Handler {
/// Handles messages from the file agent and the file monitor.
#[cfg_attr(feature = "tracing", tracing::instrument(skip_all))]
pub fn handle(&mut self, timeout: Duration) -> Result {
pub fn handle(&mut self, mode: Mode, timeout: Duration) -> Result {
// When receiving events from the file system, we debounce processing
// by the given timeout, as we need to give the file system some time
// to settle down. This ensures, that we can correctly handle renames,
@@ -121,6 +134,13 @@ impl Handler {
}
}
// Hack: we need to rebuild build and serve mode
recv(after(Duration::from_millis(100))) -> _ => {
if self.queue.is_empty() && mode == Mode::Build {
return Err(Error::Disconnected);
}
}
// Handle messages from the file monitor, which are sent whenever
// a file system event is detected on a watched path
recv(self.monitor.as_receiver()) -> message => {
crates/zensical/src/lib.rs
+11 -15
View File
@@ -37,7 +37,7 @@ use std::path::{Path, PathBuf};
use std::process;
use std::time::{Duration, Instant};
use std::{fs, io, thread};
use zrx::scheduler::action::Report;
use zrx::id::Id;
use zrx::scheduler::Scheduler;
mod config;
@@ -50,7 +50,7 @@ mod workflow;
use config::Config;
use server::{create_server, ServeOptions};
use watcher::Watcher;
use workflow::create_workspace;
use workflow::create_workflow;
// ----------------------------------------------------------------------------
// Enums
@@ -86,13 +86,6 @@ fn setup_tracing() -> tracing_chrome::FlushGuard {
guard
}
/// Handle report from the scheduler.
fn handle(report: Report) {
for diagnostic in &report {
println!("[{:?}] {}", diagnostic.severity, diagnostic.message);
}
}
/// Wait until the file at the given path is touched.
///
/// During the wait we also poll for Python signal handling so a keyboard
@@ -154,8 +147,9 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult<bool> {
}
// Create workspace and scheduler
let workspace = create_workspace(&config);
let mut scheduler = Scheduler::new(workspace.into_builder().build());
let workflow = create_workflow(&config);
let mut scheduler = Scheduler::<Id>::default();
scheduler.attach(workflow);
// Create channel for reload notifications
let (sender, receiver) = unbounded();
@@ -164,7 +158,7 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult<bool> {
// assign the agent to a variable right now, or it is dropped, and will
// automatically terminate. This is a temporary workaround until we could
// better integrate the scheduler with the agent.
let session = scheduler.session().expect("invariant");
let session = scheduler.session();
// If site should be served, create HTTP server - note that we must assign
// the agent to a variable right now or it's dropped and will automatically
@@ -188,7 +182,9 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult<bool> {
Some(create_server(&config, receiver, options.clone()))
}
};
let watcher = Watcher::new(&config, session, sender, waker.clone())?;
let serve = matches!(mode, Mode::Serve(_, _));
let watcher = Watcher::new(&config, serve, session, sender, waker.clone())?;
// Hack: the scheduler and file agent are currently not synchronized, which
// can lead to cases where the file agent is still busy reading the contents
@@ -208,7 +204,7 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult<bool> {
match mode {
// Build mode - just exit when we're done
Mode::Build(..) => {
handle(scheduler.tick_timeout(Duration::from_millis(100)));
scheduler.tick_timeout(Duration::from_millis(100));
if scheduler.is_empty() {
let elapsed = time.elapsed().as_secs_f32();
println!("Build finished in {elapsed:.2}s");
@@ -220,7 +216,7 @@ fn run(config_file: &PathBuf, mode: Mode) -> PyResult<bool> {
// the scheduler with the agent, we can remove this temporary hack
// and have immediate reloading.
Mode::Serve(..) => {
handle(scheduler.tick_timeout(Duration::from_millis(100)));
scheduler.tick_timeout(Duration::from_millis(100));
if watcher.is_terminated() {
// Wake the server
if let Some(waker) = &waker {
crates/zensical/src/structure/markdown.rs
+3 -6
View File
@@ -30,9 +30,8 @@ use pyo3::{FromPyObject, PyErr, Python};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use zrx::id::Id;
use zrx::scheduler::action::report::IntoReport;
use zrx::scheduler::action::Error;
use zrx::scheduler::Value;
use zrx::scheduler::step::{Error, Result};
use zrx::stream::Value;
use crate::structure::dynamic::Dynamic;
use crate::structure::nav::to_title;
@@ -70,9 +69,7 @@ pub struct Markdown {
impl Markdown {
/// Renders Markdown using Python Markdown.
#[cfg_attr(feature = "tracing", tracing::instrument(skip_all))]
pub fn new(
id: &Id, url: String, content: String,
) -> impl IntoReport<Markdown> {
pub fn new(id: &Id, url: String, content: String) -> Result<Markdown> {
let id = id.clone();
Python::attach(|py| {
let module = py.import("zensical.markdown")?;
crates/zensical/src/structure/nav.rs
+36 -16
View File
@@ -32,8 +32,7 @@ use pyo3::types::PyAnyMethods;
use pyo3::{FromPyObject, Python};
use serde::Serialize;
use zrx::id::Id;
use zrx::scheduler::Value;
use zrx::stream::value::Chunk;
use zrx::scheduler::{Scope, Value};
use crate::structure::markdown::Autorefs;
@@ -73,7 +72,9 @@ pub struct Navigation {
impl Navigation {
/// Creates a navigation from the given items.
pub fn new(mut items: Vec<NavigationItem>, pages: Chunk<Id, Page>) -> Self {
pub fn new(
mut items: Vec<NavigationItem>, pages: Vec<(Scope<Id>, Page)>,
) -> Self {
if items.is_empty() {
return Self::from(pages);
}
@@ -82,9 +83,9 @@ impl Navigation {
// icons from the file location of the respective page.
let pages = pages
.into_iter()
.map(|item| {
let id = item.id.location().to_string();
(id, item.data)
.map(|(id, page)| {
let id = id[0].location().to_string();
(id, page)
})
.collect::<HashMap<_, _>>();
@@ -129,7 +130,26 @@ impl Navigation {
// Determine homepage - here, we mirror MkDocs behavior, which only
// considers index pages at the root level as potential homepages
let homepage = items.iter().find(|item| item.is_index).cloned();
let mut homepage = items.iter().find(|item| item.is_index).cloned();
if homepage.is_none() {
// However, if we couldn't find anything, but there's still an index
// page, we check if it's out of navigation, and if so, use it
if let Some(page) = pages.get("index.md") {
if !Iter::new(&items)
.any(|item| item.url.as_deref() == Some(&page.url))
{
homepage = Some(NavigationItem {
title: Some(page.title.clone()),
url: Some(page.url.clone()),
canonical_url: page.canonical_url.clone(),
meta: Some(page.meta.clone()),
children: Vec::new(),
is_index: true,
active: false,
});
}
}
}
// Precompute hash
let hash = {
@@ -267,25 +287,25 @@ impl Value for Navigation {}
// ----------------------------------------------------------------------------
impl From<Chunk<Id, Page>> for Navigation {
impl From<Vec<(Scope<Id>, Page)>> for Navigation {
/// Creates a navigation from pages.
///
/// This mirrors the functionality of auto-populated navigation that MkDocs
/// provides. In the future, we intend to refactor this into a more flexible
/// system that allows for custom and modular navigation structures, but for
/// now, compatibility is key.
fn from(pages: Chunk<Id, Page>) -> Self {
fn from(pages: Vec<(Scope<Id>, Page)>) -> Self {
let mut items: Vec<NavigationItem> = Vec::new();
// Convert chunk into a vector for easier processing, and sort pages by
// the exact same method that MkDocs uses
let mut pages = Vec::from_iter(pages);
pages.sort_by_key(|item| file_sort_key(&item.id));
pages.sort_by_key(|(id, _)| file_sort_key(&id[0]));
// There can only be pages, no URLs, since we're auto-populating the
// navigation from the files in the docs directory
for page in pages {
let location = page.id.location();
for (id, page) in pages {
let location = id[0].location();
// Split location into components at slashes
let mut components = location
@@ -328,10 +348,10 @@ impl From<Chunk<Id, Page>> for Navigation {
// Insert page into the section
section.push(NavigationItem {
title: Some(page.data.title),
url: Some(page.data.url),
canonical_url: page.data.canonical_url,
meta: Some(page.data.meta.clone()),
title: Some(page.title),
url: Some(page.url),
canonical_url: page.canonical_url,
meta: Some(page.meta.clone()),
children: Vec::new(),
is_index: is_index(&file),
active: false,
crates/zensical/src/structure/page.rs
+5 -5
View File
@@ -35,7 +35,7 @@ use zrx::id::Id;
use zrx::scheduler::Value;
use crate::config::Config;
use crate::template::{Template, GENERATOR};
use crate::template::{Output, Template, GENERATOR};
use super::dynamic::Dynamic;
use super::markdown::Markdown;
@@ -108,7 +108,7 @@ impl Page {
// Create identifier builder, as we need to change the context in order
// to copy the file over to the site directory
let builder = id.to_builder().with_context(&site_dir);
let builder = id.to_builder().context(&site_dir);
let id = builder.clone().build().expect("invariant");
// Next, obtain the path, and check whether it is an index file, which
@@ -136,7 +136,7 @@ impl Page {
// more convenience function to the id crate, we can make this shorter
let path = path.to_string_lossy().into_owned();
let id = builder
.with_location(path.replace('\\', "/"))
.location(path.replace('\\', "/"))
.build()
.expect("invariant");
@@ -197,7 +197,7 @@ impl Page {
)]
pub fn render(
&mut self, config: &Config, nav: Navigation,
) -> Result<String, Error> {
) -> Result<Output, Error> {
let name = self.meta.get("template").map(ToString::to_string);
let template = Template::new(
name.unwrap_or(String::from("main.html")),
@@ -224,7 +224,7 @@ impl Page {
})?;
// Replace autorefs, if any
Ok(nav.autorefs.replace_in(output, &self.url))
Ok(Output::from(nav.autorefs.replace_in(output, &self.url)))
}
/// Returns the tags of the page.
crates/zensical/src/structure/search.rs
+13 -13
View File
@@ -28,8 +28,7 @@
use pyo3::FromPyObject;
use serde::Serialize;
use zrx::id::Id;
use zrx::scheduler::Value;
use zrx::stream::value::Chunk;
use zrx::scheduler::{Scope, Value};
use crate::config::plugins::SearchPluginConfig;
@@ -71,44 +70,45 @@ impl SearchIndex {
/// Creates a search index from pages.
#[allow(clippy::assigning_clones)]
pub fn new(
pages: Chunk<Id, Page>, nav: &Navigation, config: SearchPluginConfig,
pages: Vec<(Scope<Id>, Page)>, nav: &Navigation,
config: SearchPluginConfig,
) -> Self {
let mut items: Vec<SearchItem> = Vec::new();
// Convert chunk into a vector for easier processing, and sort pages by
// the exact same method that MkDocs uses
let mut pages = Vec::from_iter(pages);
pages.sort_by_key(|item| file_sort_key(&item.id));
pages.sort_by_key(|(id, _)| file_sort_key(&id[0]));
// Assemble search index, combining all items from all pages into a
// single, flat list, adjusting the location to include the page URL
for page in pages {
let iter = nav.ancestors(&page.data).into_iter().rev();
for (_id, page) in pages {
let iter = nav.ancestors(&page).into_iter().rev();
let mut path = iter
.map(|item| item.title.expect("invariant"))
.collect::<Vec<_>>();
// Add page title to path if not already present - this might be
// the true in case of index pages
if path.last() != Some(&page.data.title) {
path.push(page.data.title.clone());
if path.last() != Some(&page.title) {
path.push(page.title.clone());
}
// Extract page tags, if any
let tags: Vec<String> =
page.data.tags().into_iter().map(|tag| tag.name).collect();
page.tags().into_iter().map(|tag| tag.name).collect();
// For each page, adjust the location of each item and add it to
// the overall list
for mut item in page.data.search {
for mut item in page.search {
let location = match item.location {
Some(id) => format!("{}#{}", page.data.url, id),
_ => page.data.url.clone(),
Some(id) => format!("{}#{}", page.url, id),
_ => page.url.clone(),
};
// Fall back to page title, if item title is empty
if item.title.is_empty() {
item.title = page.data.title.clone();
item.title = page.title.clone();
}
// Update location and path and add item
crates/zensical/src/template.rs
+2
View File
@@ -35,9 +35,11 @@ use super::structure::nav::Navigation;
mod filter;
mod loader;
mod output;
use filter::{script_tag_filter, url_filter};
use loader::Loader;
pub use output::Output;
// ----------------------------------------------------------------------------
// Structs
crates/zensical/src/template/output.rs
+67
View File
@@ -0,0 +1,67 @@
// Copyright (c) 2025-2026 Zensical and contributors
// SPDX-License-Identifier: MIT
// All contributions are certified under the DCO
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
// ----------------------------------------------------------------------------
//! MiniJinja template output.
use std::ops::Deref;
use serde::{Deserialize, Serialize};
use zrx::stream::Value;
// ----------------------------------------------------------------------------
// Structs
// ----------------------------------------------------------------------------
/// Rendered template output.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Output(String);
// ----------------------------------------------------------------------------
// Trait implementations
// ----------------------------------------------------------------------------
impl Value for Output {}
// ----------------------------------------------------------------------------
impl From<String> for Output {
/// Creates an output from a string.
#[inline]
fn from(value: String) -> Self {
Self(value)
}
}
// ----------------------------------------------------------------------------
impl Deref for Output {
type Target = String;
/// Dereferences the output to a string.
#[inline]
fn deref(&self) -> &Self::Target {
&self.0
}
}
crates/zensical/src/watcher.rs
+13 -9
View File
@@ -39,6 +39,10 @@ use zrx::scheduler::Session;
use super::config::Config;
mod source;
pub use source::Source;
// ----------------------------------------------------------------------------
// Structs
// ----------------------------------------------------------------------------
@@ -60,8 +64,8 @@ impl Watcher {
/// Creates a file watcher.
#[allow(clippy::too_many_lines)]
pub fn new(
config: &Config, session: Session<Id, String>, reload: Sender<String>,
waker: Option<Arc<Waker>>,
config: &Config, serve: bool, session: Session<Id, Source>,
reload: Sender<String>, waker: Option<Arc<Waker>>,
) -> Result<Self> {
let mut sources = Vec::default();
@@ -106,7 +110,7 @@ impl Watcher {
// Initialize file agent - we use a debounce interval of 20ms, which
// should be sufficient to correctly determine rename events
let agent = Agent::new(Duration::from_millis(20), {
let agent = Agent::new(Duration::from_millis(20), serve, {
let config = config.clone();
move |res| {
// For now, we just swallow the event, as the file agent should
@@ -200,14 +204,15 @@ impl Watcher {
Event::Create { path, .. }
| Event::Modify { path, .. } => {
let data = path.to_string_lossy().into_owned();
session.insert(to_id(path, &sources), data)?;
session
.insert(to_id(path, &sources), data.into())?;
}
// File was renamed
Event::Rename { from, to, .. } => {
let data = to.to_string_lossy().into_owned();
session.remove(to_id(from, &sources))?;
session.insert(to_id(to, &sources), data)?;
session.insert(to_id(to, &sources), data.into())?;
}
// File was removed
@@ -252,7 +257,6 @@ impl Watcher {
}
}
// ----------------------------------------------------------------------------
// Functions
// ----------------------------------------------------------------------------
@@ -267,9 +271,9 @@ fn to_id(path: Arc<PathBuf>, sources: &[(PathBuf, String)]) -> Id {
let location = suffix.to_str().unwrap_or("");
Some(
Id::builder()
.with_provider("file")
.with_context(context.replace('\\', "/"))
.with_location(location.replace('\\', "/"))
.provider("file")
.context(context.replace('\\', "/"))
.location(location.replace('\\', "/"))
.build()
.expect("invariant"),
)
crates/zensical/src/watcher/source.rs
+72
View File
@@ -0,0 +1,72 @@
// Copyright (c) 2025-2026 Zensical and contributors
// SPDX-License-Identifier: MIT
// All contributions are certified under the DCO
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
// ----------------------------------------------------------------------------
//! Source.
use std::ops::Deref;
use zrx::stream::Value;
// ----------------------------------------------------------------------------
// Structs
// ----------------------------------------------------------------------------
/// Source.
///
/// From ZRX 0.0.17 on, all ata emitted in streams must explicitly implement the
/// `Value` trait. Right now, we just emit `String` representations of paths,
/// but as we develop the provider architecture, we'll switch to a structured
/// representation that includes the path as well as metadata.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Source {
/// Path as string.
pub path: String,
}
// ----------------------------------------------------------------------------
// Trait implementations
// ----------------------------------------------------------------------------
impl Value for Source {}
// ----------------------------------------------------------------------------
impl From<String> for Source {
/// Creates a source from a string.
#[inline]
fn from(path: String) -> Self {
Self { path }
}
}
impl Deref for Source {
type Target = String;
/// Dereferences the source to a string.
#[inline]
fn deref(&self) -> &Self::Target {
&self.path
}
}
crates/zensical/src/workflow.rs
+209 -187
View File
@@ -34,13 +34,10 @@ use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::{Arc, LazyLock};
use std::{fs, io};
use zrx::id::{Id, Matcher};
use zrx::scheduler::action::report::IntoReport;
use zrx::stream::barrier::Condition;
use zrx::stream::function::{with_id, with_splat};
use zrx::stream::value::{Chunk, Delta};
use zrx::stream::workspace::Workspace;
use zrx::stream::Stream;
use zrx::id::{id, Id, Matcher};
use zrx::module::{self, Context, Module};
use zrx::scheduler::Scope;
use zrx::stream::{Barrier, Stream, Workflow};
use super::config::Config;
use super::structure::markdown::Markdown;
@@ -48,20 +45,94 @@ use super::structure::nav::Navigation;
use super::structure::page::Page;
use super::structure::search::SearchIndex;
use super::template::Template;
use super::watcher::Source;
mod cached;
use cached::cached;
// ----------------------------------------------------------------------------
// Constants
// ----------------------------------------------------------------------------
/// Regular expression to detect use of snippets
static SNIPPET_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"^[ \t]*-+8<-+").expect("invariant"));
// ----------------------------------------------------------------------------
// Structs
// ----------------------------------------------------------------------------
/// Main module.
///
/// With the advent of the module system at the beginning of April 2026, we can
/// start our journey to migrate all logic into modules. We now move the entire
/// build process into a single module, and then factor out functionality into
/// smaller, logically self-contained units. This approach ensures that we can
/// ship the module system as fast as possible, allowing us to work on feature
/// parity, while testing the module system in a real-world codebase.
#[derive(Debug)]
pub struct Main {
/// Configuration.
config: Config,
}
// ----------------------------------------------------------------------------
// Implementations
// ----------------------------------------------------------------------------
impl Module for Main {
/// Initializes the module.
fn setup(&self, ctx: &mut Context) -> module::Result {
let files = ctx.add::<Source>();
// Set up workflow to process static assets, as well as Markdown files, and
// create a barrier to wait for the completion of all Markdown files
process_theme_assets(&self.config, &files);
process_assets(&self.config, &files);
let markdown = process_markdown(&self.config, &files);
// Return condition waiting for all Markdown files
let docs_dir = self.config.project.docs_dir.clone();
let matcher = Matcher::from_str(&format!("zrs::::{docs_dir}:**/*.md:"))
.expect("invariant");
let barrier = Barrier::new(move |id: &Scope<Id>| {
matcher.is_match(&id[0]).expect("invariant")
});
// Generate pages, and use the barrier to ensure that all pages have been
// processed, in order to create the navigation and search index
let page = generate_page(&self.config, &markdown);
let pages = page.select([(
Scope::from_iter([id!(
provider = "file",
context = ".",
location = "."
)
.unwrap()]),
barrier,
)]);
// Generate navigation and search index
let nav = generate_nav(&self.config, &pages);
generate_search_index(&self.config, &nav, &pages);
// Generate object inventory
generate_object_inventory(&self.config, &pages);
// // Render static and extra templates, as well as pages
render_templates(&self.config, &files, &nav);
render_pages(&self.config, &page, &nav);
Ok(())
}
}
// ----------------------------------------------------------------------------
// Functions
// ----------------------------------------------------------------------------
/// Create a stream to process static assets.
pub fn process_assets(config: &Config, files: &Stream<Id, String>) {
pub fn process_assets(config: &Config, files: &Stream<Id, Source>) {
let extra_templates = config.project.extra_templates.clone();
let docs_dir = config.project.docs_dir.clone();
let matcher = Arc::new(
@@ -71,7 +142,7 @@ pub fn process_assets(config: &Config, files: &Stream<Id, String>) {
// Create pipeline to copy static assets
let site_dir = config.project.site_dir.clone();
let root_dir = config.get_root_dir();
files.map(with_id(move |id: &Id, from: String| {
files.map(move |id: &Id, from: Source| {
if !matcher.is_match(id).expect("invariant") {
return Ok(());
}
@@ -88,25 +159,27 @@ pub fn process_assets(config: &Config, files: &Stream<Id, String>) {
// Create identifier builder, as we need to change the context in order
// to copy the file over to the site directory
let builder = id.to_builder().with_context(&site_dir);
let builder = id.to_builder().context(&site_dir);
let id = builder.build().expect("invariant");
// Compute parent path, create intermediate directories and copy files
let to = root_dir.join(id.to_path());
fs::create_dir_all(to.parent().expect("invariant"))?;
copy_file(from, to)
}));
fs::create_dir_all(to.parent().expect("invariant"))
.map_err(|err| Box::new(err) as Box<_>)?;
copy_file(&*from, to).map_err(|err| Box::new(err) as Box<_>)?;
Ok(())
});
}
/// Create a stream to process static assets in theme.
pub fn process_theme_assets(config: &Config, files: &Stream<Id, String>) {
pub fn process_theme_assets(config: &Config, files: &Stream<Id, Source>) {
let matcher =
Arc::new(Matcher::from_str("zrs::::templates/*::").expect("invariant"));
// Create pipeline to copy static assets
let site_dir = config.project.site_dir.clone();
let root_dir = config.get_root_dir();
files.map(with_id(move |id: &Id, from: String| {
files.map(move |id: &Id, from: Source| {
if !matcher.is_match(id).expect("invariant") {
return Ok(());
}
@@ -118,14 +191,16 @@ pub fn process_theme_assets(config: &Config, files: &Stream<Id, String>) {
// Create identifier builder, as we need to change the context in order
// to copy the file over to the site directory
let builder = id.to_builder().with_context(&site_dir);
let builder = id.to_builder().context(&site_dir);
let id = builder.build().expect("invariant");
// Compute parent path, create intermediate directories and copy files
let to = root_dir.join(id.to_path());
fs::create_dir_all(to.parent().expect("invariant"))?;
copy_file(from, to)
}));
fs::create_dir_all(to.parent().expect("invariant"))
.map_err(|err| Box::new(err) as Box<_>)?;
copy_file(&*from, to).map_err(|err| Box::new(err) as Box<_>)?;
Ok(())
});
}
/// Copy a file to a new location, without copying its permissions.
@@ -139,7 +214,7 @@ fn copy_file(
/// Create a stream to process Markdown files.
pub fn process_markdown(
config: &Config, files: &Stream<Id, String>,
config: &Config, files: &Stream<Id, Source>,
) -> Stream<Id, Markdown> {
let matcher = Arc::new(
Matcher::from_str(&format!(
@@ -152,94 +227,67 @@ pub fn process_markdown(
// Create pipeline to render Markdown files
let config = config.clone();
files
.filter(with_id(move |id: &Id, _: &_| {
matcher.is_match(id).expect("invariant")
}))
.filter(move |id: &Id, _: &_| {
Ok(matcher.is_match(id).expect("invariant"))
})
// Render Markdown if we don't have a recent cached version at our own
// disposal. Otherwise, just return that if the content did not change.
// Note that we need to limit concurrency here, or we'll overwhelm the
// Python interpreter with all tasks competing for the GIL.
.map_concurrency(
with_id(move |id: &Id, path: String| {
let data = fs::read_to_string(path)?;
.map(move |id: &Id, path: Source| {
let data = fs::read_to_string(&*path)
.map_err(|err| Box::new(err) as Box<_>)?;
// Compute URL using same logic as Page::new()
let site_dir = config.project.site_dir.clone();
let use_directory_urls = config.project.use_directory_urls;
// Compute URL using same logic as Page::new()
let site_dir = config.project.site_dir.clone();
let use_directory_urls = config.project.use_directory_urls;
let builder = id.to_builder().with_context(&site_dir);
let url_id = builder.clone().build().expect("invariant");
let builder = id.to_builder().context(&site_dir);
let url_id = builder.clone().build().expect("invariant");
let mut url_path: PathBuf =
url_id.location().to_string().into();
let is_index = url_path.ends_with("index.md")
|| url_path.ends_with("README.md");
let mut url_path: PathBuf = url_id.location().to_string().into();
let is_index = url_path.ends_with("index.md")
|| url_path.ends_with("README.md");
if url_path.ends_with("README.md") {
url_path.pop();
url_path = url_path.join("index.md");
}
if url_path.ends_with("README.md") {
url_path.pop();
url_path = url_path.join("index.md");
}
if !use_directory_urls || is_index {
url_path.set_extension("html");
} else {
url_path.set_extension("");
url_path.push("index.html");
}
if !use_directory_urls || is_index {
url_path.set_extension("html");
} else {
url_path.set_extension("");
url_path.push("index.html");
}
let url_path = url_path.to_string_lossy().into_owned();
let url_id = builder
.with_location(url_path.replace('\\', "/"))
.build()
.expect("invariant");
let url_path = url_path.to_string_lossy().into_owned();
let url_id = builder
.location(url_path.replace('\\', "/"))
.build()
.expect("invariant");
let url = url_id.as_uri().to_string();
let url = if use_directory_urls {
url.trim_end_matches("index.html").to_string()
} else {
url
};
let url = url_id.as_uri().to_string();
let url = if use_directory_urls {
url.trim_end_matches("index.html").to_string()
} else {
url
};
// Don't cache page if it inserts (pymdownx) snippets.
// This is a hack while waiting for CommonMark (AST) and components,
// as well as topic-based authoring functionality.
if SNIPPET_RE.is_match(&data) {
Markdown::new(id, url, data).into_report()
} else {
cached(
&config,
id,
(config.hash, data.clone(), url.clone()),
|(_, data, url)| Markdown::new(id, url, data),
)
.into_report()
}
}),
1,
)
}
/// Create a stream to wait for all Markdown files to be rendered.
pub fn wait_for_markdown(
config: &Config, files: &Stream<Id, String>,
) -> Stream<Id, Condition<Id>> {
let name = config.path.file_name().expect("invariant");
let matcher = Arc::new(
Matcher::from_str(&format!("zrs:::::{}:", name.to_string_lossy()))
.expect("invariant"),
);
// Set up matcher to filter for the configuration file, and return a new
// stream that emits a condition in order to implement barriers
files.filter_map(with_id(move |id: &Id, _: _| {
matcher.is_match(id).expect("invariant").then(|| {
let matcher =
Matcher::from_str("zrs:::::**/*.md:").expect("invariant");
// Return condition waiting for all Markdown files
Condition::new(matcher)
// Don't cache page if it inserts (pymdownx) snippets.
// This is a hack while waiting for CommonMark (AST) and components,
// as well as topic-based authoring functionality.
if SNIPPET_RE.is_match(&data) {
Markdown::new(id, url, data)
} else {
cached(
&config,
id.as_str(),
(config.hash, data.clone(), url.clone()),
|(_, data, url)| Markdown::new(id, url, data),
)
}
})
}))
}
/// Generate pages from Markdown files.
@@ -247,24 +295,22 @@ pub fn generate_page(
config: &Config, markdown: &Stream<Id, Markdown>,
) -> Stream<Id, Page> {
let config = config.clone();
markdown.map(with_id(move |id: &Id, markdown| {
Page::new(&config, id, markdown)
}))
markdown.map(move |id: &Id, markdown| Ok(Page::new(&config, id, markdown)))
}
/// Generate navigation from all pages.
pub fn generate_nav(
config: &Config, pages: &Stream<Id, Chunk<Id, Page>>,
config: &Config, pages: &Stream<Id, Vec<(Scope<Id>, Page)>>,
) -> Stream<Id, Navigation> {
let config = config.clone();
pages.map(move |pages: Chunk<Id, Page>| {
Navigation::new(config.project.nav.clone(), pages)
pages.map(move |pages: Vec<(Scope<Id>, Page)>| {
Ok(Navigation::new(config.project.nav.clone(), pages))
})
}
/// Generate object inventory
pub fn generate_object_inventory(
config: &Config, pages: &Stream<Id, Chunk<Id, Page>>,
config: &Config, pages: &Stream<Id, Vec<(Scope<Id>, Page)>>,
) {
// Retrieve inventory from Python interpreter using pyo3
let config = config.clone();
@@ -281,16 +327,17 @@ pub fn generate_object_inventory(
let _ = fs::create_dir_all(path.parent().expect("invariant"));
let _ = fs::write(path, &data);
}
Ok(())
});
}
/// Generate search index
pub fn generate_search_index(
config: &Config, nav: &Stream<Id, Navigation>,
pages: &Stream<Id, Chunk<Id, Page>>,
pages: &Stream<Id, Vec<(Scope<Id>, Page)>>,
) {
let config = config.clone();
pages.product(nav).delta_map(with_splat(move |pages, nav| {
pages.product(nav).map(move |pages, nav| {
let plugin = config.project.plugins.search.config.clone();
let search = SearchIndex::new(pages, &nav, plugin);
@@ -300,25 +347,28 @@ pub fn generate_search_index(
// Write search index to disk
let path = site_dir.join("search.json");
fs::create_dir_all(path.parent().expect("invariant"))?;
fs::write(path, &data)?;
fs::create_dir_all(path.parent().expect("invariant"))
.map_err(|err| Box::new(err) as Box<_>)?;
fs::write(path, &data).map_err(|err| Box::new(err) as Box<_>)?;
// If offline plugin is enabled, create search.js as well
if config.project.plugins.offline.config.enabled {
let path = site_dir.join("search.js");
fs::create_dir_all(path.parent().expect("invariant"))?;
fs::write(path, format!("var __index = {data};").as_str())?;
fs::create_dir_all(path.parent().expect("invariant"))
.map_err(|err| Box::new(err) as Box<_>)?;
fs::write(path, format!("var __index = {data};").as_str())
.map_err(|err| Box::new(err) as Box<_>)?;
}
// All files were written successfully
Ok::<_, io::Error>(())
}));
Ok(())
});
}
/// Render static and extra templates.
pub fn render_templates(
config: &Config, files: &Stream<Id, String>, nav: &Stream<Id, Navigation>,
) -> Stream<Id, Delta<Id, ()>> {
config: &Config, files: &Stream<Id, Source>, nav: &Stream<Id, Navigation>,
) -> Stream<Id, ()> {
let docs_dir = config.project.docs_dir.clone();
// Retrieve template names
@@ -339,9 +389,9 @@ pub fn render_templates(
// Create matcher from builder, and filter templates
let matcher = Arc::new(builder.build().expect("invariant"));
let templates = files.filter(with_id(move |id: &Id, _: &String| {
matcher.is_match(id).expect("invariant")
}));
let templates = files.filter(move |id: &Id, _: &Source| {
Ok(matcher.is_match(id).expect("invariant"))
});
// Add docs directory to theme templates
let mut theme_dirs = config.theme_dirs.clone();
@@ -349,35 +399,33 @@ pub fn render_templates(
// Create pipeline to render templates
let config = config.clone();
templates.product(nav).delta_map(with_splat(
move |template: String, nav: Navigation| {
let name = Path::new(&template).file_name().expect("invariant");
let site_dir = config.get_site_dir();
templates.product(nav).map(move |template: Source, nav| {
let name = Path::new(&*template).file_name().expect("invariant");
let site_dir = config.get_site_dir();
// Obtain template
let template =
Template::new(name.to_string_lossy(), theme_dirs.clone());
// Obtain template
let template =
Template::new(name.to_string_lossy(), theme_dirs.clone());
// Render template and write to disk
template
.render(&config, &nav)
.into_report()
.and_then(|report| {
let path = site_dir.join(name);
fs::create_dir_all(path.parent().expect("invariant"))?;
fs::write(path, &report.data).map_err(Into::into)
})
},
))
// Render template and write to disk
let data = template
.render(&config, &nav)
.map_err(|err| Box::new(err) as Box<_>)?;
let path = site_dir.join(name);
fs::create_dir_all(path.parent().expect("invariant"))
.map_err(|err| Box::new(err) as Box<_>)?;
fs::write(path, &data).map_err(|err| Box::new(err) as Box<_>)?;
Ok(())
})
}
/// Render pages.
pub fn render_pages(
config: &Config, page: &Stream<Id, Page>, nav: &Stream<Id, Navigation>,
) -> Stream<Id, Delta<Id, ()>> {
) -> Stream<Id, ()> {
let config = config.clone();
page.product(nav).delta_map(with_splat(
move |mut page: Page, nav: Navigation| {
page.product(nav)
.map(move |mut page: Page, nav: Navigation| {
let id = page.url.clone();
// Compute hash of page content
@@ -391,57 +439,31 @@ pub fn render_pages(
// Render page if we don't have a recent cached version at our own
// disposal. Otherwise, just return if the content did not change.
let args = (config.hash, nav.hash, hash);
cached(&config, id, args, |(_, _, _)| page.render(&config, nav))
.into_report()
.and_then(|report| {
let path = Path::new(&page.path);
fs::create_dir_all(path.parent().expect("invariant"))?;
fs::write(path, &report.data).map_err(Into::into).inspect(
|()| {
let url = percent_decode_str(&page.url);
println!("+ /{}", url.decode_utf8_lossy());
},
)
})
},
))
cached(&config, id, args, |(_, _, _)| {
Ok(page
.render(&config, nav)
.map_err(|err| Box::new(err) as Box<_>)?)
})
.and_then(|data| {
let path = Path::new(&page.path);
fs::create_dir_all(path.parent().expect("invariant"))
.map_err(|err| Box::new(err) as Box<_>)?;
fs::write(path, &*data)
.map_err(|err| Box::new(err) as Box<_>)
.map_err(Into::into)
.inspect(|()| {
let url = percent_decode_str(&page.url);
println!("+ /{}", url.decode_utf8_lossy());
})
})
})
}
/// Creates a new workspace for the given config.
pub fn create_workspace(config: &Config) -> Workspace<Id> {
let workspace = Workspace::new();
let config = config.clone();
// Right now, we use a single workflow for the entirety of the build. Later,
// when we work on the module system, modules will have their own workflows.
// Create a source for files, so the file agent can submit file creation,
// change and delete events to the workflow
let workflow = workspace.add_workflow();
let files = workflow.add_source::<String>();
// Set up workflow to process static assets, as well as Markdown files, and
// create a barrier to wait for the completion of all Markdown files
process_theme_assets(&config, &files);
process_assets(&config, &files);
let markdown = process_markdown(&config, &files);
let wait = wait_for_markdown(&config, &files);
// Generate pages, and use the barrier to ensure that all pages have been
// processed, in order to create the navigation and search index
let page = generate_page(&config, &markdown);
let pages = page.select(&wait).chunks();
// Generate navigation and search index
let nav = generate_nav(&config, &pages);
generate_search_index(&config, &nav, &pages);
// Generate object inventory
generate_object_inventory(&config, &pages);
// Render static and extra templates, as well as pages
render_templates(&config, &files, &nav);
render_pages(&config, &page, &nav);
// Return workspace
workspace
/// Creates a workflow for the given config.
pub fn create_workflow(config: &Config) -> Workflow<Id> {
let mut context = Context::default();
Main { config: config.clone() }
.setup(&mut context)
.expect("invariant");
context.into()
}
crates/zensical/src/workflow/cached.rs
+6 -9
View File
@@ -28,7 +28,7 @@
use serde::{Deserialize, Serialize};
use std::fs;
use std::hash::{DefaultHasher, Hash, Hasher};
use zrx::scheduler::action::report::IntoReport;
use zrx::scheduler::step::Result;
use zrx::scheduler::Value;
use crate::config::Config;
@@ -54,14 +54,11 @@ struct Cached<T> {
/// input arguments. Note that this is only a preliminary implementation, and
/// will be replaced with a more generic caching mechanism integrated into
/// the runtime.
pub fn cached<I, T, F, R, U>(
config: &Config, id: I, args: T, f: F,
) -> impl IntoReport<U>
pub fn cached<I, T, F, U>(config: &Config, id: I, args: T, f: F) -> Result<U>
where
I: Hash,
T: Hash,
F: FnOnce(T) -> R,
R: IntoReport<U>,
F: FnOnce(T) -> Result<U>,
U: Value + Serialize + for<'de> Deserialize<'de>,
{
// Compute hash of identifier
@@ -90,15 +87,15 @@ where
if let Ok(cached) = serde_json::from_slice::<Cached<U>>(&data) {
// In case content hashes match, return cached data
if cached.hash == hash {
return cached.data.into_report();
return Ok(cached.data);
}
}
}
// Compute artifact and convert into report - note that we need to properly
// handle encoding and file I/O errors here as well
f(args).into_report().inspect(|report| {
serde_json::to_string_pretty(&Cached { data: &report.data, hash })
f(args).inspect(|data| {
serde_json::to_string_pretty(&Cached { data, hash })
.map(|content| fs::write(path, content).expect("invariant"))
.expect("invariant");
})