fix: miscellaneous bug fixes

* Fixes a problem with incorrectly scanning and removing pending events
   from the operation log for new installations
 * Fixes a bug with the operation tree incorrectly applying query
   selectors when filtering events
 * Updates tooltips and comments in PlanView and GettingStartedGuide

docs/.prettierignore

@@ -0,0 +1 @@
+*

docs/content/0.index.md

@@ -8,16 +8,14 @@ main:
 
 :ellipsis{right=0px width=75% blur=150px}
 
-## ::block-hero
-
+::block-hero
+---
 cta:
-
-- Get started
-- /introduction/getting-started
-  secondary:
-- Open on GitHub →
-- https://github.com/garethgeorge/backrest
-
+  - Get started
+  - /introduction/getting-started
+secondary:
+  - Open on GitHub →
+  - https://github.com/garethgeorge/backrest
 ---
 
 #title
@@ -27,32 +25,29 @@ Web UI and orchestrator for [Restic](https://restic.net) backup.
 
 Backrest is a web-accessible backup solution built on top of [restic](https://restic.net/) and providing a WebUI which wraps the restic CLI and makes it easy to create repos, browse snapshots, and restore files. Additionally, Backrest can run in the background and take an opinionated approach to scheduling snapshots and orchestrating repo health operations.
 
-#extra
-::list
 
-- Import your existing restic repositories
-- Cron scheduled backups and health operations (e.g. prune and forget)
-- UI for browing and restoring files from snapshots
-- Configurable backup notifications (e.g. Discord, Slack, Shoutrrr, Gotify)
-- Add shell command hooks to run before and after backup operations.
-- Compatible with rclone remotes
-- Cross-platform support (Linux, MacOS, Windows, FreeBSD, [Docker](https://hub.docker.com/r/garethgeorge/backrest))
-- Backup to any restic supported storage (e.g. S3, B2, Azure, GCS, local, SFTP, and all [rclone remotes](https://rclone.org/))
+#extra
+  ::list
+  - Import your existing restic repositories
+  - Cron scheduled backups and health operations (e.g. prune and forget)
+  - UI for browing and restoring files from snapshots
+  - Configurable backup notifications (e.g. Discord, Slack, Shoutrrr, Gotify)
+  - Add shell command hooks to run before and after backup operations.
+  - Compatible with rclone remotes
+  - Cross-platform support (Linux, MacOS, Windows, FreeBSD, [Docker](https://hub.docker.com/r/garethgeorge/backrest))
+  - Backup to any restic supported storage (e.g. S3, B2, Azure, GCS, local, SFTP, and all [rclone remotes](https://rclone.org/))
   ::
 
 #support
 ::code-group
-
 ```bash [MacOS]
 brew tap garethgeorge/homebrew-backrest-tap
 brew install backrest
 ```
-
 ```bash [Arch Linux]
 paru -Sy backrest
 sudo systemctl enable --now backreset@$USER.service
 ```
-
 ```yaml [docker-compose]
 version: "3.2"
 services:
@@ -74,6 +69,5 @@ services:
     ports:
       - 9898:9898
 ```
-
-::
 ::
+::

docs/package.json

@@ -10,10 +10,10 @@
     "lint": "eslint ."
   },
   "devDependencies": {
-    "@nuxt-themes/docus": "^1.15.0",
+    "@nuxt-themes/docus": "^1.14.8",
     "@nuxt/devtools": "^1.3.1",
-    "@nuxt/eslint-config": "^0.2.0",
-    "@nuxtjs/plausible": "^0.2.4",
+    "@nuxt/eslint-config": "^0.3.13",
+    "@nuxtjs/plausible": "^1.0.0",
     "@types/node": "^20.12.12",
     "eslint": "^8.57.0",
     "nuxt": "^3.11.2"

internal/config/validationutil/validationutil.go

@@ -27,8 +27,8 @@ func SanitizeID(id string) string {
 // It returns an error if the ID contains invalid characters, is empty, or is too long.
 // The maxLen parameter is the maximum length of the ID. If maxLen is 0, the ID length is not checked.
 func ValidateID(id string, maxLen int) error {
-	if strings.HasPrefix(id, "__") && strings.HasSuffix(id, "__") {
-		return errors.New("IDs starting and ending with '__' are reserved by backrest")
+	if strings.HasPrefix(id, "_") && strings.HasSuffix(id, "_") {
+		return errors.New("IDs starting and ending with '_' are reserved by backrest")
 	}
 	if !idRegex.MatchString(id) {
 		return ErrInvalidChars

internal/ioutil/ioutil.go

@@ -112,14 +112,14 @@ func (w *OutputCapturer) Bytes() []byte {
 }
 
 type SynchronizedWriter struct {
-	mu sync.Mutex
+	Mu sync.Mutex
 	W  io.Writer
 }
 
 var _ io.Writer = &SynchronizedWriter{}
 
 func (w *SynchronizedWriter) Write(p []byte) (n int, err error) {
-	w.mu.Lock()
-	defer w.mu.Unlock()
+	w.Mu.Lock()
+	defer w.Mu.Unlock()
 	return w.W.Write(p)
 }

internal/oplog/migrations.go

@@ -14,6 +14,7 @@ import (
 var migrations = []func(*OpLog, *bbolt.Tx) error{
 	migration001FlowID,
 	migration002InstanceID,
+	migration003ResetLastValidated,
 }
 
 var CurrentVersion = int64(len(migrations))
@@ -116,7 +117,11 @@ func migration002InstanceID(oplog *OpLog, tx *bbolt.Tx) error {
 			return nil
 		}
 
-		op.InstanceId = "__unassociated__"
+		op.InstanceId = "_unassociated_"
 		return nil
 	})
 }
+
+func migration003ResetLastValidated(oplog *OpLog, tx *bbolt.Tx) error {
+	return tx.Bucket(SystemBucket).Delete([]byte("last_validated"))
+}

internal/oplog/oplog.go

@@ -95,6 +95,8 @@ func (o *OpLog) Scan(onIncomplete func(op *v1.Operation)) error {
 		var k, v []byte
 		if lastValidated := sysBucket.Get([]byte("last_validated")); lastValidated != nil {
 			k, v = c.Seek(lastValidated)
+		} else {
+			k, v = c.First()
 		}
 		for ; k != nil; k, v = c.Next() {
 			op := &v1.Operation{}

internal/orchestrator/repo/repo.go

@@ -42,7 +42,6 @@ func NewRepoOrchestrator(config *v1.Config, repoConfig *v1.Repo, resticPath stri
 	}
 
 	opts = append(opts, restic.WithEnviron())
-	opts = append(opts, restic.WithEnv("RESTIC_PROGRESS_FPS=2"))
 
 	if env := repoConfig.GetEnv(); len(env) != 0 {
 		for _, e := range env {

internal/orchestrator/tasks/taskprune.go

@@ -9,6 +9,7 @@ import (
 
 	v1 "github.com/garethgeorge/backrest/gen/go/v1"
 	"github.com/garethgeorge/backrest/internal/hook"
+	"github.com/garethgeorge/backrest/internal/ioutil"
 	"github.com/garethgeorge/backrest/internal/oplog"
 	"github.com/garethgeorge/backrest/internal/oplog/indexutil"
 	"go.uber.org/zap"
@@ -106,16 +107,18 @@ func (t *PruneTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner
 	ctx, cancel := context.WithCancel(ctx)
 	interval := time.NewTicker(1 * time.Second)
 	defer interval.Stop()
-	var buf synchronizedBuffer
+	var buf bytes.Buffer
+	bufWriter := ioutil.SynchronizedWriter{W: &buf}
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
-
 		defer wg.Done()
 		for {
 			select {
 			case <-interval.C:
+				bufWriter.Mu.Lock()
 				output := buf.String()
+				bufWriter.Mu.Unlock()
 				if len(output) > 8*1024 { // only provide live status upto the first 8K of output.
 					output = output[:len(output)-8*1024]
 				}
@@ -133,7 +136,7 @@ func (t *PruneTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner
 		}
 	}()
 
-	if err := repo.Prune(ctx, &buf); err != nil {
+	if err := repo.Prune(ctx, &bufWriter); err != nil {
 		cancel()
 
 		runner.ExecuteHooks([]v1.Hook_Condition{
@@ -147,7 +150,6 @@ func (t *PruneTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner
 	cancel()
 	wg.Wait()
 
-	// TODO: it would be best to store the output in separate storage for large status data.
 	output := buf.String()
 	if len(output) > 8*1024 { // only save the first 4K of output.
 		output = output[:len(output)-8*1024]
@@ -157,23 +159,3 @@ func (t *PruneTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner
 
 	return nil
 }
-
-// synchronizedBuffer is used for collecting prune command's output
-type synchronizedBuffer struct {
-	mu  sync.Mutex
-	buf bytes.Buffer
-}
-
-func (w *synchronizedBuffer) Write(p []byte) (n int, err error) {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-
-	return w.buf.Write(p)
-}
-
-func (w *synchronizedBuffer) String() string {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-
-	return w.buf.String()
-}

pkg/restic/restic.go

@@ -123,6 +123,8 @@ func (r *Repo) Backup(ctx context.Context, paths []string, progressCallback func
 	args := []string{"backup", "--json", "--exclude-caches"}
 	args = append(args, paths...)
 
+	opts = append(slices.Clone(opts), WithEnv("RESTIC_PROGRESS_FPS=2"))
+
 	cmd := r.commandWithContext(ctx, args, opts...)
 	outputForErr := ioutil.NewOutputCapturer(outputBufferLimit)
 	buf := buffer.New(32 * 1024) // 32KB IO buffer for the realtime event parsing
@@ -234,6 +236,7 @@ func (r *Repo) Prune(ctx context.Context, pruneOutput io.Writer, opts ...Generic
 }
 
 func (r *Repo) Restore(ctx context.Context, snapshot string, callback func(*RestoreProgressEntry), opts ...GenericOption) (*RestoreProgressEntry, error) {
+	opts = append(slices.Clone(opts), WithEnv("RESTIC_PROGRESS_FPS=2"))
 	cmd := r.commandWithContext(ctx, []string{"restore", "--json", snapshot}, opts...)
 	buf := buffer.New(32 * 1024) // 32KB IO buffer for the realtime event parsing
 	reader, writer := nio.Pipe(buf)

webui/src/components/OperationRow.tsx

@@ -232,7 +232,7 @@ export const OperationRow = ({
     );
   } else if (operation.op.case === "operationRestore") {
     const restore = operation.op.value;
-    const progress = Math.round((details.percentage || 0) * 1000) / 10;
+    const progress = Math.round((details.percentage || 0) * 10) / 10;
     const st = restore.status! || {};
 
     body = (

webui/src/components/OperationTree.tsx

@@ -6,6 +6,7 @@ import {
   displayTypeToString,
   getOperations,
   getTypeForDisplay,
+  matchSelector,
   shouldHideOperation,
   subscribeToOperations,
   unsubscribeFromOperations,
@@ -61,10 +62,11 @@ export const OperationTree = ({
     setSelectedBackupId(null);
     const backupCollector = new BackupInfoCollector();
     const lis = (opEvent: OperationEvent) => {
-      if (!!req.planId && opEvent.operation!.planId !== req.planId) {
-        return;
-      }
-      if (!!req.repoId && opEvent.operation!.repoId !== req.repoId) {
+      if (
+        !req.selector ||
+        !opEvent.operation ||
+        !matchSelector(req.selector, opEvent.operation)
+      ) {
         return;
       }
       if (opEvent.type !== OperationEventType.EVENT_DELETED) {
@@ -82,7 +84,7 @@ export const OperationTree = ({
           return b.startTimeMs - a.startTimeMs;
         });
         setBackups(backups);
-      }, 50),
+      }, 50)
     );
 
     getOperations(req)
@@ -141,7 +143,7 @@ export const OperationTree = ({
               }}
             >
               <BackupView backup={backup} />
-            </Modal>,
+            </Modal>
           );
         }
       }}
@@ -170,19 +172,21 @@ export const OperationTree = ({
             if (b.backupLastStatus.entry.case === "summary") {
               const s = b.backupLastStatus.entry.value;
               details.push(
-                `${formatBytes(Number(s.totalBytesProcessed))} in ${formatDuration(
-                  s.totalDuration! * 1000.0, // convert to ms
-                )}`,
+                `${formatBytes(
+                  Number(s.totalBytesProcessed)
+                )} in ${formatDuration(
+                  s.totalDuration! * 1000.0 // convert to ms
+                )}`
               );
             } else if (b.backupLastStatus.entry.case === "status") {
               const s = b.backupLastStatus.entry.value;
               const percent = Math.floor(
-                (Number(s.bytesDone) / Number(s.totalBytes)) * 100,
+                (Number(s.bytesDone) / Number(s.totalBytes)) * 100
               );
               details.push(
                 `${percent}% processed ${formatBytes(
-                  Number(s.bytesDone),
-                )} / ${formatBytes(Number(s.totalBytes))}`,
+                  Number(s.bytesDone)
+                )} / ${formatBytes(Number(s.totalBytes))}`
               );
             }
           }
@@ -251,7 +255,7 @@ const buildTreePlan = (operations: BackupInfo[]): OpTreeNode[] => {
 
 const buildTreeDay = (
   keyPrefix: string,
-  operations: BackupInfo[],
+  operations: BackupInfo[]
 ): OpTreeNode[] => {
   const grouped = _.groupBy(operations, (op) => {
     return localISOTime(op.displayTime).substring(0, 10);
@@ -312,7 +316,7 @@ const BackupView = ({ backup }: { backup?: BackupInfo }) => {
             planId: backup.planId!,
             repoId: backup.repoId!,
             snapshotId: backup.snapshotId!,
-          }),
+          })
         );
         alertApi!.success("Snapshot forgotten.");
       } catch (e) {
@@ -339,7 +343,7 @@ const BackupView = ({ backup }: { backup?: BackupInfo }) => {
           backrestService.clearHistory(
             new ClearHistoryRequest({
               ops: backup.operations.map((op) => op.id),
-            }),
+            })
           );
         }}
       >

webui/src/state/oplog.ts

@@ -481,3 +481,17 @@ export const detailsForOperation = (
     color,
   };
 };
+
+export const matchSelector = (selector: OpSelector, op: Operation) => {
+  if (selector.planId && selector.planId !== op.planId) {
+    return false;
+  }
+  if (selector.repoId && selector.repoId !== op.repoId) {
+    return false;
+  }
+  if (selector.flowId && selector.flowId !== op.flowId) {
+    return false;
+  }
+  return true;
+}
+

webui/src/views/GettingStartedGuide.tsx

@@ -3,6 +3,7 @@ import React, { useEffect, useState } from "react";
 import { backrestService } from "../api";
 import { useConfig } from "../components/ConfigProvider";
 import { Config } from "../../gen/ts/v1/config_pb";
+import { isDevBuild } from "../state/buildcfg";
 
 export const GettingStartedGuide = () => {
   const config = useConfig()[0];
@@ -55,28 +56,37 @@ export const GettingStartedGuide = () => {
             your config (or minimally a copy of your passwords) in a safe
             location e.g. a secure note in your password manager.
           </li>
+          <li>
+            Configure hooks: Backrest can deliver notifications about backup
+            events. It's strongly recommended that you configure an on error
+            hook that will notify you in the event that backups start failing
+            (e.g. an issue with storage or network connectivity). Hooks can be
+            configured either at the plan or repo level.
+          </li>
         </ul>
         <Divider orientation="left">Config View</Divider>
-        <Collapse
-          size="small"
-          items={[
-            {
-              key: "1",
-              label: "Config JSON hidden for security",
-              children: config ? (
-                <Typography>
-                  <pre>
-                    {config.toJsonString({
-                      prettySpaces: 2,
-                    })}
-                  </pre>
-                </Typography>
-              ) : (
-                <Spin />
-              ),
-            },
-          ]}
-        />
+        {isDevBuild && (
+          <Collapse
+            size="small"
+            items={[
+              {
+                key: "1",
+                label: "Config JSON hidden for security",
+                children: config ? (
+                  <Typography>
+                    <pre>
+                      {config.toJsonString({
+                        prettySpaces: 2,
+                      })}
+                    </pre>
+                  </Typography>
+                ) : (
+                  <Spin />
+                ),
+              },
+            ]}
+          />
+        )}
       </Typography.Text>
     </>
   );

webui/src/views/PlanView.tsx

@@ -46,7 +46,10 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
   const handleClearErrorHistory = async () => {
     try {
       alertsApi.info("Clearing error history...");
-      await backrestService.clearHistory({ planId: plan.id, onlyFailed: true });
+      await backrestService.clearHistory({
+        selector: new OpSelector({ planId: plan.id, repoId: plan.repo }),
+        onlyFailed: true,
+      });
       alertsApi.success("Error history cleared.");
     } catch (e: any) {
       alertsApi.error("Failed to clear error history: " + e.message);
@@ -62,6 +65,17 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
         <SpinButton type="primary" onClickAsync={handleBackupNow}>
           Backup Now
         </SpinButton>
+        <Tooltip title="Advanced users: open a restic shell to run commands on the repository. Re-index snapshots to reflect any changes in Backrest.">
+          <Button
+            type="default"
+            onClick={async () => {
+              const { RunCommandModal } = await import("./RunCommandModal");
+              showModal(<RunCommandModal repoId={plan.repo!} />);
+            }}
+          >
+            Run Command
+          </Button>
+        </Tooltip>
         <Tooltip title="Runs a prune operation on the repository that will remove old snapshots and free up space">
           <SpinButton type="default" onClickAsync={handlePruneNow}>
             Prune Now
@@ -77,17 +91,6 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
             Clear Error History
           </SpinButton>
         </Tooltip>
-        <Tooltip title="Open a restic shell to run commands on the repository.">
-          <Button
-            type="default"
-            onClick={async () => {
-              const { RunCommandModal } = await import("./RunCommandModal");
-              showModal(<RunCommandModal repoId={plan.repo!} />);
-            }}
-          >
-            Run Command
-          </Button>
-        </Tooltip>
       </Flex>
       <Tabs
         defaultActiveKey="1"

webui/src/views/RepoView.tsx

@@ -121,6 +121,18 @@ export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
         <Typography.Title>{repo.id}</Typography.Title>
       </Flex>
       <Flex gap="small" align="center" wrap="wrap">
+        <Tooltip title="Advanced users: open a restic shell to run commands on the repository. Re-index snapshots to reflect any changes in Backrest.">
+          <Button
+            type="default"
+            onClick={async () => {
+              const { RunCommandModal } = await import("./RunCommandModal");
+              showModal(<RunCommandModal repoId={repo.id!} />);
+            }}
+          >
+            Run Command
+          </Button>
+        </Tooltip>
+
         <Tooltip title="Indexes the snapshots in the repository. Snapshots are also indexed automatically after each backup.">
           <SpinButton type="default" onClickAsync={handleIndexNow}>
             Index Snapshots
@@ -132,17 +144,6 @@ export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
             Compute Stats
           </SpinButton>
         </Tooltip>
-        <Tooltip title="Open a restic shell to run commands on the repository.">
-          <Button
-            type="default"
-            onClick={async () => {
-              const { RunCommandModal } = await import("./RunCommandModal");
-              showModal(<RunCommandModal repoId={repo.id!} />);
-            }}
-          >
-            Run Command
-          </Button>
-        </Tooltip>
       </Flex>
       <Tabs defaultActiveKey={items[0].key} items={items} />
     </>