feat: schedule index operations and stats refresh from repo view

internal/orchestrator/taskprune.go

@@ -40,6 +40,14 @@ func (t *PruneTask) Name() string {
 }
 
 func (t *PruneTask) Next(now time.Time) *time.Time {
+	shouldRun, err := t.shouldRun(now)
+	if err != nil {
+		zap.S().Errorf("task %v failed to check if it should run: %v", t.Name(), err)
+	}
+	if !shouldRun {
+		return nil
+	}
+
 	ret := t.at
 	if ret != nil {
 		t.at = nil
@@ -58,6 +66,24 @@ func (t *PruneTask) Next(now time.Time) *time.Time {
 	return ret
 }
 
+func (t *PruneTask) shouldRun(now time.Time) (bool, error) {
+	if t.force {
+		return true, nil
+	}
+
+	repo, err := t.orch.GetRepo(t.plan.Repo)
+	if err != nil {
+		return false, fmt.Errorf("get repo %v: %w", t.plan.Repo, err)
+	}
+
+	nextPruneTime, err := t.getNextPruneTime(repo, repo.repoConfig.PrunePolicy)
+	if err != nil {
+		return false, fmt.Errorf("get next prune time: %w", err)
+	}
+
+	return nextPruneTime.Before(now), nil
+}
+
 func (t *PruneTask) getNextPruneTime(repo *RepoOrchestrator, policy *v1.PrunePolicy) (time.Time, error) {
 	var lastPruneTime time.Time
 	t.orch.OpLog.ForEachByRepo(t.plan.Repo, indexutil.CollectLastN(100), func(op *v1.Operation) error {
@@ -86,17 +112,6 @@ func (t *PruneTask) Run(ctx context.Context) error {
 		}
 		op.Op = opPrune
 
-		if !t.force {
-			nextPruneTime, err := t.getNextPruneTime(repo, repo.repoConfig.PrunePolicy)
-			if err != nil {
-				return fmt.Errorf("get next prune time: %w", err)
-			}
-			if nextPruneTime.After(time.Now()) {
-				op.Status = v1.OperationStatus_STATUS_SYSTEM_CANCELLED
-				return nil
-			}
-		}
-
 		ctx, cancel := context.WithCancel(ctx)
 		interval := time.NewTicker(1 * time.Second)
 		defer interval.Stop()

internal/orchestrator/tasks.go

@@ -46,7 +46,7 @@ func (t *TaskWithOperation) setOperation(op *v1.Operation) error {
 
 func (t *TaskWithOperation) runWithOpAndContext(ctx context.Context, do func(ctx context.Context, op *v1.Operation) error) error {
 	if t.op == nil {
-		return errors.New("task has no operation, a call to setOperation first is required.")
+		return errors.New("task has no operation, a call to setOperation first is required")
 	}
 	if t.running.Load() {
 		return errors.New("task is already running")

internal/orchestrator/taskstats.go

@@ -11,7 +11,7 @@ import (
 	"go.uber.org/zap"
 )
 
-var statBytesThreshold int64 = 1024 * 1024 * 1024 // 1GB
+var statBytesThreshold int64 = 10 * 1024 * 1024 * 1024 // 10 GB added.
 
 // StatsTask tracks a restic stats operation.
 type StatsTask struct {

webui/src/api.ts

@@ -5,6 +5,7 @@ import { Backrest } from "../gen/ts/v1/service_connect";
 
 const transport = createConnectTransport({
   baseUrl: "/",
+  useBinaryFormat: true,
 });
 
 export const backrestService = createPromiseClient(Backrest, transport);

webui/src/components/SpinButton.tsx

@@ -0,0 +1,29 @@
+import React from "react";
+import { Button, ButtonProps } from "antd";
+import { useState } from "react";
+
+export const SpinButton: React.FC<ButtonProps & {
+  onClickAsync: () => Promise<void>;
+}> = ({ onClickAsync, ...props }) => {
+  const [loading, setLoading] = useState(false);
+
+  const onClick = async () => {
+    if (loading) {
+      return;
+    }
+    try {
+      setLoading(true);
+      await onClickAsync();
+    } finally {
+      setLoading(false);
+    }
+  };
+
+  return (
+    <Button
+      {...props}
+      loading={loading}
+      onClick={onClick}
+    />
+  );
+}

webui/src/constants.ts

@@ -1 +1,3 @@
 export const MAX_OPERATION_HISTORY = 10000;
+export const STATUS_OPERATION_HISTORY = 10; // number of operations to load when determining plan / repo status.
+export const STATS_OPERATION_HISTORY = 100; // number of operations to load when searching for stats for a repo.

webui/src/state/oplog.ts

@@ -9,6 +9,7 @@ import { BackupProgressEntry, ResticSnapshot } from "../../gen/ts/v1/restic_pb";
 import _ from "lodash";
 import { formatDuration, formatTime } from "../lib/formatting";
 import { backrestService } from "../api";
+import { STATS_OPERATION_HISTORY } from "../constants";
 
 const subscribers: ((event: OperationEvent) => void)[] = [];
 
@@ -56,7 +57,7 @@ export const unsubscribeFromOperations = (
 export const getStatusForPlan = async (plan: string) => {
   const req = new GetOperationsRequest({
     planId: plan,
-    lastN: BigInt(8),
+    lastN: BigInt(STATS_OPERATION_HISTORY),
   });
   return await getStatus(req);
 }
@@ -64,7 +65,7 @@ export const getStatusForPlan = async (plan: string) => {
 export const getStatusForRepo = async (repo: string) => {
   const req = new GetOperationsRequest({
     repoId: repo,
-    lastN: BigInt(8),
+    lastN: BigInt(STATS_OPERATION_HISTORY),
   });
   return await getStatus(req);
 }

webui/src/views/PlanView.tsx

@@ -9,6 +9,7 @@ import { OperationTree } from "../components/OperationTree";
 import { MAX_OPERATION_HISTORY } from "../constants";
 import { backrestService } from "../api";
 import { GetOperationsRequest } from "../../gen/ts/v1/service_pb";
+import { SpinButton } from "../components/SpinButton";
 
 export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
   const alertsApi = useAlertApi()!;
@@ -30,29 +31,29 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
     }
   };
 
-  const handlePruneNow = () => {
+  const handlePruneNow = async () => {
     try {
-      backrestService.prune({ value: plan.id });
+      await backrestService.prune({ value: plan.id });
       alertsApi.success("Prune scheduled.");
     } catch (e: any) {
       alertsApi.error("Failed to schedule prune: " + e.message);
     }
   };
 
-  const handleUnlockNow = () => {
+  const handleUnlockNow = async () => {
     try {
       alertsApi.info("Unlocking repo...");
-      backrestService.unlock({ value: plan.repo! });
+      await backrestService.unlock({ value: plan.repo! });
       alertsApi.success("Repo unlocked.");
     } catch (e: any) {
       alertsApi.error("Failed to unlock repo: " + e.message);
     }
   };
 
-  const handleClearErrorHistory = () => {
+  const handleClearErrorHistory = async () => {
     try {
       alertsApi.info("Clearing error history...");
-      backrestService.clearHistory({ planId: plan.id, onlyFailed: true });
+      await backrestService.clearHistory({ planId: plan.id, onlyFailed: true });
       alertsApi.success("Error history cleared.");
     } catch (e: any) {
       alertsApi.error("Failed to clear error history: " + e.message);
@@ -67,23 +68,23 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
         </Typography.Title>
       </Flex>
       <Flex gap="small" align="center" wrap="wrap">
-        <Button type="primary" onClick={handleBackupNow}>
+        <SpinButton type="primary" onClickAsync={handleBackupNow}>
           Backup Now
-        </Button>
+        </SpinButton>
         <Tooltip title="Runs a prune operation on the repository that will remove old snapshots and free up space">
-          <Button type="default" onClick={handlePruneNow}>
+          <SpinButton type="default" onClickAsync={handlePruneNow}>
             Prune Now
-          </Button>
+          </SpinButton>
         </Tooltip>
         <Tooltip title="Removes lockfiles and checks the repository for errors. Only run if you are sure the repo is not being accessed by another system">
-          <Button type="default" onClick={handleUnlockNow}>
+          <SpinButton type="default" onClickAsync={handleUnlockNow}>
             Unlock Repo
-          </Button>
+          </SpinButton>
         </Tooltip>
         <Tooltip title="Removes failed operations from the list">
-          <Button type="default" onClick={handleClearErrorHistory}>
+          <SpinButton type="default" onClickAsync={handleClearErrorHistory}>
             Clear Error History
-          </Button>
+          </SpinButton>
         </Tooltip>
       </Flex>
       <Tabs

webui/src/views/RepoView.tsx

@@ -1,17 +1,20 @@
 import React, { useEffect, useState } from "react";
 import { Repo } from "../../gen/ts/v1/config_pb";
-import { Col, Empty, Flex, Row, Spin, Tabs, Typography } from "antd";
+import { Col, Empty, Flex, Row, Spin, TabsProps, Tabs, Tooltip, Typography } from "antd";
 import { useRecoilValue } from "recoil";
 import { configState } from "../state/config";
 import { useAlertApi } from "../components/Alerts";
 import { OperationList } from "../components/OperationList";
 import { OperationTree } from "../components/OperationTree";
-import { MAX_OPERATION_HISTORY } from "../constants";
+import { MAX_OPERATION_HISTORY, STATS_OPERATION_HISTORY } from "../constants";
 import { GetOperationsRequest } from "../../gen/ts/v1/service_pb";
 import { getOperations } from "../state/oplog";
 import { RepoStats } from "../../gen/ts/v1/restic_pb";
-import { formatBytes, formatDate, formatTime } from "../lib/formatting";
+import { formatBytes, formatTime } from "../lib/formatting";
 import { Operation } from "../../gen/ts/v1/operations_pb";
+import { backrestService } from "../api";
+import { StringValue } from "@bufbuild/protobuf";
+import { SpinButton } from "../components/SpinButton";
 
 export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
   const alertsApi = useAlertApi()!;
@@ -21,7 +24,7 @@ export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
   useEffect(() => {
     setLoading(true);
     setStatsOperation(null);
-    getOperations(new GetOperationsRequest({ repoId: repo.id!, lastN: BigInt(100) })).then((operations) => {
+    getOperations(new GetOperationsRequest({ repoId: repo.id!, lastN: BigInt(STATS_OPERATION_HISTORY) })).then((operations) => {
       for (const op of operations) {
         if (op.op.case === "operationStats") {
           const stats = op.op.value.stats;
@@ -37,6 +40,11 @@ export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
     });
   }, [repo.id]);
 
+  // Task handlers
+  const handleIndexNow = async () => {
+    await backrestService.indexSnapshots(new StringValue({ value: repo.id! }));
+  }
+
   // Gracefully handle deletions by checking if the plan is still in the config.
   const config = useRecoilValue(configState);
   let repoInConfig = config.repos?.find((p) => p.id === repo.id);
@@ -64,6 +72,7 @@ export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
           }
         </>
       ),
+      destroyInactiveTabPane: true,
     },
     {
       key: "2",
@@ -76,6 +85,7 @@ export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
           />
         </>
       ),
+      destroyInactiveTabPane: true,
     },
     {
       key: "3",
@@ -89,6 +99,7 @@ export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
           />
         </>
       ),
+      destroyInactiveTabPane: true,
     },
   ]
   return (
@@ -98,6 +109,18 @@ export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
           {repo.id}
         </Typography.Title>
       </Flex>
+      <Flex gap="small" align="center" wrap="wrap">
+        <Tooltip title="Indexes the snapshots in the repository. Snapshots are also indexed automatically after each backup.">
+          <SpinButton type="default" onClickAsync={handleIndexNow}>
+            Index Snapshots
+          </SpinButton>
+        </Tooltip>
+        <Tooltip title="Computes stats for the repository. May take some time to refresh.">
+          <SpinButton type="default" onClickAsync={handleIndexNow}>
+            Index Snapshots
+          </SpinButton>
+        </Tooltip>
+      </Flex>
       <Tabs
         defaultActiveKey={items[0].key}
         items={items}