From fc19a6cea89e2c95e8afd87ee9ae3e7684b25dba Mon Sep 17 00:00:00 2001
From: Gareth George <garethgeorge97@gmail.com>
Date: Tue, 26 Dec 2023 08:26:01 +0000
Subject: [PATCH] fix: reduce stats refresh frequency

---
 internal/orchestrator/taskstats.go | 37 +++++++++++++++++++-
 webui/src/views/RepoView.tsx       | 54 +++++++++++++++++-------------
 2 files changed, 67 insertions(+), 24 deletions(-)

diff --git a/internal/orchestrator/taskstats.go b/internal/orchestrator/taskstats.go
index 9e350764..08fd1157 100644
--- a/internal/orchestrator/taskstats.go
+++ b/internal/orchestrator/taskstats.go
@@ -7,9 +7,12 @@ import (
 	"time"
 
 	v1 "github.com/garethgeorge/backrest/gen/go/v1"
+	"github.com/garethgeorge/backrest/internal/oplog/indexutil"
 	"go.uber.org/zap"
 )
 
+var statBytesThreshold int64 = 1024 * 1024 * 1024 // 1GB
+
 // StatsTask tracks a restic stats operation.
 type StatsTask struct {
 	TaskWithOperation
@@ -18,7 +21,7 @@ type StatsTask struct {
 	at           *time.Time
 }
 
-var _ Task = &ForgetTask{}
+var _ Task = &StatsTask{}
 
 func NewOneoffStatsTask(orchestrator *Orchestrator, plan *v1.Plan, linkSnapshot string, at time.Time) *StatsTask {
 	return &StatsTask{
@@ -35,7 +38,39 @@ func (t *StatsTask) Name() string {
 	return fmt.Sprintf("stats for plan %q", t.plan.Id)
 }
 
+func (t *StatsTask) shouldRun() (bool, error) {
+	var bytesSinceLastStat int64
+	if err := t.orch.OpLog.ForEachByRepo(t.plan.Repo, indexutil.CollectLastN(50), func(op *v1.Operation) error {
+		if _, ok := op.Op.(*v1.Operation_OperationStats); ok {
+			bytesSinceLastStat = 0
+		} else if backup, ok := op.Op.(*v1.Operation_OperationBackup); ok && backup.OperationBackup.LastStatus != nil {
+			if summary, ok := backup.OperationBackup.LastStatus.Entry.(*v1.BackupProgressEntry_Summary); ok {
+				bytesSinceLastStat += summary.Summary.DataAdded
+			}
+		}
+		return nil
+	}); err != nil {
+		return false, fmt.Errorf("iterate oplog: %w", err)
+	}
+
+	zap.L().Debug("bytes since last stat", zap.Int64("bytes", bytesSinceLastStat), zap.String("repo", t.plan.Repo))
+
+	if bytesSinceLastStat > statBytesThreshold {
+		zap.S().Debugf("bytes since last stat (%v) exceeds threshold (%v)", bytesSinceLastStat, statBytesThreshold)
+		return true, nil
+	}
+	return false, nil
+}
+
 func (t *StatsTask) Next(now time.Time) *time.Time {
+	shouldRun, err := t.shouldRun()
+	if err != nil {
+		zap.S().Errorf("task %v failed to check if it should run: %v", t.Name(), err)
+	}
+	if !shouldRun {
+		return nil
+	}
+
 	ret := t.at
 	if ret != nil {
 		t.at = nil
diff --git a/webui/src/views/RepoView.tsx b/webui/src/views/RepoView.tsx
index 753cc8f0..e50dc3dd 100644
--- a/webui/src/views/RepoView.tsx
+++ b/webui/src/views/RepoView.tsx
@@ -10,22 +10,23 @@ import { MAX_OPERATION_HISTORY } from "../constants";
 import { GetOperationsRequest } from "../../gen/ts/v1/service_pb";
 import { getOperations } from "../state/oplog";
 import { RepoStats } from "../../gen/ts/v1/restic_pb";
-import { formatBytes } from "../lib/formatting";
+import { formatBytes, formatDate, formatTime } from "../lib/formatting";
+import { Operation } from "../../gen/ts/v1/operations_pb";
 
 export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
   const alertsApi = useAlertApi()!;
   const [loading, setLoading] = useState(true);
-  const [stats, setStats] = useState<RepoStats | null>(null);
+  const [statsOperation, setStatsOperation] = useState<Operation | null>(null);
 
   useEffect(() => {
     setLoading(true);
-    setStats(null);
-    getOperations(new GetOperationsRequest({ repoId: repo.id!, lastN: BigInt(10) })).then((operations) => {
+    setStatsOperation(null);
+    getOperations(new GetOperationsRequest({ repoId: repo.id!, lastN: BigInt(100) })).then((operations) => {
       for (const op of operations) {
         if (op.op.case === "operationStats") {
           const stats = op.op.value.stats;
           if (stats) {
-            setStats(stats);
+            setStatsOperation(op);
           }
         }
       }
@@ -54,24 +55,12 @@ export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
       label: "Stats",
       children: (
         <>
-          <h3>Repo Stats</h3>
-          {stats === null ? <Empty description="No data. Have you run a backup yet?" /> :
-            <Row>
-              <Col style={{ paddingRight: "20px" }}>
-                <p><strong>Total Size: </strong></p>
-                <p><strong>Total Size Uncompressed: </strong></p>
-                <p><strong>Blob Count: </strong></p>
-                <p><strong>Snapshot Count: </strong></p>
-                <p><strong>Compression Ratio: </strong></p>
-              </Col>
-              <Col>
-                <p>{formatBytes(Number(stats.totalSize))}</p>
-                <p>{formatBytes(Number(stats.totalUncompressedSize))}</p>
-                <p>{Number(stats.totalBlobCount)} blobs</p>
-                <p>{Number(stats.snapshotCount)} snapshots</p>
-                <p>{Math.round(stats.compressionRatio * 1000) / 1000}</p>
-              </Col>
-            </Row>
+          {statsOperation === null ? <Empty description="No data. Have you run a backup yet?" /> :
+            <>
+              <h3>Repo stats computed on {formatTime(Number(statsOperation.unixTimeStartMs))}</h3>
+              {statsOperation.op.case === "operationStats" && <StatsTable stats={statsOperation.op.value.stats!} />}
+              <small>Stats are refreshed periodically in the background as new data is added.</small>
+            </>
           }
         </>
       ),
@@ -116,3 +105,22 @@ export const RepoView = ({ repo }: React.PropsWithChildren<{ repo: Repo }>) => {
     </>
   );
 };
+
+const StatsTable = ({ stats }: { stats: RepoStats }) => {
+  return <Row>
+    <Col style={{ paddingRight: "20px" }}>
+      <p><strong>Total Size: </strong></p>
+      <p><strong>Total Size Uncompressed: </strong></p>
+      <p><strong>Blob Count: </strong></p>
+      <p><strong>Snapshot Count: </strong></p>
+      <p><strong>Compression Ratio: </strong></p>
+    </Col>
+    <Col>
+      <p>{formatBytes(Number(stats.totalSize))}</p>
+      <p>{formatBytes(Number(stats.totalUncompressedSize))}</p>
+      <p>{Number(stats.totalBlobCount)} blobs</p>
+      <p>{Number(stats.snapshotCount)} snapshots</p>
+      <p>{Math.round(stats.compressionRatio * 1000) / 1000}</p>
+    </Col>
+  </Row>
+}
\ No newline at end of file