created basic tree layout

gen/go/v1/operations.pb.go

@@ -177,14 +177,21 @@ type Operation struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Id              int64           `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
+	// required, primary ID of the operation.
-	RepoId          string          `protobuf:"bytes,2,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"`             // repo id if associated with a repo (always true)
+	Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
-	PlanId          string          `protobuf:"bytes,3,opt,name=plan_id,json=planId,proto3" json:"plan_id,omitempty"`             // plan id if associated with a plan (always true)
+	// required, repo id if associated with a repo
-	SnapshotId      string          `protobuf:"bytes,8,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` // snapshot id if associated with a snapshot.
+	RepoId string `protobuf:"bytes,2,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"`
-	Status          OperationStatus `protobuf:"varint,4,opt,name=status,proto3,enum=v1.OperationStatus" json:"status,omitempty"`
+	// required, plan id if associated with a plan
-	UnixTimeStartMs int64           `protobuf:"varint,5,opt,name=unix_time_start_ms,json=unixTimeStartMs,proto3" json:"unix_time_start_ms,omitempty"`
+	PlanId string `protobuf:"bytes,3,opt,name=plan_id,json=planId,proto3" json:"plan_id,omitempty"`
-	UnixTimeEndMs   int64           `protobuf:"varint,6,opt,name=unix_time_end_ms,json=unixTimeEndMs,proto3" json:"unix_time_end_ms,omitempty"`
+	// optional snapshot id if associated with a snapshot.
-	DisplayMessage  string          `protobuf:"bytes,7,opt,name=display_message,json=displayMessage,proto3" json:"display_message,omitempty"` // human readable context message (if any)
+	SnapshotId string          `protobuf:"bytes,8,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"`
+	Status     OperationStatus `protobuf:"varint,4,opt,name=status,proto3,enum=v1.OperationStatus" json:"status,omitempty"`
+	// required, unix time in milliseconds of the operation's creation (ID is derived from this)
+	UnixTimeStartMs int64 `protobuf:"varint,5,opt,name=unix_time_start_ms,json=unixTimeStartMs,proto3" json:"unix_time_start_ms,omitempty"`
+	// optional, unix time in milliseconds of the operation's completion
+	UnixTimeEndMs int64 `protobuf:"varint,6,opt,name=unix_time_end_ms,json=unixTimeEndMs,proto3" json:"unix_time_end_ms,omitempty"`
+	// optional, human readable context message, typically an error message.
+	DisplayMessage string `protobuf:"bytes,7,opt,name=display_message,json=displayMessage,proto3" json:"display_message,omitempty"`
 	// Types that are assignable to Op:
 	//
 	//	*Operation_OperationBackup

internal/oplog/oplog.go

@@ -195,17 +195,25 @@ func (o *OpLog) getOperationHelper(b *bolt.Bucket, id int64) (*v1.Operation, err
 	return &op, nil
 }
 
+func (o *OpLog) nextOperationId(b *bolt.Bucket, unixTimeMs int64) (int64, error) {
+	seq, err := b.NextSequence()
+	if err != nil {
+		return 0, fmt.Errorf("next sequence: %w", err)
+	}
+	return int64(unixTimeMs<<20) | int64(seq&((1<<20)-1)), nil
+}
+
 func (o *OpLog) addOperationHelper(tx *bolt.Tx, op *v1.Operation) error {
 	b := tx.Bucket(OpLogBucket)
 	if op.Id == 0 {
-		seq, err := b.NextSequence()
-		if err != nil {
-			return fmt.Errorf("error getting next sequence: %w", err)
-		}
 		if op.UnixTimeStartMs == 0 {
 			return fmt.Errorf("operation must have a start time")
 		}
-		op.Id = op.UnixTimeStartMs<<20 | int64(seq&(1<<20-1))
+		var err error
+		op.Id, err = o.nextOperationId(b, op.UnixTimeStartMs)
+		if err != nil {
+			return fmt.Errorf("create next operation ID: %w", err)
+		}
 	}
 
 	op.SnapshotId = NormalizeSnapshotId(op.SnapshotId)

internal/orchestrator/tasks.go

@@ -47,21 +47,6 @@ func (t *ScheduledBackupTask) Name() string {
 }
 
 func (t *ScheduledBackupTask) Next(now time.Time) *time.Time {
-	if ops, err := t.orchestrator.OpLog.GetByPlan(t.plan.Id, indexutil.CollectLastN(10)); err == nil {
-		var lastBackupOp *v1.Operation
-		for _, op := range ops {
-			if _, ok := op.Op.(*v1.Operation_OperationBackup); ok {
-				lastBackupOp = op
-			}
-		}
-
-		if lastBackupOp != nil {
-			now = time.Unix(0, lastBackupOp.UnixTimeStartMs*int64(time.Millisecond))
-		}
-	} else {
-		zap.S().Errorf("error getting last operation for plan %q when computing backup schedule: %v", t.plan.Id, err)
-	}
-
 	next := t.schedule.Next(now)
 	return &next
 }
@@ -116,7 +101,7 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan
 	startTime := time.Now()
 
 	err := WithOperation(orchestrator.OpLog, op, func() error {
-		zap.L().Info("Starting backup", zap.String("plan", plan.Id))
+		zap.L().Info("Starting backup", zap.String("plan", plan.Id), zap.Int64("opId", op.Id))
 		repo, err := orchestrator.GetRepo(plan.Repo)
 		if err != nil {
 			return fmt.Errorf("couldn't get repo %q: %w", plan.Repo, err)
@@ -230,7 +215,7 @@ func WithOperation(oplog *oplog.OpLog, op *v1.Operation, do func() error) error
 	if op.Status == v1.OperationStatus_STATUS_INPROGRESS {
 		op.Status = v1.OperationStatus_STATUS_SUCCESS
 	}
-	if e := oplog.Update(op); err != nil {
+	if e := oplog.Update(op); e != nil {
 		return multierror.Append(err, fmt.Errorf("failed to update operation in oplog: %w", e))
 	}
 	return err

proto/v1/operations.proto

@@ -11,18 +11,20 @@ message OperationList {
 }
 
 message Operation {
+  // required, primary ID of the operation.
   int64 id = 1;
-  // repo id if associated with a repo (always true)
+  // required, repo id if associated with a repo
   string repo_id = 2; 
-  // plan id if associated with a plan (always true)
+  // required, plan id if associated with a plan
   string plan_id = 3; 
-  // snapshot id if associated with a snapshot.
+  // optional snapshot id if associated with a snapshot.
   string snapshot_id = 8; 
   OperationStatus status = 4;
-  // unix time in milliseconds of the operation's creation (ID is derived from this)
+  // required, unix time in milliseconds of the operation's creation (ID is derived from this)
   int64 unix_time_start_ms = 5; 
+  // optional, unix time in milliseconds of the operation's completion
   int64 unix_time_end_ms = 6;
-  // human readable context message, typically an error message.
+  // optional, human readable context message, typically an error message.
   string display_message = 7; 
 
   oneof op {

webui/src/components/OperationList.tsx

@@ -28,7 +28,7 @@ import {
 export const OperationList = ({
   operations,
 }: React.PropsWithoutRef<{ operations: EOperation[] }>) => {
-  operations.sort((a, b) => b.parsedTime - a.parsedTime);
+  operations = [...operations].reverse();
 
   if (operations.length === 0) {
     return (

webui/src/components/OperationTree.tsx

@@ -8,30 +8,78 @@ import { formatDate, formatTime } from "../lib/formatting";
 export const OperationTree = ({
   operations,
 }: React.PropsWithoutRef<{ operations: EOperation[] }>) => {
-  operations.sort((a, b) => b.parsedTime - a.parsedTime);
+  operations = [...operations].reverse(); // reverse such that newest operations are at index 0.
-  return <Tree treeData={buildTree(operations)}></Tree>;
+
+  const treeData = buildTreeYear(operations);
+  const keys = buildDefaultExpandedKeys(treeData);
+
+  return <Tree treeData={treeData} defaultExpandedKeys={keys}></Tree>;
 };
 
-// TODO: more work on this view
+const buildDefaultExpandedKeys = (tree?: DataNode[]) => {
-const buildTree = (operations: EOperation[]): DataNode[] => {
+  const keys: string[] = [];
+  while (tree && tree.length > 0) {
+    const node = tree[0];
+    keys.push(node.key as string);
+    tree = node.children!;
+  }
+  return keys;
+};
+
+const buildTreeYear = (operations: EOperation[]): DataNode[] => {
   const grouped = _.groupBy(operations, (op) => {
-    return new Date(op.parsedTime).toLocaleDateString("default", {
+    return op.parsedDate.getFullYear();
-      month: "long",
-      year: "numeric",
-      day: "numeric",
-    });
   });
 
-  return _.keys(grouped).map((key) => {
+  const entries: DataNode[] = _.map(grouped, (value, key) => {
     return {
-      key: key,
+      key: "y" + key,
-      title: key,
+      title: "" + key,
-      children: grouped[key].map((op) => {
+      children: buildTreeMonth(value),
-        return {
+    };
-          key: op.id!,
+  });
-          title: <span>{formatTime(op.parsedTime)} - AN OPERATION</span>,
+  return entries;
-        };
+};
+
+const buildTreeMonth = (operations: EOperation[]): DataNode[] => {
+  const grouped = _.groupBy(operations, (op) => {
+    return op.parsedDate.getMonth();
+  });
+  const entries: DataNode[] = _.map(grouped, (value, key) => {
+    return {
+      key: "m" + key,
+      title: value[0].parsedDate.toLocaleString("default", {
+        month: "long",
+        year: "numeric",
       }),
+      children: buildTreeDay(value),
+    };
+  });
+  return entries;
+};
+
+const buildTreeDay = (operations: EOperation[]): DataNode[] => {
+  const grouped = _.groupBy(operations, (op) => {
+    console.log("Operation date: " + formatDate(op.parsedTime));
+    return formatDate(op.parsedTime);
+  });
+
+  const entries = _.map(grouped, (value, key) => {
+    return {
+      key: "d" + key,
+      title: formatDate(value[0].parsedTime),
+      children: buildTreeLeaf(value),
+    };
+  });
+  return entries;
+};
+
+const buildTreeLeaf = (operations: EOperation[]): DataNode[] => {
+  return _.map(operations, (op) => {
+    return {
+      key: op.id!,
+      title: formatTime(op.parsedDate) + " - ",
+      isLeaf: true,
     };
   });
 };

webui/src/constants.ts

@@ -0,0 +1,2 @@
+export const API_PREFIX = "/api";
+export const MAX_OPERATION_HISTORY = 10000;

webui/src/lib/formatting.ts

@@ -17,9 +17,11 @@ export const formatBytes = (bytes?: number | string) => {
 
 const timezoneOffsetMs = new Date().getTimezoneOffset() * 60 * 1000;
 // formatTime formats a time as YYYY-MM-DD at HH:MM AM/PM
-export const formatTime = (time: number | string) => {
+export const formatTime = (time: number | string | Date) => {
   if (typeof time === "string") {
     time = parseInt(time);
+  } else if (time instanceof Date) {
+    time = time.getTime();
   }
   const d = new Date();
   d.setTime(time - timezoneOffsetMs);
@@ -33,14 +35,16 @@ export const formatTime = (time: number | string) => {
 };
 
 // formatDate formats a time as YYYY-MM-DD
-export const formatDate = (time: number | string) => {
+export const formatDate = (time: number | string | Date) => {
   if (typeof time === "string") {
     time = parseInt(time);
+  } else if (time instanceof Date) {
+    time = time.getTime();
   }
-  const d = new Date();
+  let d = new Date();
   d.setTime(time - timezoneOffsetMs);
   const isoStr = d.toISOString();
-  return isoStr.substring(0, 4);
+  return isoStr.substring(0, 10);
 };
 
 export const formatDuration = (ms: number) => {

webui/src/state/config.ts

@@ -1,6 +1,7 @@
 import { atom, useSetRecoilState } from "recoil";
 import { Config, Repo } from "../../gen/ts/v1/config.pb";
 import { ResticUI } from "../../gen/ts/v1/service.pb";
+import { API_PREFIX } from "../constants";
 
 export const configState = atom<Config>({
   key: "config",
@@ -8,17 +9,17 @@ export const configState = atom<Config>({
 });
 
 export const fetchConfig = async (): Promise<Config> => {
-  return await ResticUI.GetConfig({}, { pathPrefix: "/api" });
+  return await ResticUI.GetConfig({}, { pathPrefix: API_PREFIX });
 };
 
 export const addRepo = async (repo: Repo): Promise<Config> => {
   return await ResticUI.AddRepo(repo, {
-    pathPrefix: "/api",
+    pathPrefix: API_PREFIX,
   });
 };
 
 export const updateConfig = async (config: Config): Promise<Config> => {
   return await ResticUI.SetConfig(config, {
-    pathPrefix: "/api",
+    pathPrefix: API_PREFIX,
   });
-};
+};

webui/src/state/oplog.ts

@@ -8,10 +8,13 @@ import {
 import { GetOperationsRequest, ResticUI } from "../../gen/ts/v1/service.pb";
 import { EventEmitter } from "events";
 import { useAlertApi } from "../components/Alerts";
+import { API_PREFIX } from "../constants";
+import { BackupProgressEntry, ResticSnapshot } from "../../gen/ts/v1/restic.pb";
 
 export type EOperation = Operation & {
   parsedId: number;
   parsedTime: number;
+  parsedDate: Date;
 };
 
 const subscribers: ((event: OperationEvent) => void)[] = [];
@@ -28,7 +31,7 @@ const subscribers: ((event: OperationEvent) => void)[] = [];
           subscribers.forEach((subscriber) => subscriber(event));
         },
         {
-          pathPrefix: "/api",
+          pathPrefix: API_PREFIX,
         }
       );
     } catch (e: any) {
@@ -52,7 +55,7 @@ export const getOperations = async ({
       lastN,
     },
     {
-      pathPrefix: "/api",
+      pathPrefix: API_PREFIX,
     }
   );
   return (opList.operations || []).map(toEop);
@@ -124,12 +127,23 @@ export const buildOperationListListener = (
 };
 
 export const toEop = (op: Operation): EOperation => {
-  const time =
+  const time = parseInt(op.unixTimeStartMs!);
-    op.operationIndexSnapshot?.snapshot?.unixTimeMs || op.unixTimeStartMs;
+  const date = new Date();
+  date.setTime(time);
 
   return {
     ...op,
     parsedId: parseInt(op.id!),
-    parsedTime: parseInt(time!),
+    parsedTime: time,
+    parsedDate: date,
   };
 };
+
+// TODO: aggregate backup info from oplog.
+interface BackupInfo {
+  opids: string[];
+  repoId: string;
+  planId: string;
+  backupLastStatus?: BackupProgressEntry;
+  snapshotInfo?: ResticSnapshot;
+}

webui/src/views/PlanView.tsx

@@ -16,6 +16,7 @@ import {
 } from "../state/oplog";
 import { OperationList } from "../components/OperationList";
 import { OperationTree } from "../components/OperationTree";
+import { MAX_OPERATION_HISTORY } from "../constants";
 
 export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
   const showModal = useShowModal();
@@ -24,7 +25,7 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
 
   useEffect(() => {
     const listener = buildOperationListListener(
-      { planId: plan.id, lastN: "10000" },
+      { planId: plan.id, lastN: "" + MAX_OPERATION_HISTORY },
       (event, changedOp, operations) => {
         setOperations([...operations]);
       }
@@ -87,7 +88,7 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
             label: "Condensed View",
             children: (
               <>
-                <OperationTree operations={[...operations]} />
+                <OperationTree operations={operations} />
               </>
             ),
           },
@@ -97,7 +98,7 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
             children: (
               <>
                 <h2>Backup Action History ({operations.length} loaded)</h2>
-                <OperationList operations={[...operations]} />
+                <OperationList operations={operations} />
               </>
             ),
           },