From 7ad3a74fb206e873354d268710e2a8c49d7c2bf9 Mon Sep 17 00:00:00 2001 From: Gareth George Date: Sat, 25 Nov 2023 17:09:00 -0800 Subject: [PATCH] created basic tree layout --- gen/go/v1/operations.pb.go | 23 +++++--- internal/oplog/oplog.go | 18 ++++-- internal/orchestrator/tasks.go | 19 +----- proto/v1/operations.proto | 12 ++-- webui/src/components/OperationList.tsx | 2 +- webui/src/components/OperationTree.tsx | 82 ++++++++++++++++++++------ webui/src/constants.ts | 2 + webui/src/lib/formatting.ts | 12 ++-- webui/src/state/config.ts | 9 +-- webui/src/state/oplog.ts | 24 ++++++-- webui/src/views/PlanView.tsx | 7 ++- 11 files changed, 141 insertions(+), 69 deletions(-) create mode 100644 webui/src/constants.ts diff --git a/gen/go/v1/operations.pb.go b/gen/go/v1/operations.pb.go index 641c1cb..e3e0616 100644 --- a/gen/go/v1/operations.pb.go +++ b/gen/go/v1/operations.pb.go @@ -177,14 +177,21 @@ type Operation struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` - RepoId string `protobuf:"bytes,2,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"` // repo id if associated with a repo (always true) - PlanId string `protobuf:"bytes,3,opt,name=plan_id,json=planId,proto3" json:"plan_id,omitempty"` // plan id if associated with a plan (always true) - SnapshotId string `protobuf:"bytes,8,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` // snapshot id if associated with a snapshot. - Status OperationStatus `protobuf:"varint,4,opt,name=status,proto3,enum=v1.OperationStatus" json:"status,omitempty"` - UnixTimeStartMs int64 `protobuf:"varint,5,opt,name=unix_time_start_ms,json=unixTimeStartMs,proto3" json:"unix_time_start_ms,omitempty"` - UnixTimeEndMs int64 `protobuf:"varint,6,opt,name=unix_time_end_ms,json=unixTimeEndMs,proto3" json:"unix_time_end_ms,omitempty"` - DisplayMessage string `protobuf:"bytes,7,opt,name=display_message,json=displayMessage,proto3" json:"display_message,omitempty"` // human readable context message (if any) + // required, primary ID of the operation. + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + // required, repo id if associated with a repo + RepoId string `protobuf:"bytes,2,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"` + // required, plan id if associated with a plan + PlanId string `protobuf:"bytes,3,opt,name=plan_id,json=planId,proto3" json:"plan_id,omitempty"` + // optional snapshot id if associated with a snapshot. + SnapshotId string `protobuf:"bytes,8,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + Status OperationStatus `protobuf:"varint,4,opt,name=status,proto3,enum=v1.OperationStatus" json:"status,omitempty"` + // required, unix time in milliseconds of the operation's creation (ID is derived from this) + UnixTimeStartMs int64 `protobuf:"varint,5,opt,name=unix_time_start_ms,json=unixTimeStartMs,proto3" json:"unix_time_start_ms,omitempty"` + // optional, unix time in milliseconds of the operation's completion + UnixTimeEndMs int64 `protobuf:"varint,6,opt,name=unix_time_end_ms,json=unixTimeEndMs,proto3" json:"unix_time_end_ms,omitempty"` + // optional, human readable context message, typically an error message. + DisplayMessage string `protobuf:"bytes,7,opt,name=display_message,json=displayMessage,proto3" json:"display_message,omitempty"` // Types that are assignable to Op: // // *Operation_OperationBackup diff --git a/internal/oplog/oplog.go b/internal/oplog/oplog.go index be356dd..b4c10c3 100644 --- a/internal/oplog/oplog.go +++ b/internal/oplog/oplog.go @@ -195,17 +195,25 @@ func (o *OpLog) getOperationHelper(b *bolt.Bucket, id int64) (*v1.Operation, err return &op, nil } +func (o *OpLog) nextOperationId(b *bolt.Bucket, unixTimeMs int64) (int64, error) { + seq, err := b.NextSequence() + if err != nil { + return 0, fmt.Errorf("next sequence: %w", err) + } + return int64(unixTimeMs<<20) | int64(seq&((1<<20)-1)), nil +} + func (o *OpLog) addOperationHelper(tx *bolt.Tx, op *v1.Operation) error { b := tx.Bucket(OpLogBucket) if op.Id == 0 { - seq, err := b.NextSequence() - if err != nil { - return fmt.Errorf("error getting next sequence: %w", err) - } if op.UnixTimeStartMs == 0 { return fmt.Errorf("operation must have a start time") } - op.Id = op.UnixTimeStartMs<<20 | int64(seq&(1<<20-1)) + var err error + op.Id, err = o.nextOperationId(b, op.UnixTimeStartMs) + if err != nil { + return fmt.Errorf("create next operation ID: %w", err) + } } op.SnapshotId = NormalizeSnapshotId(op.SnapshotId) diff --git a/internal/orchestrator/tasks.go b/internal/orchestrator/tasks.go index 3ba7358..119f00e 100644 --- a/internal/orchestrator/tasks.go +++ b/internal/orchestrator/tasks.go @@ -47,21 +47,6 @@ func (t *ScheduledBackupTask) Name() string { } func (t *ScheduledBackupTask) Next(now time.Time) *time.Time { - if ops, err := t.orchestrator.OpLog.GetByPlan(t.plan.Id, indexutil.CollectLastN(10)); err == nil { - var lastBackupOp *v1.Operation - for _, op := range ops { - if _, ok := op.Op.(*v1.Operation_OperationBackup); ok { - lastBackupOp = op - } - } - - if lastBackupOp != nil { - now = time.Unix(0, lastBackupOp.UnixTimeStartMs*int64(time.Millisecond)) - } - } else { - zap.S().Errorf("error getting last operation for plan %q when computing backup schedule: %v", t.plan.Id, err) - } - next := t.schedule.Next(now) return &next } @@ -116,7 +101,7 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan startTime := time.Now() err := WithOperation(orchestrator.OpLog, op, func() error { - zap.L().Info("Starting backup", zap.String("plan", plan.Id)) + zap.L().Info("Starting backup", zap.String("plan", plan.Id), zap.Int64("opId", op.Id)) repo, err := orchestrator.GetRepo(plan.Repo) if err != nil { return fmt.Errorf("couldn't get repo %q: %w", plan.Repo, err) @@ -230,7 +215,7 @@ func WithOperation(oplog *oplog.OpLog, op *v1.Operation, do func() error) error if op.Status == v1.OperationStatus_STATUS_INPROGRESS { op.Status = v1.OperationStatus_STATUS_SUCCESS } - if e := oplog.Update(op); err != nil { + if e := oplog.Update(op); e != nil { return multierror.Append(err, fmt.Errorf("failed to update operation in oplog: %w", e)) } return err diff --git a/proto/v1/operations.proto b/proto/v1/operations.proto index 6c2cb26..bc9ac6d 100644 --- a/proto/v1/operations.proto +++ b/proto/v1/operations.proto @@ -11,18 +11,20 @@ message OperationList { } message Operation { + // required, primary ID of the operation. int64 id = 1; - // repo id if associated with a repo (always true) + // required, repo id if associated with a repo string repo_id = 2; - // plan id if associated with a plan (always true) + // required, plan id if associated with a plan string plan_id = 3; - // snapshot id if associated with a snapshot. + // optional snapshot id if associated with a snapshot. string snapshot_id = 8; OperationStatus status = 4; - // unix time in milliseconds of the operation's creation (ID is derived from this) + // required, unix time in milliseconds of the operation's creation (ID is derived from this) int64 unix_time_start_ms = 5; + // optional, unix time in milliseconds of the operation's completion int64 unix_time_end_ms = 6; - // human readable context message, typically an error message. + // optional, human readable context message, typically an error message. string display_message = 7; oneof op { diff --git a/webui/src/components/OperationList.tsx b/webui/src/components/OperationList.tsx index 9dc6728..675d37f 100644 --- a/webui/src/components/OperationList.tsx +++ b/webui/src/components/OperationList.tsx @@ -28,7 +28,7 @@ import { export const OperationList = ({ operations, }: React.PropsWithoutRef<{ operations: EOperation[] }>) => { - operations.sort((a, b) => b.parsedTime - a.parsedTime); + operations = [...operations].reverse(); if (operations.length === 0) { return ( diff --git a/webui/src/components/OperationTree.tsx b/webui/src/components/OperationTree.tsx index e48e959..7b7407b 100644 --- a/webui/src/components/OperationTree.tsx +++ b/webui/src/components/OperationTree.tsx @@ -8,30 +8,78 @@ import { formatDate, formatTime } from "../lib/formatting"; export const OperationTree = ({ operations, }: React.PropsWithoutRef<{ operations: EOperation[] }>) => { - operations.sort((a, b) => b.parsedTime - a.parsedTime); - return ; + operations = [...operations].reverse(); // reverse such that newest operations are at index 0. + + const treeData = buildTreeYear(operations); + const keys = buildDefaultExpandedKeys(treeData); + + return ; }; -// TODO: more work on this view -const buildTree = (operations: EOperation[]): DataNode[] => { +const buildDefaultExpandedKeys = (tree?: DataNode[]) => { + const keys: string[] = []; + while (tree && tree.length > 0) { + const node = tree[0]; + keys.push(node.key as string); + tree = node.children!; + } + return keys; +}; + +const buildTreeYear = (operations: EOperation[]): DataNode[] => { const grouped = _.groupBy(operations, (op) => { - return new Date(op.parsedTime).toLocaleDateString("default", { - month: "long", - year: "numeric", - day: "numeric", - }); + return op.parsedDate.getFullYear(); }); - return _.keys(grouped).map((key) => { + const entries: DataNode[] = _.map(grouped, (value, key) => { return { - key: key, - title: key, - children: grouped[key].map((op) => { - return { - key: op.id!, - title: {formatTime(op.parsedTime)} - AN OPERATION, - }; + key: "y" + key, + title: "" + key, + children: buildTreeMonth(value), + }; + }); + return entries; +}; + +const buildTreeMonth = (operations: EOperation[]): DataNode[] => { + const grouped = _.groupBy(operations, (op) => { + return op.parsedDate.getMonth(); + }); + const entries: DataNode[] = _.map(grouped, (value, key) => { + return { + key: "m" + key, + title: value[0].parsedDate.toLocaleString("default", { + month: "long", + year: "numeric", }), + children: buildTreeDay(value), + }; + }); + return entries; +}; + +const buildTreeDay = (operations: EOperation[]): DataNode[] => { + const grouped = _.groupBy(operations, (op) => { + console.log("Operation date: " + formatDate(op.parsedTime)); + return formatDate(op.parsedTime); + }); + + const entries = _.map(grouped, (value, key) => { + return { + key: "d" + key, + title: formatDate(value[0].parsedTime), + children: buildTreeLeaf(value), + }; + }); + return entries; +}; + +const buildTreeLeaf = (operations: EOperation[]): DataNode[] => { + return _.map(operations, (op) => { + return { + key: op.id!, + title: formatTime(op.parsedDate) + " - ", + isLeaf: true, }; }); }; diff --git a/webui/src/constants.ts b/webui/src/constants.ts new file mode 100644 index 0000000..c11d281 --- /dev/null +++ b/webui/src/constants.ts @@ -0,0 +1,2 @@ +export const API_PREFIX = "/api"; +export const MAX_OPERATION_HISTORY = 10000; diff --git a/webui/src/lib/formatting.ts b/webui/src/lib/formatting.ts index e6f9b04..7510b50 100644 --- a/webui/src/lib/formatting.ts +++ b/webui/src/lib/formatting.ts @@ -17,9 +17,11 @@ export const formatBytes = (bytes?: number | string) => { const timezoneOffsetMs = new Date().getTimezoneOffset() * 60 * 1000; // formatTime formats a time as YYYY-MM-DD at HH:MM AM/PM -export const formatTime = (time: number | string) => { +export const formatTime = (time: number | string | Date) => { if (typeof time === "string") { time = parseInt(time); + } else if (time instanceof Date) { + time = time.getTime(); } const d = new Date(); d.setTime(time - timezoneOffsetMs); @@ -33,14 +35,16 @@ export const formatTime = (time: number | string) => { }; // formatDate formats a time as YYYY-MM-DD -export const formatDate = (time: number | string) => { +export const formatDate = (time: number | string | Date) => { if (typeof time === "string") { time = parseInt(time); + } else if (time instanceof Date) { + time = time.getTime(); } - const d = new Date(); + let d = new Date(); d.setTime(time - timezoneOffsetMs); const isoStr = d.toISOString(); - return isoStr.substring(0, 4); + return isoStr.substring(0, 10); }; export const formatDuration = (ms: number) => { diff --git a/webui/src/state/config.ts b/webui/src/state/config.ts index 9b8f0d2..8ff0972 100644 --- a/webui/src/state/config.ts +++ b/webui/src/state/config.ts @@ -1,6 +1,7 @@ import { atom, useSetRecoilState } from "recoil"; import { Config, Repo } from "../../gen/ts/v1/config.pb"; import { ResticUI } from "../../gen/ts/v1/service.pb"; +import { API_PREFIX } from "../constants"; export const configState = atom({ key: "config", @@ -8,17 +9,17 @@ export const configState = atom({ }); export const fetchConfig = async (): Promise => { - return await ResticUI.GetConfig({}, { pathPrefix: "/api" }); + return await ResticUI.GetConfig({}, { pathPrefix: API_PREFIX }); }; export const addRepo = async (repo: Repo): Promise => { return await ResticUI.AddRepo(repo, { - pathPrefix: "/api", + pathPrefix: API_PREFIX, }); }; export const updateConfig = async (config: Config): Promise => { return await ResticUI.SetConfig(config, { - pathPrefix: "/api", + pathPrefix: API_PREFIX, }); -}; \ No newline at end of file +}; diff --git a/webui/src/state/oplog.ts b/webui/src/state/oplog.ts index b83c880..0005b31 100644 --- a/webui/src/state/oplog.ts +++ b/webui/src/state/oplog.ts @@ -8,10 +8,13 @@ import { import { GetOperationsRequest, ResticUI } from "../../gen/ts/v1/service.pb"; import { EventEmitter } from "events"; import { useAlertApi } from "../components/Alerts"; +import { API_PREFIX } from "../constants"; +import { BackupProgressEntry, ResticSnapshot } from "../../gen/ts/v1/restic.pb"; export type EOperation = Operation & { parsedId: number; parsedTime: number; + parsedDate: Date; }; const subscribers: ((event: OperationEvent) => void)[] = []; @@ -28,7 +31,7 @@ const subscribers: ((event: OperationEvent) => void)[] = []; subscribers.forEach((subscriber) => subscriber(event)); }, { - pathPrefix: "/api", + pathPrefix: API_PREFIX, } ); } catch (e: any) { @@ -52,7 +55,7 @@ export const getOperations = async ({ lastN, }, { - pathPrefix: "/api", + pathPrefix: API_PREFIX, } ); return (opList.operations || []).map(toEop); @@ -124,12 +127,23 @@ export const buildOperationListListener = ( }; export const toEop = (op: Operation): EOperation => { - const time = - op.operationIndexSnapshot?.snapshot?.unixTimeMs || op.unixTimeStartMs; + const time = parseInt(op.unixTimeStartMs!); + const date = new Date(); + date.setTime(time); return { ...op, parsedId: parseInt(op.id!), - parsedTime: parseInt(time!), + parsedTime: time, + parsedDate: date, }; }; + +// TODO: aggregate backup info from oplog. +interface BackupInfo { + opids: string[]; + repoId: string; + planId: string; + backupLastStatus?: BackupProgressEntry; + snapshotInfo?: ResticSnapshot; +} diff --git a/webui/src/views/PlanView.tsx b/webui/src/views/PlanView.tsx index ce2cf61..f4d1c7e 100644 --- a/webui/src/views/PlanView.tsx +++ b/webui/src/views/PlanView.tsx @@ -16,6 +16,7 @@ import { } from "../state/oplog"; import { OperationList } from "../components/OperationList"; import { OperationTree } from "../components/OperationTree"; +import { MAX_OPERATION_HISTORY } from "../constants"; export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => { const showModal = useShowModal(); @@ -24,7 +25,7 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => { useEffect(() => { const listener = buildOperationListListener( - { planId: plan.id, lastN: "10000" }, + { planId: plan.id, lastN: "" + MAX_OPERATION_HISTORY }, (event, changedOp, operations) => { setOperations([...operations]); } @@ -87,7 +88,7 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => { label: "Condensed View", children: ( <> - + ), }, @@ -97,7 +98,7 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => { children: ( <>

Backup Action History ({operations.length} loaded)

- + ), },