lee/backrest
1
0
Fork 0
mirror of https://github.com/garethgeorge/backrest.git synced 2025-12-12 16:55:39 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

created basic tree layout

Browse Source
This commit is contained in:
Gareth George
2023-11-25 17:09:00 -08:00
parent ba390a2ca1
commit 7ad3a74fb2
11 changed files with 141 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
gen/go/v1/operations.pb.go
View File

@@ -177,14 +177,21 @@ type Operation struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
RepoId string `protobuf:"bytes,2,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"` // repo id if associated with a repo (always true)
PlanId string `protobuf:"bytes,3,opt,name=plan_id,json=planId,proto3" json:"plan_id,omitempty"` // plan id if associated with a plan (always true)
SnapshotId string `protobuf:"bytes,8,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` // snapshot id if associated with a snapshot.
Status OperationStatus `protobuf:"varint,4,opt,name=status,proto3,enum=v1.OperationStatus" json:"status,omitempty"`
UnixTimeStartMs int64 `protobuf:"varint,5,opt,name=unix_time_start_ms,json=unixTimeStartMs,proto3" json:"unix_time_start_ms,omitempty"`
UnixTimeEndMs int64 `protobuf:"varint,6,opt,name=unix_time_end_ms,json=unixTimeEndMs,proto3" json:"unix_time_end_ms,omitempty"`
DisplayMessage string `protobuf:"bytes,7,opt,name=display_message,json=displayMessage,proto3" json:"display_message,omitempty"` // human readable context message (if any)
// required, primary ID of the operation.
Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
// required, repo id if associated with a repo
RepoId string `protobuf:"bytes,2,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"`
// required, plan id if associated with a plan
PlanId string `protobuf:"bytes,3,opt,name=plan_id,json=planId,proto3" json:"plan_id,omitempty"`
// optional snapshot id if associated with a snapshot.
SnapshotId string `protobuf:"bytes,8,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"`
Status OperationStatus `protobuf:"varint,4,opt,name=status,proto3,enum=v1.OperationStatus" json:"status,omitempty"`
// required, unix time in milliseconds of the operation's creation (ID is derived from this)
UnixTimeStartMs int64 `protobuf:"varint,5,opt,name=unix_time_start_ms,json=unixTimeStartMs,proto3" json:"unix_time_start_ms,omitempty"`
// optional, unix time in milliseconds of the operation's completion
UnixTimeEndMs int64 `protobuf:"varint,6,opt,name=unix_time_end_ms,json=unixTimeEndMs,proto3" json:"unix_time_end_ms,omitempty"`
// optional, human readable context message, typically an error message.
DisplayMessage string `protobuf:"bytes,7,opt,name=display_message,json=displayMessage,proto3" json:"display_message,omitempty"`
// Types that are assignable to Op:
//
// *Operation_OperationBackup

18
internal/oplog/oplog.go
View File

@@ -195,17 +195,25 @@ func (o *OpLog) getOperationHelper(b *bolt.Bucket, id int64) (*v1.Operation, err
return &op, nil
}
func (o *OpLog) nextOperationId(b *bolt.Bucket, unixTimeMs int64) (int64, error) {
seq, err := b.NextSequence()
if err != nil {
return 0, fmt.Errorf("next sequence: %w", err)
}
return int64(unixTimeMs<<20) | int64(seq&((1<<20)-1)), nil
}
func (o *OpLog) addOperationHelper(tx *bolt.Tx, op *v1.Operation) error {
b := tx.Bucket(OpLogBucket)
if op.Id == 0 {
seq, err := b.NextSequence()
if err != nil {
return fmt.Errorf("error getting next sequence: %w", err)
}
if op.UnixTimeStartMs == 0 {
return fmt.Errorf("operation must have a start time")
}
op.Id = op.UnixTimeStartMs<<20 | int64(seq&(1<<20-1))
var err error
op.Id, err = o.nextOperationId(b, op.UnixTimeStartMs)
if err != nil {
return fmt.Errorf("create next operation ID: %w", err)
}
}
op.SnapshotId = NormalizeSnapshotId(op.SnapshotId)

19
internal/orchestrator/tasks.go
View File

@@ -47,21 +47,6 @@ func (t *ScheduledBackupTask) Name() string {
}
func (t *ScheduledBackupTask) Next(now time.Time) *time.Time {
if ops, err := t.orchestrator.OpLog.GetByPlan(t.plan.Id, indexutil.CollectLastN(10)); err == nil {
var lastBackupOp *v1.Operation
for _, op := range ops {
if _, ok := op.Op.(*v1.Operation_OperationBackup); ok {
lastBackupOp = op
}
}
if lastBackupOp != nil {
now = time.Unix(0, lastBackupOp.UnixTimeStartMs*int64(time.Millisecond))
}
} else {
zap.S().Errorf("error getting last operation for plan %q when computing backup schedule: %v", t.plan.Id, err)
}
next := t.schedule.Next(now)
return &next
}
@@ -116,7 +101,7 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan
startTime := time.Now()
err := WithOperation(orchestrator.OpLog, op, func() error {
zap.L().Info("Starting backup", zap.String("plan", plan.Id))
zap.L().Info("Starting backup", zap.String("plan", plan.Id), zap.Int64("opId", op.Id))
repo, err := orchestrator.GetRepo(plan.Repo)
if err != nil {
return fmt.Errorf("couldn't get repo %q: %w", plan.Repo, err)
@@ -230,7 +215,7 @@ func WithOperation(oplog *oplog.OpLog, op *v1.Operation, do func() error) error
if op.Status == v1.OperationStatus_STATUS_INPROGRESS {
op.Status = v1.OperationStatus_STATUS_SUCCESS
}
if e := oplog.Update(op); err != nil {
if e := oplog.Update(op); e != nil {
return multierror.Append(err, fmt.Errorf("failed to update operation in oplog: %w", e))
}
return err

12
proto/v1/operations.proto
View File

@@ -11,18 +11,20 @@ message OperationList {
}
message Operation {
// required, primary ID of the operation.
int64 id = 1;
// repo id if associated with a repo (always true)
// required, repo id if associated with a repo
string repo_id = 2;
// plan id if associated with a plan (always true)
// required, plan id if associated with a plan
string plan_id = 3;
// snapshot id if associated with a snapshot.
// optional snapshot id if associated with a snapshot.
string snapshot_id = 8;
OperationStatus status = 4;
// unix time in milliseconds of the operation's creation (ID is derived from this)
// required, unix time in milliseconds of the operation's creation (ID is derived from this)
int64 unix_time_start_ms = 5;
// optional, unix time in milliseconds of the operation's completion
int64 unix_time_end_ms = 6;
// human readable context message, typically an error message.
// optional, human readable context message, typically an error message.
string display_message = 7;
oneof op {

2
webui/src/components/OperationList.tsx
View File

@@ -28,7 +28,7 @@ import {
export const OperationList = ({
operations,
}: React.PropsWithoutRef<{ operations: EOperation[] }>) => {
operations.sort((a, b) => b.parsedTime - a.parsedTime);
operations = [...operations].reverse();
if (operations.length === 0) {
return (

82
webui/src/components/OperationTree.tsx
View File

@@ -8,30 +8,78 @@ import { formatDate, formatTime } from "../lib/formatting";
export const OperationTree = ({
operations,
}: React.PropsWithoutRef<{ operations: EOperation[] }>) => {
operations.sort((a, b) => b.parsedTime - a.parsedTime);
return <Tree treeData={buildTree(operations)}></Tree>;
operations = [...operations].reverse(); // reverse such that newest operations are at index 0.
const treeData = buildTreeYear(operations);
const keys = buildDefaultExpandedKeys(treeData);
return <Tree treeData={treeData} defaultExpandedKeys={keys}></Tree>;
};
// TODO: more work on this view
const buildTree = (operations: EOperation[]): DataNode[] => {
const buildDefaultExpandedKeys = (tree?: DataNode[]) => {
const keys: string[] = [];
while (tree && tree.length > 0) {
const node = tree[0];
keys.push(node.key as string);
tree = node.children!;
}
return keys;
};
const buildTreeYear = (operations: EOperation[]): DataNode[] => {
const grouped = _.groupBy(operations, (op) => {
return new Date(op.parsedTime).toLocaleDateString("default", {
month: "long",
year: "numeric",
day: "numeric",
});
return op.parsedDate.getFullYear();
});
return _.keys(grouped).map((key) => {
const entries: DataNode[] = _.map(grouped, (value, key) => {
return {
key: key,
title: key,
children: grouped[key].map((op) => {
return {
key: op.id!,
title: <span>{formatTime(op.parsedTime)} - AN OPERATION</span>,
};
key: "y" + key,
title: "" + key,
children: buildTreeMonth(value),
};
});
return entries;
};
const buildTreeMonth = (operations: EOperation[]): DataNode[] => {
const grouped = _.groupBy(operations, (op) => {
return op.parsedDate.getMonth();
});
const entries: DataNode[] = _.map(grouped, (value, key) => {
return {
key: "m" + key,
title: value[0].parsedDate.toLocaleString("default", {
month: "long",
year: "numeric",
}),
children: buildTreeDay(value),
};
});
return entries;
};
const buildTreeDay = (operations: EOperation[]): DataNode[] => {
const grouped = _.groupBy(operations, (op) => {
console.log("Operation date: " + formatDate(op.parsedTime));
return formatDate(op.parsedTime);
});
const entries = _.map(grouped, (value, key) => {
return {
key: "d" + key,
title: formatDate(value[0].parsedTime),
children: buildTreeLeaf(value),
};
});
return entries;
};
const buildTreeLeaf = (operations: EOperation[]): DataNode[] => {
return _.map(operations, (op) => {
return {
key: op.id!,
title: formatTime(op.parsedDate) + " - ",
isLeaf: true,
};
});
};

2
webui/src/constants.ts Normal file
View File

@@ -0,0 +1,2 @@
export const API_PREFIX = "/api";
export const MAX_OPERATION_HISTORY = 10000;

12
webui/src/lib/formatting.ts
View File

@@ -17,9 +17,11 @@ export const formatBytes = (bytes?: number | string) => {
const timezoneOffsetMs = new Date().getTimezoneOffset() * 60 * 1000;
// formatTime formats a time as YYYY-MM-DD at HH:MM AM/PM
export const formatTime = (time: number | string) => {
export const formatTime = (time: number | string | Date) => {
if (typeof time === "string") {
time = parseInt(time);
} else if (time instanceof Date) {
time = time.getTime();
}
const d = new Date();
d.setTime(time - timezoneOffsetMs);
@@ -33,14 +35,16 @@ export const formatTime = (time: number | string) => {
};
// formatDate formats a time as YYYY-MM-DD
export const formatDate = (time: number | string) => {
export const formatDate = (time: number | string | Date) => {
if (typeof time === "string") {
time = parseInt(time);
} else if (time instanceof Date) {
time = time.getTime();
}
const d = new Date();
let d = new Date();
d.setTime(time - timezoneOffsetMs);
const isoStr = d.toISOString();
return isoStr.substring(0, 4);
return isoStr.substring(0, 10);
};
export const formatDuration = (ms: number) => {

7
webui/src/state/config.ts
View File

@@ -1,6 +1,7 @@
import { atom, useSetRecoilState } from "recoil";
import { Config, Repo } from "../../gen/ts/v1/config.pb";
import { ResticUI } from "../../gen/ts/v1/service.pb";
import { API_PREFIX } from "../constants";
export const configState = atom<Config>({
key: "config",
@@ -8,17 +9,17 @@ export const configState = atom<Config>({
});
export const fetchConfig = async (): Promise<Config> => {
return await ResticUI.GetConfig({}, { pathPrefix: "/api" });
return await ResticUI.GetConfig({}, { pathPrefix: API_PREFIX });
};
export const addRepo = async (repo: Repo): Promise<Config> => {
return await ResticUI.AddRepo(repo, {
pathPrefix: "/api",
pathPrefix: API_PREFIX,
});
};
export const updateConfig = async (config: Config): Promise<Config> => {
return await ResticUI.SetConfig(config, {
pathPrefix: "/api",
pathPrefix: API_PREFIX,
});
};

24
webui/src/state/oplog.ts
View File

@@ -8,10 +8,13 @@ import {
import { GetOperationsRequest, ResticUI } from "../../gen/ts/v1/service.pb";
import { EventEmitter } from "events";
import { useAlertApi } from "../components/Alerts";
import { API_PREFIX } from "../constants";
import { BackupProgressEntry, ResticSnapshot } from "../../gen/ts/v1/restic.pb";
export type EOperation = Operation & {
parsedId: number;
parsedTime: number;
parsedDate: Date;
};
const subscribers: ((event: OperationEvent) => void)[] = [];
@@ -28,7 +31,7 @@ const subscribers: ((event: OperationEvent) => void)[] = [];
subscribers.forEach((subscriber) => subscriber(event));
},
{
pathPrefix: "/api",
pathPrefix: API_PREFIX,
}
);
} catch (e: any) {
@@ -52,7 +55,7 @@ export const getOperations = async ({
lastN,
},
{
pathPrefix: "/api",
pathPrefix: API_PREFIX,
}
);
return (opList.operations || []).map(toEop);
@@ -124,12 +127,23 @@ export const buildOperationListListener = (
};
export const toEop = (op: Operation): EOperation => {
const time =
op.operationIndexSnapshot?.snapshot?.unixTimeMs || op.unixTimeStartMs;
const time = parseInt(op.unixTimeStartMs!);
const date = new Date();
date.setTime(time);
return {
...op,
parsedId: parseInt(op.id!),
parsedTime: parseInt(time!),
parsedTime: time,
parsedDate: date,
};
};
// TODO: aggregate backup info from oplog.
interface BackupInfo {
opids: string[];
repoId: string;
planId: string;
backupLastStatus?: BackupProgressEntry;
snapshotInfo?: ResticSnapshot;
}

7
webui/src/views/PlanView.tsx
View File

@@ -16,6 +16,7 @@ import {
} from "../state/oplog";
import { OperationList } from "../components/OperationList";
import { OperationTree } from "../components/OperationTree";
import { MAX_OPERATION_HISTORY } from "../constants";
export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
const showModal = useShowModal();
@@ -24,7 +25,7 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
useEffect(() => {
const listener = buildOperationListListener(
{ planId: plan.id, lastN: "10000" },
{ planId: plan.id, lastN: "" + MAX_OPERATION_HISTORY },
(event, changedOp, operations) => {
setOperations([...operations]);
}
@@ -87,7 +88,7 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
label: "Condensed View",
children: (
<>
<OperationTree operations={[...operations]} />
<OperationTree operations={operations} />
</>
),
},
@@ -97,7 +98,7 @@ export const PlanView = ({ plan }: React.PropsWithChildren<{ plan: Plan }>) => {
children: (
<>
<h2>Backup Action History ({operations.length} loaded)</h2>
<OperationList operations={[...operations]} />
<OperationList operations={operations} />
</>
),
},