fix: forget deadlocking and misc smaller bugs

This commit is contained in:
garethgeorge
2023-12-02 13:21:57 -08:00
parent 725fe57537
commit 265fdd1673
6 changed files with 30 additions and 27 deletions
+1 -1
View File
@@ -41,7 +41,7 @@ func BindAddress() string {
}
return val
}
return "127.0.0.1:9898"
return ":9898"
}
func ResticBinPath() string {
+3 -3
View File
@@ -148,11 +148,11 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan
return fmt.Errorf("backup operation: %w", err)
}
at := time.Now()
if plan.Retention != nil {
orchestrator.ScheduleTask(NewOneofForgetTask(orchestrator, plan, op.SnapshotId, time.Now()), taskPriorityForget)
orchestrator.ScheduleTask(NewOneofForgetTask(orchestrator, plan, op.SnapshotId, at), taskPriorityForget)
}
orchestrator.ScheduleTask(NewOneofIndexSnapshotsTask(orchestrator, plan, time.Now()), taskPriorityIndexSnapshots)
orchestrator.ScheduleTask(NewOneofIndexSnapshotsTask(orchestrator, plan, at), taskPriorityIndexSnapshots)
return nil
}
+16 -9
View File
@@ -80,21 +80,28 @@ func (t *ForgetTask) Run(ctx context.Context) error {
forgetOp.OperationForget.Forget = append(forgetOp.OperationForget.Forget, forgot...)
var ops []*v1.Operation
for _, forgot := range forgot {
if e := t.orchestrator.OpLog.ForEachBySnapshotId(forgot.Id, indexutil.CollectAll(), func(op *v1.Operation) error {
if indexOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok {
indexOp.OperationIndexSnapshot.Forgot = true
if err := t.orchestrator.OpLog.Update(op); err != nil {
return fmt.Errorf("mark index snapshot %v as forgotten: %w", op.Id, err)
}
}
// Soft delete the operation (can be recovered if necessary, todo: implement recovery).
return t.orchestrator.OpLog.Delete(op.Id)
ops = append(ops, op)
return nil
}); e != nil {
err = multierror.Append(err, fmt.Errorf("cleanup snapshot %v: %w", forgot.Id, e))
}
}
for _, op := range ops {
if indexOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok {
indexOp.OperationIndexSnapshot.Forgot = true
if e := t.orchestrator.OpLog.Update(op); err != nil {
err = multierror.Append(err, fmt.Errorf("mark index snapshot %v as forgotten: %w", op.Id, e))
continue
}
}
// Soft delete the operation (can be recovered if necessary, todo: implement recovery).
if e := t.orchestrator.OpLog.Delete(op.Id); err != nil {
err = multierror.Append(err, fmt.Errorf("delete operation %v: %w", op.Id, e))
}
}
return err
}); err != nil {
+9 -3
View File
@@ -115,15 +115,21 @@ func (r *RepoOrchestrator) Forget(ctx context.Context, plan *v1.Plan) ([]*v1.Res
l := zap.L().With(zap.String("repo", r.repoConfig.Id), zap.String("plan", plan.Id))
l.Debug("Forget snapshots", zap.Any("policy", policy))
result, err := r.repo.Forget(ctx, protoutil.RetentionPolicyFromProto(plan.Retention), restic.WithFlags("--tag", tagForPlan(plan), "--group-by", ""))
result, err := r.repo.Forget(
ctx, protoutil.RetentionPolicyFromProto(plan.Retention),
restic.WithFlags("--tag", tagForPlan(plan)), restic.WithFlags("--group-by", "tag"))
if err != nil {
return nil, fmt.Errorf("get snapshots for repo %v: %w", r.repoConfig.Id, err)
}
l.Debug("Forget result", zap.Int("forgot", len(result.Remove)), zap.Int("keep", len(result.Keep)))
l.Debug("Forget result", zap.Any("result", result))
var forgotten []*v1.ResticSnapshot
for _, snapshot := range result.Remove {
forgotten = append(forgotten, protoutil.SnapshotToProto(&snapshot))
snapshotProto := protoutil.SnapshotToProto(&snapshot)
if err := protoutil.ValidateSnapshot(snapshotProto); err != nil {
return nil, fmt.Errorf("snapshot validation failed: %w", err)
}
forgotten = append(forgotten, snapshotProto)
}
return forgotten, nil
-10
View File
@@ -214,16 +214,6 @@ func (r *Repo) Forget(ctx context.Context, policy *RetentionPolicy, opts ...Gene
return nil, NewCmdError(cmd, output, fmt.Errorf("invalid forget result: %w", err))
}
// then run the prune command
args = []string{"prune", "--json"}
args = append(args, r.extraArgs...)
args = append(args, opt.extraArgs...)
args = append(args, policy.toPruneFlags()...)
cmd = exec.CommandContext(ctx, r.cmd, args...)
cmd.Env = append(cmd.Env, r.buildEnv()...)
cmd.Env = append(cmd.Env, opt.extraEnv...)
return &result[0], nil
}
+1 -1
View File
@@ -48,7 +48,7 @@ export const formatDate = (time: number | string | Date) => {
};
export const formatDuration = (ms: number) => {
const seconds = Math.ceil(ms / 100);
const seconds = Math.ceil(ms / 1000);
const minutes = Math.floor(seconds / 60);
const hours = Math.floor(minutes / 60);
if (hours === 0 && minutes === 0) {