lee/backrest
1
0
Fork 0
mirror of https://github.com/garethgeorge/backrest.git synced 2026-09-23 08:25:40 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix: standardize on fully qualified snapshot_id and decouple protobufs from restic package

Browse Source
This commit is contained in:
Gareth George
2023-11-26 10:28:02 -08:00
parent 194b9495ab
commit 4f0a47267c
16 changed files with 376 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-and-test.yml
+22 -23
View File
@@ -5,36 +5,35 @@ name: Build and Test
on:
push:
branches: [ "main" ]
branches: ["main"]
pull_request:
branches: [ "main" ]
branches: ["main"]
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
- name: Setup NodeJS
uses: actions/setup-node@v4
with:
node-version: '20'
- name: Install Restic
run: sudo apt install -y restic
- name: Build WebUI
run: cd webui && npm install && npm run build
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: "1.21"
- name: Build
run: go build -v ./...
- name: Setup NodeJS
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Test
run: go test ./...
- name: Install Restic
run: sudo apt install -y restic && restic self-update --output ./restic
- name: Build WebUI
run: cd webui && npm install && npm run build
- name: Build
run: go build -v ./...
- name: Test
run: PATH=$(pwd):$PATH go test ./...
internal/api/api.go
+1 -2
View File
@@ -53,7 +53,6 @@ func loggingFunc(l *zap.Logger) logging.Logger {
})
}
func serveGRPC(ctx context.Context, socket string, server *Server) error {
lis, err := net.Listen("unix", socket)
if err != nil {
@@ -110,4 +109,4 @@ func ServeAPI(ctx context.Context, server *Server, mux *http.ServeMux) error {
apiMux := runtime.NewServeMux()
mux.Handle("/api/", http.StripPrefix("/api", apiMux))
return serveHTTPHandlers(ctx, server, apiMux)
}
}
internal/api/server.go
+2 -2
View File
@@ -13,6 +13,7 @@ import (
"github.com/garethgeorge/resticui/internal/oplog"
"github.com/garethgeorge/resticui/internal/oplog/indexutil"
"github.com/garethgeorge/resticui/internal/orchestrator"
"github.com/garethgeorge/resticui/internal/protoutil"
"github.com/garethgeorge/resticui/pkg/restic"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
@@ -110,7 +111,6 @@ func (s *Server) ListSnapshots(ctx context.Context, query *v1.ListSnapshotsReque
if err != nil {
return nil, fmt.Errorf("failed to get plan %q: %w", query.PlanId, err)
}
snapshots, err = repo.SnapshotsForPlan(ctx, plan)
} else {
snapshots, err = repo.Snapshots(ctx)
@@ -123,7 +123,7 @@ func (s *Server) ListSnapshots(ctx context.Context, query *v1.ListSnapshotsReque
// Transform the snapshots and return them.
var rs []*v1.ResticSnapshot
for _, snapshot := range snapshots {
rs = append(rs, snapshot.ToProto())
rs = append(rs, protoutil.SnapshotToProto(snapshot))
}
return &v1.ResticSnapshotList{
internal/oplog/oplog.go
+14 -11
View File
@@ -12,6 +12,8 @@ import (
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"github.com/garethgeorge/resticui/internal/oplog/indexutil"
"github.com/garethgeorge/resticui/internal/oplog/serializationutil"
"github.com/garethgeorge/resticui/internal/protoutil"
"github.com/garethgeorge/resticui/pkg/restic"
bolt "go.etcd.io/bbolt"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
@@ -35,6 +37,7 @@ var (
RepoIndexBucket = []byte("oplog.repo_idx") // repo_index tracks IDs of operations affecting a given repo
PlanIndexBucket = []byte("oplog.plan_idx") // plan_index tracks IDs of operations affecting a given plan
SnapshotIndexBucket = []byte("oplog.snapshot_idx") // snapshot_index tracks IDs of operations affecting a given snapshot
indexBuckets = [][]byte{RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket}
)
// OpLog represents a log of operations performed.
@@ -63,6 +66,11 @@ func NewOpLog(databasePath string) (*OpLog, error) {
o.nextId.Store(1)
if err := db.Update(func(tx *bolt.Tx) error {
sysBucket, err := tx.CreateBucketIfNotExists(SystemBucket)
if err != nil {
return fmt.Errorf("creating system bucket: %s", err)
}
// Create the buckets if they don't exist
for _, bucket := range [][]byte{
SystemBucket, OpLogBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket,
@@ -72,8 +80,6 @@ func NewOpLog(databasePath string) (*OpLog, error) {
}
}
sysBucket := tx.Bucket(SystemBucket)
// Validate the operation log on startup.
opLogBucket := tx.Bucket(OpLogBucket)
c := opLogBucket.Cursor()
@@ -216,7 +222,9 @@ func (o *OpLog) addOperationHelper(tx *bolt.Tx, op *v1.Operation) error {
}
}
op.SnapshotId = NormalizeSnapshotId(op.SnapshotId)
if err := protoutil.ValidateOperation(op); err != nil {
return fmt.Errorf("validating operation: %w", err)
}
bytes, err := proto.Marshal(op)
if err != nil {
@@ -315,7 +323,9 @@ func (o *OpLog) GetByPlan(planId string, collector indexutil.Collector) ([]*v1.O
}
func (o *OpLog) GetBySnapshotId(snapshotId string, collector indexutil.Collector) ([]*v1.Operation, error) {
snapshotId = NormalizeSnapshotId(snapshotId)
if err := restic.ValidateSnapshotId(snapshotId); err != nil {
return nil, err
}
var err error
var ops []*v1.Operation
o.db.View(func(tx *bolt.Tx) error {
@@ -374,10 +384,3 @@ func (o *OpLog) Unsubscribe(callback *func(EventType, *v1.Operation)) {
}
}
}
func NormalizeSnapshotId(id string) string {
if len(id) < 8 {
return id
}
return id[:8]
}
internal/oplog/oplog_test.go
+24 -11
View File
@@ -8,6 +8,11 @@ import (
"github.com/garethgeorge/resticui/internal/oplog/indexutil"
)
const (
snapshotId = "1234567890123456789012345678901234567890123456789012345678901234"
snapshotId2 = "abcdefgh01234567890123456789012345678901234567890123456789012345"
)
func TestCreate(t *testing.T) {
// t.Parallel()
log, err := NewOpLog(t.TempDir() + "/test.boltdb")
@@ -21,7 +26,6 @@ func TestCreate(t *testing.T) {
}
func TestAddOperation(t *testing.T) {
// t.Parallel()
log, err := NewOpLog(t.TempDir() + "/test.boltdb")
if err != nil {
t.Fatalf("error creating oplog: %s", err)
@@ -38,12 +42,14 @@ func TestAddOperation(t *testing.T) {
op: &v1.Operation{
UnixTimeStartMs: 1234,
},
wantErr: false,
wantErr: true,
},
{
name: "basic backup operation",
op: &v1.Operation{
UnixTimeStartMs: 1234,
RepoId: "testrepo",
PlanId: "testplan",
Op: &v1.Operation_OperationBackup{},
},
wantErr: false,
@@ -52,6 +58,8 @@ func TestAddOperation(t *testing.T) {
name: "basic snapshot operation",
op: &v1.Operation{
UnixTimeStartMs: 1234,
RepoId: "testrepo",
PlanId: "testplan",
Op: &v1.Operation_OperationIndexSnapshot{
OperationIndexSnapshot: &v1.OperationIndexSnapshot{
Snapshot: &v1.ResticSnapshot{
@@ -66,31 +74,36 @@ func TestAddOperation(t *testing.T) {
name: "operation with ID",
op: &v1.Operation{
Id: 1,
RepoId: "testrepo",
PlanId: "testplan",
UnixTimeStartMs: 1234,
Op: &v1.Operation_OperationBackup{},
},
wantErr: true,
},
{
name: "operation with repo",
name: "operation with repo only",
op: &v1.Operation{
UnixTimeStartMs: 1234,
RepoId: "testrepo",
Op: &v1.Operation_OperationBackup{},
},
wantErr: true,
},
{
name: "operation with plan",
name: "operation with plan only",
op: &v1.Operation{
UnixTimeStartMs: 1234,
PlanId: "testplan",
Op: &v1.Operation_OperationBackup{},
},
wantErr: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
if err := log.Add(tc.op); (err != nil) != tc.wantErr {
t.Errorf("Add() error = %v, wantErr %v", err, tc.wantErr)
}
@@ -250,14 +263,14 @@ func TestIndexSnapshot(t *testing.T) {
UnixTimeStartMs: 1234,
PlanId: "plan1",
RepoId: "repo1",
SnapshotId: "abcdefgh",
SnapshotId: snapshotId,
Op: &v1.Operation_OperationIndexSnapshot{},
}
if err := log.Add(op); err != nil {
t.Fatalf("error adding operation: %s", err)
}
ops, err := log.GetBySnapshotId("abcdefgh", indexutil.CollectAll())
ops, err := log.GetBySnapshotId(snapshotId, indexutil.CollectAll())
if err != nil {
t.Fatalf("error checking for snapshot: %s", err)
}
@@ -282,7 +295,7 @@ func TestUpdateOperation(t *testing.T) {
UnixTimeStartMs: 1234,
PlanId: "oldplan",
RepoId: "oldrepo",
SnapshotId: "12345678",
SnapshotId: snapshotId,
}
if err := log.Add(op); err != nil {
t.Fatalf("error adding operation: %s", err)
@@ -300,14 +313,14 @@ func TestUpdateOperation(t *testing.T) {
} else if len(ops) != 1 {
t.Fatalf("want 1 operation, got %d", len(ops))
}
if ops, err := log.GetBySnapshotId("12345678", indexutil.CollectAll()); err != nil {
if ops, err := log.GetBySnapshotId(snapshotId, indexutil.CollectAll()); err != nil {
t.Fatalf("error checking for snapshot: %s", err)
} else if len(ops) != 1 {
t.Fatalf("want 1 operation, got %d", len(ops))
}
// Update indexed values
op.SnapshotId = "abcdefgh"
op.SnapshotId = snapshotId2
op.PlanId = "myplan"
op.RepoId = "myrepo"
if err := log.Update(op); err != nil {
@@ -318,7 +331,7 @@ func TestUpdateOperation(t *testing.T) {
if opId != op.Id {
t.Errorf("want operation ID %d, got %d", opId, op.Id)
}
if ops, err := log.GetBySnapshotId("abcdefgh", indexutil.CollectAll()); err != nil {
if ops, err := log.GetBySnapshotId(snapshotId2, indexutil.CollectAll()); err != nil {
t.Fatalf("error checking for snapshot: %s", err)
} else if len(ops) != 1 {
t.Fatalf("want 1 operation, got %d", len(ops))
@@ -347,7 +360,7 @@ func TestUpdateOperation(t *testing.T) {
} else if len(ops) != 0 {
t.Fatalf("want 0 operations, got %d", len(ops))
}
if ops, err := log.GetBySnapshotId("12345678", indexutil.CollectAll()); err != nil {
if ops, err := log.GetBySnapshotId(snapshotId, indexutil.CollectAll()); err != nil {
t.Fatalf("error checking for snapshot: %s", err)
} else if len(ops) != 0 {
t.Fatalf("want 0 operations, got %d", len(ops))
internal/orchestrator/repo_test.go
+1 -1
View File
@@ -108,7 +108,7 @@ func TestSnapshotParenting(t *testing.T) {
prev := snapshots[i-1]
curr := snapshots[i]
if prev.ToProto().UnixTimeMs >= curr.ToProto().UnixTimeMs {
if prev.UnixTimeMs() >= curr.UnixTimeMs() {
t.Errorf("snapshots are out of order")
}
internal/orchestrator/tasks.go
+7 -3
View File
@@ -8,6 +8,7 @@ import (
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"github.com/garethgeorge/resticui/internal/oplog"
"github.com/garethgeorge/resticui/internal/oplog/indexutil"
"github.com/garethgeorge/resticui/internal/protoutil"
"github.com/garethgeorge/resticui/pkg/restic"
"github.com/gitploy-io/cronexpr"
"github.com/hashicorp/go-multierror"
@@ -114,7 +115,7 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan
}
lastSent = time.Now()
backupOp.OperationBackup.LastStatus = entry.ToProto()
backupOp.OperationBackup.LastStatus = protoutil.BackupProgressEntryToProto(entry)
if err := orchestrator.OpLog.Update(op); err != nil {
zap.S().Errorf("failed to update oplog with progress for backup: %v", err)
}
@@ -125,7 +126,10 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan
}
op.SnapshotId = summary.SnapshotId
backupOp.OperationBackup.LastStatus = summary.ToProto()
backupOp.OperationBackup.LastStatus = protoutil.BackupProgressEntryToProto(summary)
if backupOp.OperationBackup.LastStatus == nil {
return fmt.Errorf("expected a final backup progress entry, got nil")
}
zap.L().Info("backup complete", zap.String("plan", plan.Id), zap.Duration("duration", time.Since(startTime)))
return nil
@@ -167,7 +171,7 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, plan
continue
}
snapshotProto := snapshot.ToProto()
snapshotProto := protoutil.SnapshotToProto(snapshot)
indexOps = append(indexOps, &v1.Operation{
RepoId: plan.Repo,
PlanId: plan.Id,
internal/protoutil/conversion.go
+73
View File
@@ -0,0 +1,73 @@
package protoutil
import (
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"github.com/garethgeorge/resticui/pkg/restic"
)
func SnapshotToProto(s *restic.Snapshot) *v1.ResticSnapshot {
return &v1.ResticSnapshot{
Id: s.Id,
UnixTimeMs: s.UnixTimeMs(),
Tree: s.Tree,
Paths: s.Paths,
Hostname: s.Hostname,
Username: s.Username,
Tags: s.Tags,
Parent: s.Parent,
}
}
func LsEntryToProto(e *restic.LsEntry) *v1.LsEntry {
return &v1.LsEntry{
Name: e.Name,
Type: e.Type,
Path: e.Path,
Uid: int64(e.Uid),
Gid: int64(e.Gid),
Size: int64(e.Size),
Mode: int64(e.Mode),
Mtime: e.Mtime,
Atime: e.Atime,
Ctime: e.Ctime,
}
}
func BackupProgressEntryToProto(b *restic.BackupProgressEntry) *v1.BackupProgressEntry {
switch b.MessageType {
case "summary":
return &v1.BackupProgressEntry{
Entry: &v1.BackupProgressEntry_Summary{
Summary: &v1.BackupProgressSummary{
FilesNew: int64(b.FilesNew),
FilesChanged: int64(b.FilesChanged),
FilesUnmodified: int64(b.FilesUnmodified),
DirsNew: int64(b.DirsNew),
DirsChanged: int64(b.DirsChanged),
DirsUnmodified: int64(b.DirsUnmodified),
DataBlobs: int64(b.DataBlobs),
TreeBlobs: int64(b.TreeBlobs),
DataAdded: int64(b.DataAdded),
TotalFilesProcessed: int64(b.TotalFilesProcessed),
TotalBytesProcessed: int64(b.TotalBytesProcessed),
TotalDuration: float64(b.TotalDuration),
SnapshotId: b.SnapshotId,
},
},
}
case "status":
return &v1.BackupProgressEntry{
Entry: &v1.BackupProgressEntry_Status{
Status: &v1.BackupProgressStatusEntry{
PercentDone: b.PercentDone,
TotalFiles: int64(b.TotalFiles),
FilesDone: int64(b.FilesDone),
TotalBytes: int64(b.TotalBytes),
BytesDone: int64(b.BytesDone),
},
},
}
default:
return nil
}
}
internal/protoutil/conversion_test.go
+118
View File
@@ -0,0 +1,118 @@
package protoutil
import (
"testing"
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"github.com/garethgeorge/resticui/pkg/restic"
"google.golang.org/protobuf/proto"
)
func TestSnapshotToProto(t *testing.T) {
snapshot := &restic.Snapshot{
Id: "db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52",
Time: "2023-11-10T19:14:17.053824063-08:00",
Tree: "3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2",
Paths: []string{"/resticui"},
Hostname: "pop-os",
Username: "dontpanic",
Tags: []string{},
Parent: "",
}
want := &v1.ResticSnapshot{
Id: "db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52",
UnixTimeMs: 1699672457053,
Tree: "3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2",
Paths: []string{"/resticui"},
Hostname: "pop-os",
Username: "dontpanic",
Tags: []string{},
Parent: "",
}
got := SnapshotToProto(snapshot)
if !proto.Equal(want, got) {
t.Errorf("wanted %+v, got: %+v", want, got)
}
}
func TestBackupProgressEntryToProto(t *testing.T) {
cases := []struct {
name string
entry *restic.BackupProgressEntry
want *v1.BackupProgressEntry
}{
{
name: "summary",
entry: &restic.BackupProgressEntry{
MessageType: "summary",
FilesNew: 1,
FilesChanged: 2,
FilesUnmodified: 3,
DirsNew: 4,
DirsChanged: 5,
DirsUnmodified: 6,
DataBlobs: 7,
TreeBlobs: 8,
DataAdded: 9,
TotalFilesProcessed: 10,
TotalBytesProcessed: 11,
TotalDuration: 12.0,
SnapshotId: "db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52",
PercentDone: 13.0, // should be ignored.
},
want: &v1.BackupProgressEntry{
Entry: &v1.BackupProgressEntry_Summary{
Summary: &v1.BackupProgressSummary{
FilesNew: 1,
FilesChanged: 2,
FilesUnmodified: 3,
DirsNew: 4,
DirsChanged: 5,
DirsUnmodified: 6,
DataBlobs: 7,
TreeBlobs: 8,
DataAdded: 9,
TotalFilesProcessed: 10,
TotalBytesProcessed: 11,
TotalDuration: 12.0,
SnapshotId: "db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52",
},
},
},
},
{
name: "status",
entry: &restic.BackupProgressEntry{
MessageType: "status",
PercentDone: 13.0,
TotalFiles: 14,
FilesDone: 15,
TotalBytes: 16,
BytesDone: 17,
},
want: &v1.BackupProgressEntry{
Entry: &v1.BackupProgressEntry_Status{
Status: &v1.BackupProgressStatusEntry{
PercentDone: 13.0,
TotalFiles: 14,
FilesDone: 15,
TotalBytes: 16,
BytesDone: 17,
},
},
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := BackupProgressEntryToProto(c.entry)
if !proto.Equal(got, c.want) {
t.Errorf("wanted: %+v, got: %+v", c.want, got)
}
})
}
}
internal/protoutil/validation.go
+42
View File
@@ -0,0 +1,42 @@
package protoutil
import (
"errors"
"fmt"
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"github.com/garethgeorge/resticui/pkg/restic"
)
// ValidateOperation verifies critical properties of the operation proto.
func ValidateOperation(op *v1.Operation) error {
if op.Id == 0 {
return errors.New("operation.id is required")
}
if op.RepoId == "" {
return errors.New("operation.repo_id is required")
}
if op.PlanId == "" {
return errors.New("operation.plan_id is required")
}
if op.SnapshotId != "" {
if err := restic.ValidateSnapshotId(op.SnapshotId); err != nil {
return fmt.Errorf("operation.snapshot_id is invalid: %w", err)
}
}
return nil
}
// ValidateSnapshot verifies critical properties of the snapshot proto representation.
func ValidateSnapshot(s *v1.ResticSnapshot) error {
if s.Id == "" {
return errors.New("snapshot.id is required")
}
if s.UnixTimeMs == 0 {
return errors.New("snapshot.unix_time_ms must be non-zero")
}
if err := restic.ValidateSnapshotId(s.Id); err != nil {
return err
}
return nil
}
pkg/restic/outputs.go
+46 -49
View File
@@ -3,9 +3,9 @@ package restic
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"os/exec"
"slices"
"time"
@@ -25,19 +25,6 @@ type Snapshot struct {
unixTimeMs int64 `json:"-"`
}
func (s *Snapshot) ToProto() *v1.ResticSnapshot {
return &v1.ResticSnapshot{
Id: s.Id,
UnixTimeMs: s.UnixTimeMs(),
Tree: s.Tree,
Paths: s.Paths,
Hostname: s.Hostname,
Username: s.Username,
Tags: s.Tags,
Parent: s.Parent,
}
}
func (s *Snapshot) UnixTimeMs() int64 {
if s.unixTimeMs != 0 {
return s.unixTimeMs
@@ -50,6 +37,16 @@ func (s *Snapshot) UnixTimeMs() int64 {
return s.unixTimeMs
}
func (s *Snapshot) Validate() error {
if err := ValidateSnapshotId(s.Id); err != nil {
return fmt.Errorf("snapshot.id invalid: %v", err)
}
if s.Time == "" || s.UnixTimeMs() == 0 {
return fmt.Errorf("snapshot.time invalid: %v", s.Time)
}
return nil
}
type BackupProgressEntry struct {
// Common fields
MessageType string `json:"message_type"` // "summary" or "status"
@@ -77,44 +74,17 @@ type BackupProgressEntry struct {
BytesDone int `json:"bytes_done"`
}
func (b *BackupProgressEntry) ToProto() *v1.BackupProgressEntry {
switch b.MessageType {
case "summary":
return &v1.BackupProgressEntry{
Entry: &v1.BackupProgressEntry_Summary{
Summary: &v1.BackupProgressSummary{
FilesNew: int64(b.FilesNew),
FilesChanged: int64(b.FilesChanged),
FilesUnmodified: int64(b.FilesUnmodified),
DirsNew: int64(b.DirsNew),
DirsChanged: int64(b.DirsChanged),
DirsUnmodified: int64(b.DirsUnmodified),
DataBlobs: int64(b.DataBlobs),
TreeBlobs: int64(b.TreeBlobs),
DataAdded: int64(b.DataAdded),
TotalFilesProcessed: int64(b.TotalFilesProcessed),
TotalBytesProcessed: int64(b.TotalBytesProcessed),
TotalDuration: float64(b.TotalDuration),
SnapshotId: b.SnapshotId,
},
},
func (b *BackupProgressEntry) Validate() error {
if b.MessageType == "summary" {
if b.SnapshotId == "" {
return errors.New("summary message must have snapshot_id")
}
case "status":
return &v1.BackupProgressEntry{
Entry: &v1.BackupProgressEntry_Status{
Status: &v1.BackupProgressStatusEntry{
PercentDone: b.PercentDone,
TotalFiles: int64(b.TotalFiles),
FilesDone: int64(b.FilesDone),
TotalBytes: int64(b.TotalBytes),
BytesDone: int64(b.BytesDone),
},
},
if err := ValidateSnapshotId(b.SnapshotId); err != nil {
return err
}
default:
log.Fatalf("unknown message type: %s", b.MessageType)
return nil
}
return nil
}
// readBackupProgressEntrys returns the summary event or an error if the command failed.
@@ -134,6 +104,9 @@ func readBackupProgressEntries(cmd *exec.Cmd, output io.Reader, callback func(ev
return nil, NewCmdError(cmd, bytes, fmt.Errorf("command output was not JSON: %w", err))
}
if err := event.Validate(); err != nil {
return nil, err
}
}
// remaining events are parsed as JSON
@@ -144,6 +117,9 @@ func readBackupProgressEntries(cmd *exec.Cmd, output io.Reader, callback func(ev
if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
return nil, fmt.Errorf("failed to parse JSON: %w", err)
}
if err := event.Validate(); err != nil {
return nil, err
}
if callback != nil {
callback(event)
@@ -217,3 +193,24 @@ type ForgetResult struct {
Keep []Snapshot `json:"keep"`
Remove []Snapshot `json:"remove"`
}
func (r *ForgetResult) Validate() error {
for _, s := range r.Keep {
if err := ValidateSnapshotId(s.Id); err != nil {
return err
}
}
for _, s := range r.Remove {
if err := ValidateSnapshotId(s.Id); err != nil {
return err
}
}
return nil
}
func ValidateSnapshotId(id string) error {
if len(id) != 64 {
return fmt.Errorf("restic may be out of date (check with `restic self-upgrade`): snapshot ID must be 64 chars, got %v chars", len(id))
}
return nil
}
pkg/restic/outputs_test.go
+1 -39
View File
@@ -9,7 +9,7 @@ import (
func TestReadBackupProgressEntries(t *testing.T) {
t.Parallel()
testInput := `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":15}
{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":166,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":128,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":166,"total_bytes_processed":16754463,"total_duration":0.235433378,"snapshot_id":"bca1043e"}`
{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":166,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":128,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":166,"total_bytes_processed":16754463,"total_duration":0.235433378,"id":"d4558b360cc1b7966e416e010382ab8feb49d14da7832266832d69a43af10147"}`
b := bytes.NewBuffer([]byte(testInput))
@@ -27,7 +27,6 @@ func TestReadBackupProgressEntries(t *testing.T) {
}
}
func TestReadLs(t *testing.T) {
testInput := `{"time":"2023-11-10T19:14:17.053824063-08:00","tree":"3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2","paths":["/resticui"],"hostname":"pop-os","username":"dontpanic","uid":1000,"gid":1000,"id":"db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52","short_id":"db155169","struct_type":"snapshot"}
{"name":".git","type":"dir","path":"/.git","uid":1000,"gid":1000,"mode":2147484157,"mtime":"2023-11-10T18:32:38.156599473-08:00","atime":"2023-11-10T18:32:38.156599473-08:00","ctime":"2023-11-10T18:32:38.156599473-08:00","struct_type":"node"}
@@ -47,40 +46,3 @@ func TestReadLs(t *testing.T) {
t.Errorf("wanted 3 entries, got: %d", len(entries))
}
}
func TestSnapshotToProto(t *testing.T) {
snapshot := &Snapshot{
Id: "db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52",
Time: "2023-11-10T19:14:17.053824063-08:00",
Tree: "3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2",
Paths: []string{"/resticui"},
Hostname: "pop-os",
Username: "dontpanic",
Tags: []string{},
Parent: "",
}
proto := snapshot.ToProto()
if proto.Id != snapshot.Id {
t.Errorf("wanted id %q, got: %q", snapshot.Id, proto.Id)
}
if proto.Tree != snapshot.Tree {
t.Errorf("wanted tree %q, got: %q", snapshot.Tree, proto.Tree)
}
if proto.Hostname != snapshot.Hostname {
t.Errorf("wanted hostname %q, got: %q", snapshot.Hostname, proto.Hostname)
}
if proto.Username != snapshot.Username {
t.Errorf("wanted username %q, got: %q", snapshot.Username, proto.Username)
}
if len(proto.Tags) != len(snapshot.Tags) {
t.Errorf("wanted %d tags, got: %d", len(snapshot.Tags), len(proto.Tags))
}
if proto.Parent != snapshot.Parent {
t.Errorf("wanted parent %q, got: %q", snapshot.Parent, proto.Parent)
}
if proto.UnixTimeMs != 1699672457053 {
t.Errorf("wanted unix time %d, got: %d", 1699672457053, proto.UnixTimeMs)
}
}
pkg/restic/restic.go
+8 -1
View File
@@ -167,7 +167,11 @@ func (r *Repo) Snapshots(ctx context.Context, opts ...GenericOption) ([]*Snapsho
if err := json.Unmarshal(output, &snapshots); err != nil {
return nil, NewCmdError(cmd, output, fmt.Errorf("command output is not valid JSON: %w", err))
}
for _, snapshot := range snapshots {
if err := snapshot.Validate(); err != nil {
return nil, fmt.Errorf("invalid snapshot: %w", err)
}
}
return snapshots, nil
}
@@ -199,6 +203,9 @@ func (r *Repo) Forget(ctx context.Context, policy RetentionPolicy, pruneOutput i
if len(result) != 1 {
return nil, fmt.Errorf("expected 1 output from forget, got %v", len(result))
}
if err := result[0].Validate(); err != nil {
return nil, NewCmdError(cmd, output, fmt.Errorf("invalid forget result: %w", err))
}
// then run the prune command
args = []string{"prune", "--json"}
pkg/restic/restic_test.go
+2 -2
View File
@@ -155,8 +155,8 @@ func TestSnapshot(t *testing.T) {
// Ensure that snapshot timestamps are set, this is critical for correct ordering in the orchestrator.
for _, snapshot := range snapshots {
if p := snapshot.ToProto(); p.UnixTimeMs == 0 {
t.Errorf("wanted snapshot time to be non-zero, got: %v", p.UnixTimeMs)
if snapshot.UnixTimeMs() == 0 {
t.Errorf("wanted snapshot time to be non-zero, got: %v", snapshot.UnixTimeMs())
}
}
})
webui/src/components/OperationList.tsx
+7 -9
View File
@@ -162,15 +162,12 @@ export const OperationRow = ({
<>
<Collapse
size="small"
defaultActiveKey={
operation.status === OperationStatus.STATUS_INPROGRESS
? [1]
: undefined
}
destroyInactivePanel
defaultActiveKey={[1]}
items={[
{
key: 1,
label: "Details",
label: "Backup Details",
children: (
<BackupOperationStatus status={backupOp.lastStatus} />
),
@@ -213,10 +210,11 @@ const SnapshotInfo = ({
return (
<Collapse
size="small"
defaultActiveKey={[1]}
items={[
{
key: 1,
label: "Details",
label: "Snapshot Details",
children: (
<>
<Typography.Text>
@@ -245,7 +243,7 @@ const SnapshotInfo = ({
},
{
key: 2,
label: "Browse",
label: "Browse and Restore Files in Backup",
children: (
<SnapshotBrowser snapshotId={snapshot.id!} repoId={repoId} />
),
@@ -294,7 +292,7 @@ const BackupOperationStatus = ({
<>
<Typography.Text>
<Typography.Text strong>Snapshot ID: </Typography.Text>
{sum.snapshotId}
{normalizeSnapshotId(sum.snapshotId!)}
</Typography.Text>
<Row gutter={16}>
<Col span={8}>
webui/src/components/OperationTree.tsx
+8 -6
View File
@@ -8,7 +8,7 @@ import {
toEop,
unsubscribeFromOperations,
} from "../state/oplog";
import { Col, Empty, Row, Tree } from "antd";
import { Col, Divider, Empty, Row, Tree } from "antd";
import _ from "lodash";
import { DataNode } from "antd/es/tree";
import {
@@ -112,11 +112,13 @@ export const OperationTree = ({
showIcon
defaultExpandedKeys={[backups[0].id!]}
onSelect={(keys, info) => {
setSelectedBackupId(
info.selectedNodes.length > 0
? info.selectedNodes[0].backup!.id!
: null
);
if (info.selectedNodes.length === 0) return;
const backup = info.selectedNodes[0].backup;
if (!backup) {
setSelectedBackupId(null);
return;
}
setSelectedBackupId(backup.id!);
}}
titleRender={(node: OpTreeNode): React.ReactNode => {
if (node.title) {