diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 9e213118..0b9017af 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -5,36 +5,35 @@ name: Build and Test
on:
push:
- branches: [ "main" ]
+ branches: ["main"]
pull_request:
- branches: [ "main" ]
+ branches: ["main"]
workflow_dispatch:
jobs:
-
build:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v3
- - name: Set up Go
- uses: actions/setup-go@v4
- with:
- go-version: '1.21'
-
- - name: Setup NodeJS
- uses: actions/setup-node@v4
- with:
- node-version: '20'
-
- - name: Install Restic
- run: sudo apt install -y restic
-
- - name: Build WebUI
- run: cd webui && npm install && npm run build
+ - name: Set up Go
+ uses: actions/setup-go@v4
+ with:
+ go-version: "1.21"
- - name: Build
- run: go build -v ./...
+ - name: Setup NodeJS
+ uses: actions/setup-node@v4
+ with:
+ node-version: "20"
- - name: Test
- run: go test ./...
+ - name: Install Restic
+ run: sudo apt install -y restic && restic self-update --output ./restic
+
+ - name: Build WebUI
+ run: cd webui && npm install && npm run build
+
+ - name: Build
+ run: go build -v ./...
+
+ - name: Test
+ run: PATH=$(pwd):$PATH go test ./...
diff --git a/internal/api/api.go b/internal/api/api.go
index 8f1f1cb8..88ce13c2 100644
--- a/internal/api/api.go
+++ b/internal/api/api.go
@@ -53,7 +53,6 @@ func loggingFunc(l *zap.Logger) logging.Logger {
})
}
-
func serveGRPC(ctx context.Context, socket string, server *Server) error {
lis, err := net.Listen("unix", socket)
if err != nil {
@@ -110,4 +109,4 @@ func ServeAPI(ctx context.Context, server *Server, mux *http.ServeMux) error {
apiMux := runtime.NewServeMux()
mux.Handle("/api/", http.StripPrefix("/api", apiMux))
return serveHTTPHandlers(ctx, server, apiMux)
-}
\ No newline at end of file
+}
diff --git a/internal/api/server.go b/internal/api/server.go
index 87eac8fe..63fa5b50 100644
--- a/internal/api/server.go
+++ b/internal/api/server.go
@@ -13,6 +13,7 @@ import (
"github.com/garethgeorge/resticui/internal/oplog"
"github.com/garethgeorge/resticui/internal/oplog/indexutil"
"github.com/garethgeorge/resticui/internal/orchestrator"
+ "github.com/garethgeorge/resticui/internal/protoutil"
"github.com/garethgeorge/resticui/pkg/restic"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
@@ -110,7 +111,6 @@ func (s *Server) ListSnapshots(ctx context.Context, query *v1.ListSnapshotsReque
if err != nil {
return nil, fmt.Errorf("failed to get plan %q: %w", query.PlanId, err)
}
-
snapshots, err = repo.SnapshotsForPlan(ctx, plan)
} else {
snapshots, err = repo.Snapshots(ctx)
@@ -123,7 +123,7 @@ func (s *Server) ListSnapshots(ctx context.Context, query *v1.ListSnapshotsReque
// Transform the snapshots and return them.
var rs []*v1.ResticSnapshot
for _, snapshot := range snapshots {
- rs = append(rs, snapshot.ToProto())
+ rs = append(rs, protoutil.SnapshotToProto(snapshot))
}
return &v1.ResticSnapshotList{
diff --git a/internal/oplog/oplog.go b/internal/oplog/oplog.go
index b4c10c3a..826457d6 100644
--- a/internal/oplog/oplog.go
+++ b/internal/oplog/oplog.go
@@ -12,6 +12,8 @@ import (
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"github.com/garethgeorge/resticui/internal/oplog/indexutil"
"github.com/garethgeorge/resticui/internal/oplog/serializationutil"
+ "github.com/garethgeorge/resticui/internal/protoutil"
+ "github.com/garethgeorge/resticui/pkg/restic"
bolt "go.etcd.io/bbolt"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
@@ -35,6 +37,7 @@ var (
RepoIndexBucket = []byte("oplog.repo_idx") // repo_index tracks IDs of operations affecting a given repo
PlanIndexBucket = []byte("oplog.plan_idx") // plan_index tracks IDs of operations affecting a given plan
SnapshotIndexBucket = []byte("oplog.snapshot_idx") // snapshot_index tracks IDs of operations affecting a given snapshot
+ indexBuckets = [][]byte{RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket}
)
// OpLog represents a log of operations performed.
@@ -63,6 +66,11 @@ func NewOpLog(databasePath string) (*OpLog, error) {
o.nextId.Store(1)
if err := db.Update(func(tx *bolt.Tx) error {
+ sysBucket, err := tx.CreateBucketIfNotExists(SystemBucket)
+ if err != nil {
+ return fmt.Errorf("creating system bucket: %s", err)
+ }
+
// Create the buckets if they don't exist
for _, bucket := range [][]byte{
SystemBucket, OpLogBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket,
@@ -72,8 +80,6 @@ func NewOpLog(databasePath string) (*OpLog, error) {
}
}
- sysBucket := tx.Bucket(SystemBucket)
-
// Validate the operation log on startup.
opLogBucket := tx.Bucket(OpLogBucket)
c := opLogBucket.Cursor()
@@ -216,7 +222,9 @@ func (o *OpLog) addOperationHelper(tx *bolt.Tx, op *v1.Operation) error {
}
}
- op.SnapshotId = NormalizeSnapshotId(op.SnapshotId)
+ if err := protoutil.ValidateOperation(op); err != nil {
+ return fmt.Errorf("validating operation: %w", err)
+ }
bytes, err := proto.Marshal(op)
if err != nil {
@@ -315,7 +323,9 @@ func (o *OpLog) GetByPlan(planId string, collector indexutil.Collector) ([]*v1.O
}
func (o *OpLog) GetBySnapshotId(snapshotId string, collector indexutil.Collector) ([]*v1.Operation, error) {
- snapshotId = NormalizeSnapshotId(snapshotId)
+ if err := restic.ValidateSnapshotId(snapshotId); err != nil {
+ return nil, err
+ }
var err error
var ops []*v1.Operation
o.db.View(func(tx *bolt.Tx) error {
@@ -374,10 +384,3 @@ func (o *OpLog) Unsubscribe(callback *func(EventType, *v1.Operation)) {
}
}
}
-
-func NormalizeSnapshotId(id string) string {
- if len(id) < 8 {
- return id
- }
- return id[:8]
-}
diff --git a/internal/oplog/oplog_test.go b/internal/oplog/oplog_test.go
index 215ec36c..30db9290 100644
--- a/internal/oplog/oplog_test.go
+++ b/internal/oplog/oplog_test.go
@@ -8,6 +8,11 @@ import (
"github.com/garethgeorge/resticui/internal/oplog/indexutil"
)
+const (
+ snapshotId = "1234567890123456789012345678901234567890123456789012345678901234"
+ snapshotId2 = "abcdefgh01234567890123456789012345678901234567890123456789012345"
+)
+
func TestCreate(t *testing.T) {
// t.Parallel()
log, err := NewOpLog(t.TempDir() + "/test.boltdb")
@@ -21,7 +26,6 @@ func TestCreate(t *testing.T) {
}
func TestAddOperation(t *testing.T) {
- // t.Parallel()
log, err := NewOpLog(t.TempDir() + "/test.boltdb")
if err != nil {
t.Fatalf("error creating oplog: %s", err)
@@ -38,12 +42,14 @@ func TestAddOperation(t *testing.T) {
op: &v1.Operation{
UnixTimeStartMs: 1234,
},
- wantErr: false,
+ wantErr: true,
},
{
name: "basic backup operation",
op: &v1.Operation{
UnixTimeStartMs: 1234,
+ RepoId: "testrepo",
+ PlanId: "testplan",
Op: &v1.Operation_OperationBackup{},
},
wantErr: false,
@@ -52,6 +58,8 @@ func TestAddOperation(t *testing.T) {
name: "basic snapshot operation",
op: &v1.Operation{
UnixTimeStartMs: 1234,
+ RepoId: "testrepo",
+ PlanId: "testplan",
Op: &v1.Operation_OperationIndexSnapshot{
OperationIndexSnapshot: &v1.OperationIndexSnapshot{
Snapshot: &v1.ResticSnapshot{
@@ -66,31 +74,36 @@ func TestAddOperation(t *testing.T) {
name: "operation with ID",
op: &v1.Operation{
Id: 1,
+ RepoId: "testrepo",
+ PlanId: "testplan",
UnixTimeStartMs: 1234,
Op: &v1.Operation_OperationBackup{},
},
wantErr: true,
},
{
- name: "operation with repo",
+ name: "operation with repo only",
op: &v1.Operation{
UnixTimeStartMs: 1234,
RepoId: "testrepo",
Op: &v1.Operation_OperationBackup{},
},
+ wantErr: true,
},
{
- name: "operation with plan",
+ name: "operation with plan only",
op: &v1.Operation{
UnixTimeStartMs: 1234,
PlanId: "testplan",
Op: &v1.Operation_OperationBackup{},
},
+ wantErr: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
if err := log.Add(tc.op); (err != nil) != tc.wantErr {
t.Errorf("Add() error = %v, wantErr %v", err, tc.wantErr)
}
@@ -250,14 +263,14 @@ func TestIndexSnapshot(t *testing.T) {
UnixTimeStartMs: 1234,
PlanId: "plan1",
RepoId: "repo1",
- SnapshotId: "abcdefgh",
+ SnapshotId: snapshotId,
Op: &v1.Operation_OperationIndexSnapshot{},
}
if err := log.Add(op); err != nil {
t.Fatalf("error adding operation: %s", err)
}
- ops, err := log.GetBySnapshotId("abcdefgh", indexutil.CollectAll())
+ ops, err := log.GetBySnapshotId(snapshotId, indexutil.CollectAll())
if err != nil {
t.Fatalf("error checking for snapshot: %s", err)
}
@@ -282,7 +295,7 @@ func TestUpdateOperation(t *testing.T) {
UnixTimeStartMs: 1234,
PlanId: "oldplan",
RepoId: "oldrepo",
- SnapshotId: "12345678",
+ SnapshotId: snapshotId,
}
if err := log.Add(op); err != nil {
t.Fatalf("error adding operation: %s", err)
@@ -300,14 +313,14 @@ func TestUpdateOperation(t *testing.T) {
} else if len(ops) != 1 {
t.Fatalf("want 1 operation, got %d", len(ops))
}
- if ops, err := log.GetBySnapshotId("12345678", indexutil.CollectAll()); err != nil {
+ if ops, err := log.GetBySnapshotId(snapshotId, indexutil.CollectAll()); err != nil {
t.Fatalf("error checking for snapshot: %s", err)
} else if len(ops) != 1 {
t.Fatalf("want 1 operation, got %d", len(ops))
}
// Update indexed values
- op.SnapshotId = "abcdefgh"
+ op.SnapshotId = snapshotId2
op.PlanId = "myplan"
op.RepoId = "myrepo"
if err := log.Update(op); err != nil {
@@ -318,7 +331,7 @@ func TestUpdateOperation(t *testing.T) {
if opId != op.Id {
t.Errorf("want operation ID %d, got %d", opId, op.Id)
}
- if ops, err := log.GetBySnapshotId("abcdefgh", indexutil.CollectAll()); err != nil {
+ if ops, err := log.GetBySnapshotId(snapshotId2, indexutil.CollectAll()); err != nil {
t.Fatalf("error checking for snapshot: %s", err)
} else if len(ops) != 1 {
t.Fatalf("want 1 operation, got %d", len(ops))
@@ -347,7 +360,7 @@ func TestUpdateOperation(t *testing.T) {
} else if len(ops) != 0 {
t.Fatalf("want 0 operations, got %d", len(ops))
}
- if ops, err := log.GetBySnapshotId("12345678", indexutil.CollectAll()); err != nil {
+ if ops, err := log.GetBySnapshotId(snapshotId, indexutil.CollectAll()); err != nil {
t.Fatalf("error checking for snapshot: %s", err)
} else if len(ops) != 0 {
t.Fatalf("want 0 operations, got %d", len(ops))
diff --git a/internal/orchestrator/repo_test.go b/internal/orchestrator/repo_test.go
index 6994beea..a8fa42a1 100644
--- a/internal/orchestrator/repo_test.go
+++ b/internal/orchestrator/repo_test.go
@@ -108,7 +108,7 @@ func TestSnapshotParenting(t *testing.T) {
prev := snapshots[i-1]
curr := snapshots[i]
- if prev.ToProto().UnixTimeMs >= curr.ToProto().UnixTimeMs {
+ if prev.UnixTimeMs() >= curr.UnixTimeMs() {
t.Errorf("snapshots are out of order")
}
diff --git a/internal/orchestrator/tasks.go b/internal/orchestrator/tasks.go
index eb574506..5d4044f8 100644
--- a/internal/orchestrator/tasks.go
+++ b/internal/orchestrator/tasks.go
@@ -8,6 +8,7 @@ import (
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"github.com/garethgeorge/resticui/internal/oplog"
"github.com/garethgeorge/resticui/internal/oplog/indexutil"
+ "github.com/garethgeorge/resticui/internal/protoutil"
"github.com/garethgeorge/resticui/pkg/restic"
"github.com/gitploy-io/cronexpr"
"github.com/hashicorp/go-multierror"
@@ -114,7 +115,7 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan
}
lastSent = time.Now()
- backupOp.OperationBackup.LastStatus = entry.ToProto()
+ backupOp.OperationBackup.LastStatus = protoutil.BackupProgressEntryToProto(entry)
if err := orchestrator.OpLog.Update(op); err != nil {
zap.S().Errorf("failed to update oplog with progress for backup: %v", err)
}
@@ -125,7 +126,10 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan
}
op.SnapshotId = summary.SnapshotId
- backupOp.OperationBackup.LastStatus = summary.ToProto()
+ backupOp.OperationBackup.LastStatus = protoutil.BackupProgressEntryToProto(summary)
+ if backupOp.OperationBackup.LastStatus == nil {
+ return fmt.Errorf("expected a final backup progress entry, got nil")
+ }
zap.L().Info("backup complete", zap.String("plan", plan.Id), zap.Duration("duration", time.Since(startTime)))
return nil
@@ -167,7 +171,7 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, plan
continue
}
- snapshotProto := snapshot.ToProto()
+ snapshotProto := protoutil.SnapshotToProto(snapshot)
indexOps = append(indexOps, &v1.Operation{
RepoId: plan.Repo,
PlanId: plan.Id,
diff --git a/internal/protoutil/conversion.go b/internal/protoutil/conversion.go
new file mode 100644
index 00000000..a29aa820
--- /dev/null
+++ b/internal/protoutil/conversion.go
@@ -0,0 +1,73 @@
+package protoutil
+
+import (
+ v1 "github.com/garethgeorge/resticui/gen/go/v1"
+ "github.com/garethgeorge/resticui/pkg/restic"
+)
+
+func SnapshotToProto(s *restic.Snapshot) *v1.ResticSnapshot {
+ return &v1.ResticSnapshot{
+ Id: s.Id,
+ UnixTimeMs: s.UnixTimeMs(),
+ Tree: s.Tree,
+ Paths: s.Paths,
+ Hostname: s.Hostname,
+ Username: s.Username,
+ Tags: s.Tags,
+ Parent: s.Parent,
+ }
+}
+
+func LsEntryToProto(e *restic.LsEntry) *v1.LsEntry {
+ return &v1.LsEntry{
+ Name: e.Name,
+ Type: e.Type,
+ Path: e.Path,
+ Uid: int64(e.Uid),
+ Gid: int64(e.Gid),
+ Size: int64(e.Size),
+ Mode: int64(e.Mode),
+ Mtime: e.Mtime,
+ Atime: e.Atime,
+ Ctime: e.Ctime,
+ }
+}
+
+func BackupProgressEntryToProto(b *restic.BackupProgressEntry) *v1.BackupProgressEntry {
+ switch b.MessageType {
+ case "summary":
+ return &v1.BackupProgressEntry{
+ Entry: &v1.BackupProgressEntry_Summary{
+ Summary: &v1.BackupProgressSummary{
+ FilesNew: int64(b.FilesNew),
+ FilesChanged: int64(b.FilesChanged),
+ FilesUnmodified: int64(b.FilesUnmodified),
+ DirsNew: int64(b.DirsNew),
+ DirsChanged: int64(b.DirsChanged),
+ DirsUnmodified: int64(b.DirsUnmodified),
+ DataBlobs: int64(b.DataBlobs),
+ TreeBlobs: int64(b.TreeBlobs),
+ DataAdded: int64(b.DataAdded),
+ TotalFilesProcessed: int64(b.TotalFilesProcessed),
+ TotalBytesProcessed: int64(b.TotalBytesProcessed),
+ TotalDuration: float64(b.TotalDuration),
+ SnapshotId: b.SnapshotId,
+ },
+ },
+ }
+ case "status":
+ return &v1.BackupProgressEntry{
+ Entry: &v1.BackupProgressEntry_Status{
+ Status: &v1.BackupProgressStatusEntry{
+ PercentDone: b.PercentDone,
+ TotalFiles: int64(b.TotalFiles),
+ FilesDone: int64(b.FilesDone),
+ TotalBytes: int64(b.TotalBytes),
+ BytesDone: int64(b.BytesDone),
+ },
+ },
+ }
+ default:
+ return nil
+ }
+}
diff --git a/internal/protoutil/conversion_test.go b/internal/protoutil/conversion_test.go
new file mode 100644
index 00000000..2c7940e6
--- /dev/null
+++ b/internal/protoutil/conversion_test.go
@@ -0,0 +1,118 @@
+package protoutil
+
+import (
+ "testing"
+
+ v1 "github.com/garethgeorge/resticui/gen/go/v1"
+ "github.com/garethgeorge/resticui/pkg/restic"
+ "google.golang.org/protobuf/proto"
+)
+
+func TestSnapshotToProto(t *testing.T) {
+ snapshot := &restic.Snapshot{
+ Id: "db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52",
+ Time: "2023-11-10T19:14:17.053824063-08:00",
+ Tree: "3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2",
+ Paths: []string{"/resticui"},
+ Hostname: "pop-os",
+ Username: "dontpanic",
+ Tags: []string{},
+ Parent: "",
+ }
+
+ want := &v1.ResticSnapshot{
+ Id: "db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52",
+ UnixTimeMs: 1699672457053,
+ Tree: "3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2",
+ Paths: []string{"/resticui"},
+ Hostname: "pop-os",
+ Username: "dontpanic",
+ Tags: []string{},
+ Parent: "",
+ }
+
+ got := SnapshotToProto(snapshot)
+
+ if !proto.Equal(want, got) {
+ t.Errorf("wanted %+v, got: %+v", want, got)
+ }
+}
+
+func TestBackupProgressEntryToProto(t *testing.T) {
+ cases := []struct {
+ name string
+ entry *restic.BackupProgressEntry
+ want *v1.BackupProgressEntry
+ }{
+ {
+ name: "summary",
+ entry: &restic.BackupProgressEntry{
+ MessageType: "summary",
+ FilesNew: 1,
+ FilesChanged: 2,
+ FilesUnmodified: 3,
+ DirsNew: 4,
+ DirsChanged: 5,
+ DirsUnmodified: 6,
+ DataBlobs: 7,
+ TreeBlobs: 8,
+ DataAdded: 9,
+ TotalFilesProcessed: 10,
+ TotalBytesProcessed: 11,
+ TotalDuration: 12.0,
+ SnapshotId: "db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52",
+ PercentDone: 13.0, // should be ignored.
+ },
+ want: &v1.BackupProgressEntry{
+ Entry: &v1.BackupProgressEntry_Summary{
+ Summary: &v1.BackupProgressSummary{
+ FilesNew: 1,
+ FilesChanged: 2,
+ FilesUnmodified: 3,
+ DirsNew: 4,
+ DirsChanged: 5,
+ DirsUnmodified: 6,
+ DataBlobs: 7,
+ TreeBlobs: 8,
+ DataAdded: 9,
+ TotalFilesProcessed: 10,
+ TotalBytesProcessed: 11,
+ TotalDuration: 12.0,
+ SnapshotId: "db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52",
+ },
+ },
+ },
+ },
+ {
+ name: "status",
+ entry: &restic.BackupProgressEntry{
+ MessageType: "status",
+ PercentDone: 13.0,
+ TotalFiles: 14,
+ FilesDone: 15,
+ TotalBytes: 16,
+ BytesDone: 17,
+ },
+ want: &v1.BackupProgressEntry{
+ Entry: &v1.BackupProgressEntry_Status{
+ Status: &v1.BackupProgressStatusEntry{
+ PercentDone: 13.0,
+ TotalFiles: 14,
+ FilesDone: 15,
+ TotalBytes: 16,
+ BytesDone: 17,
+ },
+ },
+ },
+ },
+ }
+
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ got := BackupProgressEntryToProto(c.entry)
+ if !proto.Equal(got, c.want) {
+ t.Errorf("wanted: %+v, got: %+v", c.want, got)
+ }
+ })
+ }
+}
diff --git a/internal/protoutil/validation.go b/internal/protoutil/validation.go
new file mode 100644
index 00000000..9bd64f2e
--- /dev/null
+++ b/internal/protoutil/validation.go
@@ -0,0 +1,42 @@
+package protoutil
+
+import (
+ "errors"
+ "fmt"
+
+ v1 "github.com/garethgeorge/resticui/gen/go/v1"
+ "github.com/garethgeorge/resticui/pkg/restic"
+)
+
+// ValidateOperation verifies critical properties of the operation proto.
+func ValidateOperation(op *v1.Operation) error {
+ if op.Id == 0 {
+ return errors.New("operation.id is required")
+ }
+ if op.RepoId == "" {
+ return errors.New("operation.repo_id is required")
+ }
+ if op.PlanId == "" {
+ return errors.New("operation.plan_id is required")
+ }
+ if op.SnapshotId != "" {
+ if err := restic.ValidateSnapshotId(op.SnapshotId); err != nil {
+ return fmt.Errorf("operation.snapshot_id is invalid: %w", err)
+ }
+ }
+ return nil
+}
+
+// ValidateSnapshot verifies critical properties of the snapshot proto representation.
+func ValidateSnapshot(s *v1.ResticSnapshot) error {
+ if s.Id == "" {
+ return errors.New("snapshot.id is required")
+ }
+ if s.UnixTimeMs == 0 {
+ return errors.New("snapshot.unix_time_ms must be non-zero")
+ }
+ if err := restic.ValidateSnapshotId(s.Id); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/pkg/restic/outputs.go b/pkg/restic/outputs.go
index 693432c6..6a99b4f0 100644
--- a/pkg/restic/outputs.go
+++ b/pkg/restic/outputs.go
@@ -3,9 +3,9 @@ package restic
import (
"bufio"
"encoding/json"
+ "errors"
"fmt"
"io"
- "log"
"os/exec"
"slices"
"time"
@@ -25,19 +25,6 @@ type Snapshot struct {
unixTimeMs int64 `json:"-"`
}
-func (s *Snapshot) ToProto() *v1.ResticSnapshot {
- return &v1.ResticSnapshot{
- Id: s.Id,
- UnixTimeMs: s.UnixTimeMs(),
- Tree: s.Tree,
- Paths: s.Paths,
- Hostname: s.Hostname,
- Username: s.Username,
- Tags: s.Tags,
- Parent: s.Parent,
- }
-}
-
func (s *Snapshot) UnixTimeMs() int64 {
if s.unixTimeMs != 0 {
return s.unixTimeMs
@@ -50,6 +37,16 @@ func (s *Snapshot) UnixTimeMs() int64 {
return s.unixTimeMs
}
+func (s *Snapshot) Validate() error {
+ if err := ValidateSnapshotId(s.Id); err != nil {
+ return fmt.Errorf("snapshot.id invalid: %v", err)
+ }
+ if s.Time == "" || s.UnixTimeMs() == 0 {
+ return fmt.Errorf("snapshot.time invalid: %v", s.Time)
+ }
+ return nil
+}
+
type BackupProgressEntry struct {
// Common fields
MessageType string `json:"message_type"` // "summary" or "status"
@@ -77,44 +74,17 @@ type BackupProgressEntry struct {
BytesDone int `json:"bytes_done"`
}
-func (b *BackupProgressEntry) ToProto() *v1.BackupProgressEntry {
- switch b.MessageType {
- case "summary":
- return &v1.BackupProgressEntry{
- Entry: &v1.BackupProgressEntry_Summary{
- Summary: &v1.BackupProgressSummary{
- FilesNew: int64(b.FilesNew),
- FilesChanged: int64(b.FilesChanged),
- FilesUnmodified: int64(b.FilesUnmodified),
- DirsNew: int64(b.DirsNew),
- DirsChanged: int64(b.DirsChanged),
- DirsUnmodified: int64(b.DirsUnmodified),
- DataBlobs: int64(b.DataBlobs),
- TreeBlobs: int64(b.TreeBlobs),
- DataAdded: int64(b.DataAdded),
- TotalFilesProcessed: int64(b.TotalFilesProcessed),
- TotalBytesProcessed: int64(b.TotalBytesProcessed),
- TotalDuration: float64(b.TotalDuration),
- SnapshotId: b.SnapshotId,
- },
- },
+func (b *BackupProgressEntry) Validate() error {
+ if b.MessageType == "summary" {
+ if b.SnapshotId == "" {
+ return errors.New("summary message must have snapshot_id")
}
- case "status":
- return &v1.BackupProgressEntry{
- Entry: &v1.BackupProgressEntry_Status{
- Status: &v1.BackupProgressStatusEntry{
- PercentDone: b.PercentDone,
- TotalFiles: int64(b.TotalFiles),
- FilesDone: int64(b.FilesDone),
- TotalBytes: int64(b.TotalBytes),
- BytesDone: int64(b.BytesDone),
- },
- },
+ if err := ValidateSnapshotId(b.SnapshotId); err != nil {
+ return err
}
- default:
- log.Fatalf("unknown message type: %s", b.MessageType)
- return nil
}
+
+ return nil
}
// readBackupProgressEntrys returns the summary event or an error if the command failed.
@@ -134,6 +104,9 @@ func readBackupProgressEntries(cmd *exec.Cmd, output io.Reader, callback func(ev
return nil, NewCmdError(cmd, bytes, fmt.Errorf("command output was not JSON: %w", err))
}
+ if err := event.Validate(); err != nil {
+ return nil, err
+ }
}
// remaining events are parsed as JSON
@@ -144,6 +117,9 @@ func readBackupProgressEntries(cmd *exec.Cmd, output io.Reader, callback func(ev
if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
return nil, fmt.Errorf("failed to parse JSON: %w", err)
}
+ if err := event.Validate(); err != nil {
+ return nil, err
+ }
if callback != nil {
callback(event)
@@ -217,3 +193,24 @@ type ForgetResult struct {
Keep []Snapshot `json:"keep"`
Remove []Snapshot `json:"remove"`
}
+
+func (r *ForgetResult) Validate() error {
+ for _, s := range r.Keep {
+ if err := ValidateSnapshotId(s.Id); err != nil {
+ return err
+ }
+ }
+ for _, s := range r.Remove {
+ if err := ValidateSnapshotId(s.Id); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func ValidateSnapshotId(id string) error {
+ if len(id) != 64 {
+ return fmt.Errorf("restic may be out of date (check with `restic self-upgrade`): snapshot ID must be 64 chars, got %v chars", len(id))
+ }
+ return nil
+}
diff --git a/pkg/restic/outputs_test.go b/pkg/restic/outputs_test.go
index 63aafb4a..1b218b45 100644
--- a/pkg/restic/outputs_test.go
+++ b/pkg/restic/outputs_test.go
@@ -9,7 +9,7 @@ import (
func TestReadBackupProgressEntries(t *testing.T) {
t.Parallel()
testInput := `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":15}
- {"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":166,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":128,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":166,"total_bytes_processed":16754463,"total_duration":0.235433378,"snapshot_id":"bca1043e"}`
+ {"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":166,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":128,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":166,"total_bytes_processed":16754463,"total_duration":0.235433378,"id":"d4558b360cc1b7966e416e010382ab8feb49d14da7832266832d69a43af10147"}`
b := bytes.NewBuffer([]byte(testInput))
@@ -27,7 +27,6 @@ func TestReadBackupProgressEntries(t *testing.T) {
}
}
-
func TestReadLs(t *testing.T) {
testInput := `{"time":"2023-11-10T19:14:17.053824063-08:00","tree":"3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2","paths":["/resticui"],"hostname":"pop-os","username":"dontpanic","uid":1000,"gid":1000,"id":"db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52","short_id":"db155169","struct_type":"snapshot"}
{"name":".git","type":"dir","path":"/.git","uid":1000,"gid":1000,"mode":2147484157,"mtime":"2023-11-10T18:32:38.156599473-08:00","atime":"2023-11-10T18:32:38.156599473-08:00","ctime":"2023-11-10T18:32:38.156599473-08:00","struct_type":"node"}
@@ -47,40 +46,3 @@ func TestReadLs(t *testing.T) {
t.Errorf("wanted 3 entries, got: %d", len(entries))
}
}
-
-func TestSnapshotToProto(t *testing.T) {
- snapshot := &Snapshot{
- Id: "db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52",
- Time: "2023-11-10T19:14:17.053824063-08:00",
- Tree: "3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2",
- Paths: []string{"/resticui"},
- Hostname: "pop-os",
- Username: "dontpanic",
- Tags: []string{},
- Parent: "",
- }
-
- proto := snapshot.ToProto()
-
- if proto.Id != snapshot.Id {
- t.Errorf("wanted id %q, got: %q", snapshot.Id, proto.Id)
- }
- if proto.Tree != snapshot.Tree {
- t.Errorf("wanted tree %q, got: %q", snapshot.Tree, proto.Tree)
- }
- if proto.Hostname != snapshot.Hostname {
- t.Errorf("wanted hostname %q, got: %q", snapshot.Hostname, proto.Hostname)
- }
- if proto.Username != snapshot.Username {
- t.Errorf("wanted username %q, got: %q", snapshot.Username, proto.Username)
- }
- if len(proto.Tags) != len(snapshot.Tags) {
- t.Errorf("wanted %d tags, got: %d", len(snapshot.Tags), len(proto.Tags))
- }
- if proto.Parent != snapshot.Parent {
- t.Errorf("wanted parent %q, got: %q", snapshot.Parent, proto.Parent)
- }
- if proto.UnixTimeMs != 1699672457053 {
- t.Errorf("wanted unix time %d, got: %d", 1699672457053, proto.UnixTimeMs)
- }
-}
\ No newline at end of file
diff --git a/pkg/restic/restic.go b/pkg/restic/restic.go
index 5b73e7cb..69c53ba7 100644
--- a/pkg/restic/restic.go
+++ b/pkg/restic/restic.go
@@ -167,7 +167,11 @@ func (r *Repo) Snapshots(ctx context.Context, opts ...GenericOption) ([]*Snapsho
if err := json.Unmarshal(output, &snapshots); err != nil {
return nil, NewCmdError(cmd, output, fmt.Errorf("command output is not valid JSON: %w", err))
}
-
+ for _, snapshot := range snapshots {
+ if err := snapshot.Validate(); err != nil {
+ return nil, fmt.Errorf("invalid snapshot: %w", err)
+ }
+ }
return snapshots, nil
}
@@ -199,6 +203,9 @@ func (r *Repo) Forget(ctx context.Context, policy RetentionPolicy, pruneOutput i
if len(result) != 1 {
return nil, fmt.Errorf("expected 1 output from forget, got %v", len(result))
}
+ if err := result[0].Validate(); err != nil {
+ return nil, NewCmdError(cmd, output, fmt.Errorf("invalid forget result: %w", err))
+ }
// then run the prune command
args = []string{"prune", "--json"}
diff --git a/pkg/restic/restic_test.go b/pkg/restic/restic_test.go
index 5bd213a6..0b5b288b 100644
--- a/pkg/restic/restic_test.go
+++ b/pkg/restic/restic_test.go
@@ -155,8 +155,8 @@ func TestSnapshot(t *testing.T) {
// Ensure that snapshot timestamps are set, this is critical for correct ordering in the orchestrator.
for _, snapshot := range snapshots {
- if p := snapshot.ToProto(); p.UnixTimeMs == 0 {
- t.Errorf("wanted snapshot time to be non-zero, got: %v", p.UnixTimeMs)
+ if snapshot.UnixTimeMs() == 0 {
+ t.Errorf("wanted snapshot time to be non-zero, got: %v", snapshot.UnixTimeMs())
}
}
})
diff --git a/webui/src/components/OperationList.tsx b/webui/src/components/OperationList.tsx
index 078cd2fd..bc677b39 100644
--- a/webui/src/components/OperationList.tsx
+++ b/webui/src/components/OperationList.tsx
@@ -162,15 +162,12 @@ export const OperationRow = ({
<>
),
@@ -213,10 +210,11 @@ const SnapshotInfo = ({
return (
@@ -245,7 +243,7 @@ const SnapshotInfo = ({
},
{
key: 2,
- label: "Browse",
+ label: "Browse and Restore Files in Backup",
children: (
),
@@ -294,7 +292,7 @@ const BackupOperationStatus = ({
<>
Snapshot ID:
- {sum.snapshotId}
+ {normalizeSnapshotId(sum.snapshotId!)}
diff --git a/webui/src/components/OperationTree.tsx b/webui/src/components/OperationTree.tsx
index 40ef3e39..8fabb3dd 100644
--- a/webui/src/components/OperationTree.tsx
+++ b/webui/src/components/OperationTree.tsx
@@ -8,7 +8,7 @@ import {
toEop,
unsubscribeFromOperations,
} from "../state/oplog";
-import { Col, Empty, Row, Tree } from "antd";
+import { Col, Divider, Empty, Row, Tree } from "antd";
import _ from "lodash";
import { DataNode } from "antd/es/tree";
import {
@@ -112,11 +112,13 @@ export const OperationTree = ({
showIcon
defaultExpandedKeys={[backups[0].id!]}
onSelect={(keys, info) => {
- setSelectedBackupId(
- info.selectedNodes.length > 0
- ? info.selectedNodes[0].backup!.id!
- : null
- );
+ if (info.selectedNodes.length === 0) return;
+ const backup = info.selectedNodes[0].backup;
+ if (!backup) {
+ setSelectedBackupId(null);
+ return;
+ }
+ setSelectedBackupId(backup.id!);
}}
titleRender={(node: OpTreeNode): React.ReactNode => {
if (node.title) {