From a2af27065c239e078a6db2c110bb9d0a43b4ca38 Mon Sep 17 00:00:00 2001 From: garethgeorge Date: Fri, 10 Nov 2023 19:36:24 -0800 Subject: [PATCH] Implement Snapshots and ListDirectory operations --- internal/restic/error.go | 7 +- internal/restic/outputs.go | 57 +++++++++++----- internal/restic/outputs_test.go | 48 ++++++++++++++ internal/restic/restic.go | 113 ++++++++++++++++++++++++++++---- internal/restic/restic_test.go | 82 +++++++++++++++++++++-- 5 files changed, 272 insertions(+), 35 deletions(-) create mode 100644 internal/restic/outputs_test.go diff --git a/internal/restic/error.go b/internal/restic/error.go index 4b497d1e..9f458c86 100644 --- a/internal/restic/error.go +++ b/internal/restic/error.go @@ -12,7 +12,7 @@ type CmdError struct { } func (e *CmdError) Error() string { - m := fmt.Sprintf("command %s failed: %s", e.Command, e.Err.Error()) + m := fmt.Sprintf("command %q failed: %s", e.Command, e.Err.Error()) if e.Output != "" { m += "\nDetails: \n" + e.Output } @@ -23,6 +23,11 @@ func (e *CmdError) Unwrap() error { return e.Err } +func (e *CmdError) Is(target error) bool { + _, ok := target.(*CmdError) + return ok +} + // NewCmdError creates a new error indicating that running a command failed. func NewCmdError(cmd *exec.Cmd, output []byte, err error) *CmdError { cerr := &CmdError{ diff --git a/internal/restic/outputs.go b/internal/restic/outputs.go index e21866e1..0cc7e02d 100644 --- a/internal/restic/outputs.go +++ b/internal/restic/outputs.go @@ -3,15 +3,25 @@ package restic import ( "bufio" "encoding/json" - "errors" "fmt" "io" "os/exec" "slices" - - "go.uber.org/zap" ) +type LsEntry struct { + Name string `json:"name"` + Type string `json:"type"` + Path string `json:"path"` + Uid int `json:"uid"` + Gid int `json:"gid"` + Size int `json:"size"` + Mode int `json:"mode"` + Mtime string `json:"mtime"` + Atime string `json:"atime"` + Ctime string `json:"ctime"` +} + type Snapshot struct { Time string `json:"time"` Tree string `json:"tree"` @@ -67,16 +77,7 @@ func readBackupEvents(cmd *exec.Cmd, output io.Reader, callback func(event *Back bytes = append(bytes, scanner.Bytes()...) } - jsonErr := fmt.Errorf("command output was not JSON: %w", err) - - if err := cmd.Wait(); err != nil { - return nil, NewCmdError(cmd, bytes, errors.Join( - fmt.Errorf("command failed: %w", err), - fmt.Errorf("command output was not JSON: %w", err), - )) - } - - return nil, NewCmdError(cmd, bytes, jsonErr) + return nil, NewCmdError(cmd, bytes, fmt.Errorf("command output was not JSON: %w", err)) } } @@ -89,7 +90,9 @@ func readBackupEvents(cmd *exec.Cmd, output io.Reader, callback func(event *Back return nil, fmt.Errorf("failed to parse JSON: %w", err) } - callback(event) + if callback != nil { + callback(event) + } if event.MessageType == "summary" { summary = event @@ -100,7 +103,29 @@ func readBackupEvents(cmd *exec.Cmd, output io.Reader, callback func(event *Back return summary, fmt.Errorf("scanner encountered error: %w", err) } - zap.L().Debug("finished reading events", zap.String("command", cmd.String())) - return summary, nil } + +func readLs(output io.Reader) (*Snapshot, []*LsEntry, error) { + scanner := bufio.NewScanner(output) + scanner.Split(bufio.ScanLines) + + if !scanner.Scan() { + return nil, nil, fmt.Errorf("failed to read first line, expected snapshot info") + } + + var snapshot *Snapshot + if err := json.Unmarshal(scanner.Bytes(), &snapshot); err != nil { + return nil, nil, fmt.Errorf("failed to parse JSON: %w", err) + } + + var entries []*LsEntry + for scanner.Scan() { + var entry *LsEntry + if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil { + return nil, nil, fmt.Errorf("failed to parse JSON: %w", err) + } + entries = append(entries, entry) + } + return snapshot, entries, nil +} \ No newline at end of file diff --git a/internal/restic/outputs_test.go b/internal/restic/outputs_test.go new file mode 100644 index 00000000..9a8aa2e8 --- /dev/null +++ b/internal/restic/outputs_test.go @@ -0,0 +1,48 @@ +package restic + +import ( + "bytes" + "os/exec" + "testing" +) + +func TestReadBackupEvents(t *testing.T) { + t.Parallel() + testInput := `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":15} + {"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":166,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":128,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":166,"total_bytes_processed":16754463,"total_duration":0.235433378,"snapshot_id":"bca1043e"}` + + b := bytes.NewBuffer([]byte(testInput)) + + summary, err := readBackupEvents(&exec.Cmd{}, b, func(event *BackupEvent) { + t.Logf("event: %v", event) + }) + if err != nil { + t.Fatalf("failed to read backup events: %v", err) + } + if summary == nil { + t.Fatalf("wanted summary, got: nil") + } + if summary.TotalFilesProcessed != 166 { + t.Errorf("wanted 166 files processed, got: %d", summary.TotalFilesProcessed) + } +} + +func TestReadLs(t *testing.T) { + testInput := `{"time":"2023-11-10T19:14:17.053824063-08:00","tree":"3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2","paths":["/home/dontpanic/Documents/github/garethgeorge/resticui"],"hostname":"pop-os","username":"dontpanic","uid":1000,"gid":1000,"id":"db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52","short_id":"db155169","struct_type":"snapshot"} + {"name":".git","type":"dir","path":"/.git","uid":1000,"gid":1000,"mode":2147484157,"mtime":"2023-11-10T18:32:38.156599473-08:00","atime":"2023-11-10T18:32:38.156599473-08:00","ctime":"2023-11-10T18:32:38.156599473-08:00","struct_type":"node"} + {"name":".gitignore","type":"file","path":"/.gitignore","uid":1000,"gid":1000,"size":22,"mode":436,"mtime":"2023-11-10T00:41:26.611346634-08:00","atime":"2023-11-10T00:41:26.611346634-08:00","ctime":"2023-11-10T00:41:26.611346634-08:00","struct_type":"node"} + {"name":"README.md","type":"file","path":"/README.md","uid":1000,"gid":1000,"size":762,"mode":436,"mtime":"2023-11-10T00:59:06.842538768-08:00","atime":"2023-11-10T00:59:06.842538768-08:00","ctime":"2023-11-10T00:59:06.842538768-08:00","struct_type":"node"}` + + b := bytes.NewBuffer([]byte(testInput)) + + snapshot, entries, err := readLs(b) + if err != nil { + t.Fatalf("failed to read ls output: %v", err) + } + if snapshot == nil { + t.Fatalf("wanted snapshot, got: nil") + } + if len(entries) != 3 { + t.Errorf("wanted 3 entries, got: %d", len(entries)) + } +} \ No newline at end of file diff --git a/internal/restic/restic.go b/internal/restic/restic.go index 61157937..f164264e 100644 --- a/internal/restic/restic.go +++ b/internal/restic/restic.go @@ -1,17 +1,22 @@ package restic import ( + "bytes" "context" + "encoding/json" + "errors" "fmt" "io" "os" "os/exec" + "sync" v1 "github.com/garethgeorge/resticui/gen/go/v1" - "golang.org/x/sync/errgroup" + "github.com/hashicorp/go-multierror" ) type Repo struct { + mu sync.Mutex cmd string repo *v1.Repo flags []string @@ -63,7 +68,17 @@ func (r *Repo) init(ctx context.Context) error { return nil } +func (r *Repo) Init(ctx context.Context) error { + r.mu.Lock() + defer r.mu.Unlock() + r.initialized = false + return r.init(ctx) +} + func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent), opts ...BackupOption) (*BackupEvent, error) { + r.mu.Lock() + defer r.mu.Unlock() + if err := r.init(ctx); err != nil { return nil, fmt.Errorf("failed to initialize repo: %w", err) } @@ -73,6 +88,12 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent), o(opt) } + for _, p := range opt.paths { + if _, err := os.Stat(p); err != nil { + return nil, fmt.Errorf("path %s does not exist: %w", p, err) + } + } + args := []string{"backup", "--json", "--exclude-caches"} args = append(args, r.flags...) args = append(args, opt.paths...) @@ -92,31 +113,97 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent), return nil, NewCmdError(cmd, nil, err) } + var wg sync.WaitGroup var summary *BackupEvent - var errgroup errgroup.Group + var cmdErr error + var readErr error - errgroup.Go(func() error { + wg.Add(1) + go func() { + defer wg.Done() var err error summary, err = readBackupEvents(cmd, reader, progressCallback) if err != nil { - return fmt.Errorf("processing command output: %w", err) + readErr = fmt.Errorf("processing command output: %w", err) } - return nil - }) + }() - errgroup.Go(func() error { + wg.Add(1) + go func() { defer writer.Close() + defer wg.Done() if err := cmd.Wait(); err != nil { - return NewCmdError(cmd, nil, err) + cmdErr = NewCmdError(cmd, nil, err) } - return nil - }) + }() - if err := errgroup.Wait(); err != nil { - return nil, err + wg.Wait() + + var err error + if cmdErr != nil || readErr != nil { + err = multierror.Append(nil, cmdErr, readErr) + } + return summary, err +} + +func (r *Repo) Snapshots(ctx context.Context) ([]*Snapshot, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if err := r.init(ctx); err != nil { + return nil, fmt.Errorf("failed to initialize repo: %w", err) } - return summary, nil + args := []string{"snapshots", "--json"} + args = append(args, r.flags...) + + cmd := exec.CommandContext(ctx, r.cmd, args...) + cmd.Env = append(cmd.Env, r.buildEnv()...) + + output, err := cmd.CombinedOutput() + if err != nil { + return nil, NewCmdError(cmd, output, err) + } + + var snapshots []*Snapshot + if err := json.Unmarshal(output, &snapshots); err != nil { + return nil, NewCmdError(cmd, output, fmt.Errorf("command output is not valid JSON: %w", err)) + } + + return snapshots, nil +} + +func (r *Repo) ListDirectory(ctx context.Context, snapshot string, path string) (*Snapshot, []*LsEntry, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if path == "" { + // an empty path can trigger very expensive operations (e.g. iterates all files in the snapshot) + return nil, nil, errors.New("path must not be empty") + } + + if err := r.init(ctx); err != nil { + return nil, nil, fmt.Errorf("failed to initialize repo: %w", err) + } + + args := []string{"ls", "--json", snapshot, path} + args = append(args, r.flags...) + + cmd := exec.CommandContext(ctx, r.cmd, args...) + cmd.Env = append(cmd.Env, r.buildEnv()...) + + output, err := cmd.CombinedOutput() + if err != nil { + return nil, nil, NewCmdError(cmd, output, err) + } + + + snapshots, entries, err := readLs(bytes.NewBuffer(output)) + if err != nil { + return nil, nil, NewCmdError(cmd, output, err) + } + + return snapshots, entries, nil } type RepoOpts struct { diff --git a/internal/restic/restic_test.go b/internal/restic/restic_test.go index 5636720d..e41a4fdf 100644 --- a/internal/restic/restic_test.go +++ b/internal/restic/restic_test.go @@ -2,6 +2,7 @@ package restic import ( "context" + "encoding/json" "testing" v1 "github.com/garethgeorge/resticui/gen/go/v1" @@ -9,6 +10,7 @@ import ( ) func TestResticInit(t *testing.T) { + t.Parallel() repo := t.TempDir() r := NewRepo(&v1.Repo{ @@ -21,6 +23,7 @@ func TestResticInit(t *testing.T) { } func TestResticBackup(t *testing.T) { + t.Parallel() repo := t.TempDir() // create a new repo with cache disabled for testing @@ -29,9 +32,7 @@ func TestResticBackup(t *testing.T) { Uri: repo, Password: "test", }, WithRepoFlags("--no-cache")) - - r.init(context.Background()) - + testData := test.CreateTestData(t) testData2 := test.CreateTestData(t) @@ -39,6 +40,7 @@ func TestResticBackup(t *testing.T) { name string opts []BackupOption files int // expected files at the end of the backup + wantErr bool }{ { name: "no options", @@ -60,6 +62,11 @@ func TestResticBackup(t *testing.T) { opts: []BackupOption{WithBackupPaths(testData), WithBackupExcludes("file*")}, files: 0, }, + { + name: "with nothing to backup", + opts: []BackupOption{}, + wantErr: true, + }, } for _, tc := range tests { @@ -67,8 +74,12 @@ func TestResticBackup(t *testing.T) { summary, err := r.Backup(context.Background(), func(event *BackupEvent) { t.Logf("backup event: %v", event) }, tc.opts...) - if err != nil { - t.Errorf("failed to backup: %v", err) + if (err != nil) != tc.wantErr { + t.Fatalf("wanted error: %v, got: %v", tc.wantErr, err) + } + + if tc.wantErr { + return } if summary == nil { @@ -82,3 +93,64 @@ func TestResticBackup(t *testing.T) { } } +func TestSnapshot(t *testing.T) { + t.Parallel() + + repo := t.TempDir() + + r := NewRepo(&v1.Repo{ + Id: "test", + Uri: repo, + Password: "test", + }, WithRepoFlags("--no-cache")) + + testData := test.CreateTestData(t) + + for i := 0; i < 10; i++ { + _, err := r.Backup(context.Background(), nil, WithBackupPaths(testData)) + if err != nil { + t.Fatalf("failed to backup and create new snapshot: %v", err) + } + } + + snapshots, err := r.Snapshots(context.Background()) + if err != nil { + t.Fatalf("failed to list snapshots: %v", err) + } + + if len(snapshots) != 10 { + t.Errorf("wanted 10 snapshots, got: %d", len(snapshots)) + } + + data, _ := json.Marshal(snapshots) + + t.Logf("snapshots: %v", string(data)) +} + +func TestLs(t *testing.T) { + t.Parallel() + + repo := t.TempDir() + r := NewRepo(&v1.Repo{ + Id: "test", + Uri: repo, + Password: "test", + }, WithRepoFlags("--no-cache")) + + testData := test.CreateTestData(t) + + snapshot, err := r.Backup(context.Background(), nil, WithBackupPaths(testData)) + if err != nil { + t.Fatalf("failed to backup and create new snapshot: %v", err) + } + + _, entries, err := r.ListDirectory(context.Background(), snapshot.SnapshotId, testData) + + if err != nil { + t.Fatalf("failed to list directory: %v", err) + } + + if len(entries) != 101 { + t.Errorf("wanted 101 entries, got: %d", len(entries)) + } +} \ No newline at end of file