lee/backrest
1
0
Fork 0
mirror of https://github.com/garethgeorge/backrest.git synced 2026-05-13 00:00:49 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Implement Snapshots and ListDirectory operations

Browse Source
This commit is contained in:
garethgeorge
2023-11-10 19:36:24 -08:00
parent aeb831868b
commit a2af27065c
5 changed files with 272 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/restic/error.go
+6 -1
View File
@@ -12,7 +12,7 @@ type CmdError struct {
}
func (e *CmdError) Error() string {
m := fmt.Sprintf("command %s failed: %s", e.Command, e.Err.Error())
m := fmt.Sprintf("command %q failed: %s", e.Command, e.Err.Error())
if e.Output != "" {
m += "\nDetails: \n" + e.Output
}
@@ -23,6 +23,11 @@ func (e *CmdError) Unwrap() error {
return e.Err
}
func (e *CmdError) Is(target error) bool {
_, ok := target.(*CmdError)
return ok
}
// NewCmdError creates a new error indicating that running a command failed.
func NewCmdError(cmd *exec.Cmd, output []byte, err error) *CmdError {
cerr := &CmdError{
internal/restic/outputs.go
+41 -16
View File
@@ -3,15 +3,25 @@ package restic
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"os/exec"
"slices"
"go.uber.org/zap"
)
type LsEntry struct {
Name string `json:"name"`
Type string `json:"type"`
Path string `json:"path"`
Uid int `json:"uid"`
Gid int `json:"gid"`
Size int `json:"size"`
Mode int `json:"mode"`
Mtime string `json:"mtime"`
Atime string `json:"atime"`
Ctime string `json:"ctime"`
}
type Snapshot struct {
Time string `json:"time"`
Tree string `json:"tree"`
@@ -67,16 +77,7 @@ func readBackupEvents(cmd *exec.Cmd, output io.Reader, callback func(event *Back
bytes = append(bytes, scanner.Bytes()...)
}
jsonErr := fmt.Errorf("command output was not JSON: %w", err)
if err := cmd.Wait(); err != nil {
return nil, NewCmdError(cmd, bytes, errors.Join(
fmt.Errorf("command failed: %w", err),
fmt.Errorf("command output was not JSON: %w", err),
))
}
return nil, NewCmdError(cmd, bytes, jsonErr)
return nil, NewCmdError(cmd, bytes, fmt.Errorf("command output was not JSON: %w", err))
}
}
@@ -89,7 +90,9 @@ func readBackupEvents(cmd *exec.Cmd, output io.Reader, callback func(event *Back
return nil, fmt.Errorf("failed to parse JSON: %w", err)
}
callback(event)
if callback != nil {
callback(event)
}
if event.MessageType == "summary" {
summary = event
@@ -100,7 +103,29 @@ func readBackupEvents(cmd *exec.Cmd, output io.Reader, callback func(event *Back
return summary, fmt.Errorf("scanner encountered error: %w", err)
}
zap.L().Debug("finished reading events", zap.String("command", cmd.String()))
return summary, nil
}
func readLs(output io.Reader) (*Snapshot, []*LsEntry, error) {
scanner := bufio.NewScanner(output)
scanner.Split(bufio.ScanLines)
if !scanner.Scan() {
return nil, nil, fmt.Errorf("failed to read first line, expected snapshot info")
}
var snapshot *Snapshot
if err := json.Unmarshal(scanner.Bytes(), &snapshot); err != nil {
return nil, nil, fmt.Errorf("failed to parse JSON: %w", err)
}
var entries []*LsEntry
for scanner.Scan() {
var entry *LsEntry
if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {
return nil, nil, fmt.Errorf("failed to parse JSON: %w", err)
}
entries = append(entries, entry)
}
return snapshot, entries, nil
}
internal/restic/outputs_test.go
+48
View File
@@ -0,0 +1,48 @@
package restic
import (
"bytes"
"os/exec"
"testing"
)
func TestReadBackupEvents(t *testing.T) {
t.Parallel()
testInput := `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":15}
{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":166,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":128,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":166,"total_bytes_processed":16754463,"total_duration":0.235433378,"snapshot_id":"bca1043e"}`
b := bytes.NewBuffer([]byte(testInput))
summary, err := readBackupEvents(&exec.Cmd{}, b, func(event *BackupEvent) {
t.Logf("event: %v", event)
})
if err != nil {
t.Fatalf("failed to read backup events: %v", err)
}
if summary == nil {
t.Fatalf("wanted summary, got: nil")
}
if summary.TotalFilesProcessed != 166 {
t.Errorf("wanted 166 files processed, got: %d", summary.TotalFilesProcessed)
}
}
func TestReadLs(t *testing.T) {
testInput := `{"time":"2023-11-10T19:14:17.053824063-08:00","tree":"3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2","paths":["/home/dontpanic/Documents/github/garethgeorge/resticui"],"hostname":"pop-os","username":"dontpanic","uid":1000,"gid":1000,"id":"db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52","short_id":"db155169","struct_type":"snapshot"}
{"name":".git","type":"dir","path":"/.git","uid":1000,"gid":1000,"mode":2147484157,"mtime":"2023-11-10T18:32:38.156599473-08:00","atime":"2023-11-10T18:32:38.156599473-08:00","ctime":"2023-11-10T18:32:38.156599473-08:00","struct_type":"node"}
{"name":".gitignore","type":"file","path":"/.gitignore","uid":1000,"gid":1000,"size":22,"mode":436,"mtime":"2023-11-10T00:41:26.611346634-08:00","atime":"2023-11-10T00:41:26.611346634-08:00","ctime":"2023-11-10T00:41:26.611346634-08:00","struct_type":"node"}
{"name":"README.md","type":"file","path":"/README.md","uid":1000,"gid":1000,"size":762,"mode":436,"mtime":"2023-11-10T00:59:06.842538768-08:00","atime":"2023-11-10T00:59:06.842538768-08:00","ctime":"2023-11-10T00:59:06.842538768-08:00","struct_type":"node"}`
b := bytes.NewBuffer([]byte(testInput))
snapshot, entries, err := readLs(b)
if err != nil {
t.Fatalf("failed to read ls output: %v", err)
}
if snapshot == nil {
t.Fatalf("wanted snapshot, got: nil")
}
if len(entries) != 3 {
t.Errorf("wanted 3 entries, got: %d", len(entries))
}
}
internal/restic/restic.go
+100 -13
View File
@@ -1,17 +1,22 @@
package restic
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"sync"
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"golang.org/x/sync/errgroup"
"github.com/hashicorp/go-multierror"
)
type Repo struct {
mu sync.Mutex
cmd string
repo *v1.Repo
flags []string
@@ -63,7 +68,17 @@ func (r *Repo) init(ctx context.Context) error {
return nil
}
func (r *Repo) Init(ctx context.Context) error {
r.mu.Lock()
defer r.mu.Unlock()
r.initialized = false
return r.init(ctx)
}
func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent), opts ...BackupOption) (*BackupEvent, error) {
r.mu.Lock()
defer r.mu.Unlock()
if err := r.init(ctx); err != nil {
return nil, fmt.Errorf("failed to initialize repo: %w", err)
}
@@ -73,6 +88,12 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent),
o(opt)
}
for _, p := range opt.paths {
if _, err := os.Stat(p); err != nil {
return nil, fmt.Errorf("path %s does not exist: %w", p, err)
}
}
args := []string{"backup", "--json", "--exclude-caches"}
args = append(args, r.flags...)
args = append(args, opt.paths...)
@@ -92,31 +113,97 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent),
return nil, NewCmdError(cmd, nil, err)
}
var wg sync.WaitGroup
var summary *BackupEvent
var errgroup errgroup.Group
var cmdErr error
var readErr error
errgroup.Go(func() error {
wg.Add(1)
go func() {
defer wg.Done()
var err error
summary, err = readBackupEvents(cmd, reader, progressCallback)
if err != nil {
return fmt.Errorf("processing command output: %w", err)
readErr = fmt.Errorf("processing command output: %w", err)
}
return nil
})
}()
errgroup.Go(func() error {
wg.Add(1)
go func() {
defer writer.Close()
defer wg.Done()
if err := cmd.Wait(); err != nil {
return NewCmdError(cmd, nil, err)
cmdErr = NewCmdError(cmd, nil, err)
}
return nil
})
}()
if err := errgroup.Wait(); err != nil {
return nil, err
wg.Wait()
var err error
if cmdErr != nil || readErr != nil {
err = multierror.Append(nil, cmdErr, readErr)
}
return summary, err
}
func (r *Repo) Snapshots(ctx context.Context) ([]*Snapshot, error) {
r.mu.Lock()
defer r.mu.Unlock()
if err := r.init(ctx); err != nil {
return nil, fmt.Errorf("failed to initialize repo: %w", err)
}
return summary, nil
args := []string{"snapshots", "--json"}
args = append(args, r.flags...)
cmd := exec.CommandContext(ctx, r.cmd, args...)
cmd.Env = append(cmd.Env, r.buildEnv()...)
output, err := cmd.CombinedOutput()
if err != nil {
return nil, NewCmdError(cmd, output, err)
}
var snapshots []*Snapshot
if err := json.Unmarshal(output, &snapshots); err != nil {
return nil, NewCmdError(cmd, output, fmt.Errorf("command output is not valid JSON: %w", err))
}
return snapshots, nil
}
func (r *Repo) ListDirectory(ctx context.Context, snapshot string, path string) (*Snapshot, []*LsEntry, error) {
r.mu.Lock()
defer r.mu.Unlock()
if path == "" {
// an empty path can trigger very expensive operations (e.g. iterates all files in the snapshot)
return nil, nil, errors.New("path must not be empty")
}
if err := r.init(ctx); err != nil {
return nil, nil, fmt.Errorf("failed to initialize repo: %w", err)
}
args := []string{"ls", "--json", snapshot, path}
args = append(args, r.flags...)
cmd := exec.CommandContext(ctx, r.cmd, args...)
cmd.Env = append(cmd.Env, r.buildEnv()...)
output, err := cmd.CombinedOutput()
if err != nil {
return nil, nil, NewCmdError(cmd, output, err)
}
snapshots, entries, err := readLs(bytes.NewBuffer(output))
if err != nil {
return nil, nil, NewCmdError(cmd, output, err)
}
return snapshots, entries, nil
}
type RepoOpts struct {
internal/restic/restic_test.go
+77 -5
View File
@@ -2,6 +2,7 @@ package restic
import (
"context"
"encoding/json"
"testing"
v1 "github.com/garethgeorge/resticui/gen/go/v1"
@@ -9,6 +10,7 @@ import (
)
func TestResticInit(t *testing.T) {
t.Parallel()
repo := t.TempDir()
r := NewRepo(&v1.Repo{
@@ -21,6 +23,7 @@ func TestResticInit(t *testing.T) {
}
func TestResticBackup(t *testing.T) {
t.Parallel()
repo := t.TempDir()
// create a new repo with cache disabled for testing
@@ -29,9 +32,7 @@ func TestResticBackup(t *testing.T) {
Uri: repo,
Password: "test",
}, WithRepoFlags("--no-cache"))
r.init(context.Background())
testData := test.CreateTestData(t)
testData2 := test.CreateTestData(t)
@@ -39,6 +40,7 @@ func TestResticBackup(t *testing.T) {
name string
opts []BackupOption
files int // expected files at the end of the backup
wantErr bool
}{
{
name: "no options",
@@ -60,6 +62,11 @@ func TestResticBackup(t *testing.T) {
opts: []BackupOption{WithBackupPaths(testData), WithBackupExcludes("file*")},
files: 0,
},
{
name: "with nothing to backup",
opts: []BackupOption{},
wantErr: true,
},
}
for _, tc := range tests {
@@ -67,8 +74,12 @@ func TestResticBackup(t *testing.T) {
summary, err := r.Backup(context.Background(), func(event *BackupEvent) {
t.Logf("backup event: %v", event)
}, tc.opts...)
if err != nil {
t.Errorf("failed to backup: %v", err)
if (err != nil) != tc.wantErr {
t.Fatalf("wanted error: %v, got: %v", tc.wantErr, err)
}
if tc.wantErr {
return
}
if summary == nil {
@@ -82,3 +93,64 @@ func TestResticBackup(t *testing.T) {
}
}
func TestSnapshot(t *testing.T) {
t.Parallel()
repo := t.TempDir()
r := NewRepo(&v1.Repo{
Id: "test",
Uri: repo,
Password: "test",
}, WithRepoFlags("--no-cache"))
testData := test.CreateTestData(t)
for i := 0; i < 10; i++ {
_, err := r.Backup(context.Background(), nil, WithBackupPaths(testData))
if err != nil {
t.Fatalf("failed to backup and create new snapshot: %v", err)
}
}
snapshots, err := r.Snapshots(context.Background())
if err != nil {
t.Fatalf("failed to list snapshots: %v", err)
}
if len(snapshots) != 10 {
t.Errorf("wanted 10 snapshots, got: %d", len(snapshots))
}
data, _ := json.Marshal(snapshots)
t.Logf("snapshots: %v", string(data))
}
func TestLs(t *testing.T) {
t.Parallel()
repo := t.TempDir()
r := NewRepo(&v1.Repo{
Id: "test",
Uri: repo,
Password: "test",
}, WithRepoFlags("--no-cache"))
testData := test.CreateTestData(t)
snapshot, err := r.Backup(context.Background(), nil, WithBackupPaths(testData))
if err != nil {
t.Fatalf("failed to backup and create new snapshot: %v", err)
}
_, entries, err := r.ListDirectory(context.Background(), snapshot.SnapshotId, testData)
if err != nil {
t.Fatalf("failed to list directory: %v", err)
}
if len(entries) != 101 {
t.Errorf("wanted 101 entries, got: %d", len(entries))
}
}