lee/backrest
1
0
Fork 0
mirror of https://github.com/garethgeorge/backrest.git synced 2026-08-26 02:46:28 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: implement forget and prune support in restic pkg

Browse Source
This commit is contained in:
Gareth George
2023-11-24 17:18:12 -08:00
parent b507d42cb9
commit ffb4573737
3 changed files with 258 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/restic/outputs.go
+33 -28
View File
@@ -13,34 +13,6 @@ import (
v1 "github.com/garethgeorge/resticui/gen/go/v1"
)
type LsEntry struct {
Name string `json:"name"`
Type string `json:"type"`
Path string `json:"path"`
Uid int `json:"uid"`
Gid int `json:"gid"`
Size int `json:"size"`
Mode int `json:"mode"`
Mtime string `json:"mtime"`
Atime string `json:"atime"`
Ctime string `json:"ctime"`
}
func (e *LsEntry) ToProto() *v1.LsEntry {
return &v1.LsEntry{
Name: e.Name,
Type: e.Type,
Path: e.Path,
Uid: int64(e.Uid),
Gid: int64(e.Gid),
Size: int64(e.Size),
Mode: int64(e.Mode),
Mtime: e.Mtime,
Atime: e.Atime,
Ctime: e.Ctime,
}
}
type Snapshot struct {
Id string `json:"id"`
Time string `json:"time"`
@@ -189,6 +161,34 @@ func readBackupProgressEntries(cmd *exec.Cmd, output io.Reader, callback func(ev
return summary, nil
}
type LsEntry struct {
Name string `json:"name"`
Type string `json:"type"`
Path string `json:"path"`
Uid int `json:"uid"`
Gid int `json:"gid"`
Size int `json:"size"`
Mode int `json:"mode"`
Mtime string `json:"mtime"`
Atime string `json:"atime"`
Ctime string `json:"ctime"`
}
func (e *LsEntry) ToProto() *v1.LsEntry {
return &v1.LsEntry{
Name: e.Name,
Type: e.Type,
Path: e.Path,
Uid: int64(e.Uid),
Gid: int64(e.Gid),
Size: int64(e.Size),
Mode: int64(e.Mode),
Mtime: e.Mtime,
Atime: e.Atime,
Ctime: e.Ctime,
}
}
func readLs(output io.Reader) (*Snapshot, []*LsEntry, error) {
scanner := bufio.NewScanner(output)
scanner.Split(bufio.ScanLines)
@@ -212,3 +212,8 @@ func readLs(output io.Reader) (*Snapshot, []*LsEntry, error) {
}
return snapshot, entries, nil
}
type ForgetResult struct {
Keep []Snapshot `json:"keep"`
Remove []Snapshot `json:"remove"`
}
pkg/restic/restic.go
+116 -24
View File
@@ -17,13 +17,13 @@ import (
)
type Repo struct {
mu sync.Mutex
cmd string
repo *v1.Repo
mu sync.Mutex
cmd string
repo *v1.Repo
initialized bool
extraArgs []string
extraEnv []string
extraEnv []string
}
// NewRepo instantiates a new repository. TODO: should not accept a v1.Repo, should instead be configured by parameters.
@@ -34,11 +34,11 @@ func NewRepo(repo *v1.Repo, opts ...GenericOption) *Repo {
}
return &Repo{
cmd: "restic", // TODO: configurable binary path
repo: repo,
cmd: "restic", // TODO: configurable binary path
repo: repo,
initialized: false,
extraArgs: opt.extraArgs,
extraEnv: opt.extraEnv,
extraArgs: opt.extraArgs,
extraEnv: opt.extraEnv,
}
}
@@ -85,10 +85,6 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupProgress
r.mu.Lock()
defer r.mu.Unlock()
if err := r.init(ctx); err != nil {
return nil, fmt.Errorf("failed to initialize repo: %w", err)
}
opt := &BackupOpts{}
for _, o := range opts {
o(opt)
@@ -115,12 +111,12 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupProgress
if err := cmd.Start(); err != nil {
return nil, NewCmdError(cmd, nil, err)
}
var wg sync.WaitGroup
var summary *BackupProgressEntry
var cmdErr error
var cmdErr error
var readErr error
wg.Add(1)
go func() {
defer wg.Done()
@@ -141,7 +137,7 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupProgress
}()
wg.Wait()
var err error
if cmdErr != nil || readErr != nil {
err = multierror.Append(nil, cmdErr, readErr)
@@ -153,10 +149,6 @@ func (r *Repo) Snapshots(ctx context.Context, opts ...GenericOption) ([]*Snapsho
r.mu.Lock()
defer r.mu.Unlock()
if err := r.init(ctx); err != nil {
return nil, fmt.Errorf("failed to initialize repo: %w", err)
}
opt := resolveOpts(opts)
args := []string{"snapshots", "--json"}
@@ -180,6 +172,60 @@ func (r *Repo) Snapshots(ctx context.Context, opts ...GenericOption) ([]*Snapsho
return snapshots, nil
}
func (r *Repo) Forget(ctx context.Context, policy RetentionPolicy, pruneOutput io.Writer, opts ...GenericOption) (*ForgetResult, error) {
r.mu.Lock()
defer r.mu.Unlock()
// first run the forget command
opt := resolveOpts(opts)
args := []string{"forget", "--json"}
args = append(args, r.extraArgs...)
args = append(args, opt.extraArgs...)
args = append(args, policy.toForgetFlags()...)
cmd := exec.CommandContext(ctx, r.cmd, args...)
cmd.Env = append(cmd.Env, r.buildEnv()...)
cmd.Env = append(cmd.Env, opt.extraEnv...)
output, err := cmd.CombinedOutput()
if err != nil {
return nil, NewCmdError(cmd, output, err)
}
var result []ForgetResult
if err := json.Unmarshal(output, &result); err != nil {
return nil, NewCmdError(cmd, output, fmt.Errorf("command output is not valid JSON: %w", err))
}
if len(result) != 1 {
return nil, fmt.Errorf("expected 1 output from forget, got %v", len(result))
}
// then run the prune command
args = []string{"prune", "--json"}
args = append(args, r.extraArgs...)
args = append(args, opt.extraArgs...)
args = append(args, policy.toPruneFlags()...)
cmd = exec.CommandContext(ctx, r.cmd, args...)
cmd.Env = append(cmd.Env, r.buildEnv()...)
cmd.Env = append(cmd.Env, opt.extraEnv...)
buf := bytes.NewBuffer(nil)
var writer io.Writer = buf
if pruneOutput != nil {
writer = io.MultiWriter(pruneOutput, buf)
}
cmd.Stdout = writer
cmd.Stderr = writer
if err := cmd.Run(); err != nil {
return nil, NewCmdError(cmd, buf.Bytes(), err)
}
return &result[0], nil
}
func (r *Repo) ListDirectory(ctx context.Context, snapshot string, path string, opts ...GenericOption) (*Snapshot, []*LsEntry, error) {
r.mu.Lock()
defer r.mu.Unlock()
@@ -216,8 +262,53 @@ func (r *Repo) ListDirectory(ctx context.Context, snapshot string, path string,
return snapshots, entries, nil
}
type RetentionPolicy struct {
MaxUnused string // e.g. a percentage i.e. 25% or a number of megabytes.
KeepLastN int // keep the last n snapshots.
KeepHourly int // keep the last n hourly snapshots.
KeepDaily int // keep the last n daily snapshots.
KeepWeekly int // keep the last n weekly snapshots.
KeepMonthly int // keep the last n monthly snapshots.
KeepYearly int // keep the last n yearly snapshots.
KeepWithinDuration string // keep snapshots within a duration e.g. 1y2m3d4h5m6s
}
func (r *RetentionPolicy) toForgetFlags() []string {
flags := []string{}
if r.KeepLastN != 0 {
flags = append(flags, "--keep-last", fmt.Sprintf("%d", r.KeepLastN))
}
if r.KeepHourly != 0 {
flags = append(flags, "--keep-hourly", fmt.Sprintf("%d", r.KeepHourly))
}
if r.KeepDaily != 0 {
flags = append(flags, "--keep-daily", fmt.Sprintf("%d", r.KeepDaily))
}
if r.KeepWeekly != 0 {
flags = append(flags, "--keep-weekly", fmt.Sprintf("%d", r.KeepWeekly))
}
if r.KeepMonthly != 0 {
flags = append(flags, "--keep-monthly", fmt.Sprintf("%d", r.KeepMonthly))
}
if r.KeepYearly != 0 {
flags = append(flags, "--keep-yearly", fmt.Sprintf("%d", r.KeepYearly))
}
if r.KeepWithinDuration != "" {
flags = append(flags, "--keep-within", r.KeepWithinDuration)
}
return flags
}
func (r *RetentionPolicy) toPruneFlags() []string {
flags := []string{}
if r.MaxUnused != "" {
flags = append(flags, "--max-unused", r.MaxUnused)
}
return flags
}
type BackupOpts struct {
paths []string
paths []string
extraArgs []string
}
@@ -253,7 +344,7 @@ func WithBackupParent(parent string) BackupOption {
type GenericOpts struct {
extraArgs []string
extraEnv []string
extraEnv []string
}
func resolveOpts(opts []GenericOption) *GenericOpts {
@@ -287,14 +378,15 @@ func WithEnv(env ...string) GenericOption {
}
var EnvToPropagate = []string{"PATH", "HOME", "XDG_CACHE_HOME"}
func WithPropagatedEnvVars(extras ...string) GenericOption {
var extension []string
for _, env := range EnvToPropagate {
if val, ok := os.LookupEnv(env); ok {
extension = append(extension, env + "=" + val)
extension = append(extension, env+"="+val)
}
}
return WithEnv(extension...)
}
}
pkg/restic/restic_test.go
+109 -30
View File
@@ -1,8 +1,12 @@
package restic
import (
"bytes"
"context"
"fmt"
"reflect"
"slices"
"strings"
"testing"
v1 "github.com/garethgeorge/resticui/gen/go/v1"
@@ -14,12 +18,14 @@ func TestResticInit(t *testing.T) {
repo := t.TempDir()
r := NewRepo(&v1.Repo{
Id: "test",
Uri: repo,
Id: "test",
Uri: repo,
Password: "test",
}, WithFlags("--no-cache"))
r.init(context.Background())
if err := r.Init(context.Background()); err != nil {
t.Fatalf("failed to init repo: %v", err)
}
}
func TestResticBackup(t *testing.T) {
@@ -28,43 +34,46 @@ func TestResticBackup(t *testing.T) {
// create a new repo with cache disabled for testing
r := NewRepo(&v1.Repo{
Id: "test",
Uri: repo,
Id: "test",
Uri: repo,
Password: "test",
}, WithFlags("--no-cache"))
if err := r.Init(context.Background()); err != nil {
t.Fatalf("failed to init repo: %v", err)
}
testData := test.CreateTestData(t)
testData2 := test.CreateTestData(t)
var tests = []struct {
name string
opts []BackupOption
files int // expected files at the end of the backup
name string
opts []BackupOption
files int // expected files at the end of the backup
wantErr bool
}{
{
name: "no options",
opts: []BackupOption{WithBackupPaths(testData)},
name: "no options",
opts: []BackupOption{WithBackupPaths(testData)},
files: 100,
},
{
name: "with two paths",
opts:[]BackupOption{WithBackupPaths(testData), WithBackupPaths(testData2)},
name: "with two paths",
opts: []BackupOption{WithBackupPaths(testData), WithBackupPaths(testData2)},
files: 200,
},
{
name: "with exclude",
opts: []BackupOption{WithBackupPaths(testData), WithBackupExcludes("file1*")},
name: "with exclude",
opts: []BackupOption{WithBackupPaths(testData), WithBackupExcludes("file1*")},
files: 90,
},
{
name: "with exclude pattern",
opts: []BackupOption{WithBackupPaths(testData), WithBackupExcludes("file*")},
name: "with exclude pattern",
opts: []BackupOption{WithBackupPaths(testData), WithBackupExcludes("file*")},
files: 0,
},
{
name: "with nothing to backup",
opts: []BackupOption{},
name: "with nothing to backup",
opts: []BackupOption{},
wantErr: true,
},
}
@@ -99,10 +108,13 @@ func TestSnapshot(t *testing.T) {
repo := t.TempDir()
r := NewRepo(&v1.Repo{
Id: "test",
Uri: repo,
Id: "test",
Uri: repo,
Password: "test",
}, WithFlags("--no-cache"))
if err := r.Init(context.Background()); err != nil {
t.Fatalf("failed to init repo: %v", err)
}
testData := test.CreateTestData(t)
@@ -114,18 +126,18 @@ func TestSnapshot(t *testing.T) {
}
var tests = []struct {
name string
opts []GenericOption
name string
opts []GenericOption
count int
}{
{
name: "no options",
opts: []GenericOption{},
name: "no options",
opts: []GenericOption{},
count: 10,
},
{
name: "with tag",
opts: []GenericOption{WithTags("tag1")},
name: "with tag",
opts: []GenericOption{WithTags("tag1")},
count: 1,
},
}
@@ -156,10 +168,13 @@ func TestLs(t *testing.T) {
repo := t.TempDir()
r := NewRepo(&v1.Repo{
Id: "test",
Uri: repo,
Id: "test",
Uri: repo,
Password: "test",
}, WithFlags("--no-cache"))
if err := r.Init(context.Background()); err != nil {
t.Fatalf("failed to init repo: %v", err)
}
testData := test.CreateTestData(t)
@@ -177,4 +192,68 @@ func TestLs(t *testing.T) {
if len(entries) != 101 {
t.Errorf("wanted 101 entries, got: %d", len(entries))
}
}
}
func TestResticForget(t *testing.T) {
t.Parallel()
repo := t.TempDir()
r := NewRepo(&v1.Repo{
Id: "test",
Uri: repo,
Password: "test",
}, WithFlags("--no-cache"))
if err := r.Init(context.Background()); err != nil {
t.Fatalf("failed to init repo: %v", err)
}
testData := test.CreateTestData(t)
ids := make([]string, 0)
for i := 0; i < 10; i++ {
output, err := r.Backup(context.Background(), nil, WithBackupPaths(testData))
if err != nil {
t.Fatalf("failed to backup and create new snapshot: %v", err)
}
ids = append(ids, output.SnapshotId)
}
// prune all snapshots
output := bytes.NewBuffer(nil)
res, err := r.Forget(context.Background(), RetentionPolicy{KeepLastN: 3}, output)
if err != nil {
t.Fatalf("failed to prune snapshots: %v", err)
}
if len(res.Keep) != 3 {
t.Errorf("wanted 3 snapshots to be kept, got: %d", len(res.Keep))
}
if len(res.Remove) != 7 {
t.Errorf("wanted 7 snapshots to be removed, got: %d", len(res.Remove))
}
removedIds := make([]string, 0)
for _, snapshot := range res.Remove {
removedIds = append(removedIds, snapshot.Id)
}
slices.Reverse(removedIds)
keptIds := make([]string, 0)
for _, snapshot := range res.Keep {
keptIds = append(keptIds, snapshot.Id)
}
slices.Reverse(keptIds)
if !reflect.DeepEqual(removedIds, ids[:7]) {
t.Errorf("wanted removed ids to be %v, got: %v", ids[:7], removedIds)
}
if !reflect.DeepEqual(keptIds, ids[7:]) {
t.Errorf("wanted kept ids to be %v, got: %v", ids[7:], keptIds)
}
if !strings.Contains(output.String(), "total prune") {
t.Errorf("wanted prune output, got: %s", output.String())
}
}