lee/backrest
1
0
Fork 0
mirror of https://github.com/garethgeorge/backrest.git synced 2026-08-25 02:16:38 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

refactor restic api options

Browse Source
This commit is contained in:
garethgeorge
2023-11-10 20:38:43 -08:00
parent b9b7950798
commit 1a7e6504cd
7 changed files with 148 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/state/state.go
-4
View File
@@ -1,4 +0,0 @@
package state
type State interface {
}
pkg/restic/outputs.go
+7 -6
View File
@@ -30,9 +30,10 @@ type Snapshot struct {
Username string `json:"username"`
Id string `json:"id"`
ShortId string `json:"short_id"`
Tags []string `json:"tags"`
}
type BackupEvent struct {
type BackupProgressEntry struct {
// Common fields
MessageType string `json:"message_type"` // "summary" or "status"
@@ -62,14 +63,14 @@ type BackupEvent struct {
Error string `json:"error"`
}
// readBackupEvents returns the summary event or an error if the command failed.
func readBackupEvents(cmd *exec.Cmd, output io.Reader, callback func(event *BackupEvent)) (*BackupEvent, error) {
// readBackupProgressEntrys returns the summary event or an error if the command failed.
func readBackupProgressEntries(cmd *exec.Cmd, output io.Reader, callback func(event *BackupProgressEntry)) (*BackupProgressEntry, error) {
scanner := bufio.NewScanner(output)
scanner.Split(bufio.ScanLines)
// first event is handled specially to detect non-JSON output and fast-path out.
if scanner.Scan() {
var event BackupEvent
var event BackupProgressEntry
if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
var bytes = slices.Clone(scanner.Bytes())
@@ -82,10 +83,10 @@ func readBackupEvents(cmd *exec.Cmd, output io.Reader, callback func(event *Back
}
// remaining events are parsed as JSON
var summary *BackupEvent
var summary *BackupProgressEntry
for scanner.Scan() {
var event *BackupEvent
var event *BackupProgressEntry
if err := json.Unmarshal(scanner.Bytes(), &event); err != nil {
return nil, fmt.Errorf("failed to parse JSON: %w", err)
}
pkg/restic/outputs_test.go
+3 -3
View File
@@ -6,14 +6,14 @@ import (
"testing"
)
func TestReadBackupEvents(t *testing.T) {
func TestReadBackupProgressEntries(t *testing.T) {
t.Parallel()
testInput := `{"message_type":"status","percent_done":0,"total_files":1,"total_bytes":15}
{"message_type":"summary","files_new":0,"files_changed":0,"files_unmodified":166,"dirs_new":0,"dirs_changed":0,"dirs_unmodified":128,"data_blobs":0,"tree_blobs":0,"data_added":0,"total_files_processed":166,"total_bytes_processed":16754463,"total_duration":0.235433378,"snapshot_id":"bca1043e"}`
b := bytes.NewBuffer([]byte(testInput))
summary, err := readBackupEvents(&exec.Cmd{}, b, func(event *BackupEvent) {
summary, err := readBackupProgressEntries(&exec.Cmd{}, b, func(event *BackupProgressEntry) {
t.Logf("event: %v", event)
})
if err != nil {
@@ -28,7 +28,7 @@ func TestReadBackupEvents(t *testing.T) {
}
func TestReadLs(t *testing.T) {
testInput := `{"time":"2023-11-10T19:14:17.053824063-08:00","tree":"3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2","paths":["/home/dontpanic/Documents/github/garethgeorge/resticui"],"hostname":"pop-os","username":"dontpanic","uid":1000,"gid":1000,"id":"db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52","short_id":"db155169","struct_type":"snapshot"}
testInput := `{"time":"2023-11-10T19:14:17.053824063-08:00","tree":"3e2918b261948e69602ee9504b8f475bcc7cdc4dcec0b3f34ecdb014287d07b2","paths":["/resticui"],"hostname":"pop-os","username":"dontpanic","uid":1000,"gid":1000,"id":"db155169d788e6e432e320aedbdff5a54cc439653093bb56944a67682528aa52","short_id":"db155169","struct_type":"snapshot"}
{"name":".git","type":"dir","path":"/.git","uid":1000,"gid":1000,"mode":2147484157,"mtime":"2023-11-10T18:32:38.156599473-08:00","atime":"2023-11-10T18:32:38.156599473-08:00","ctime":"2023-11-10T18:32:38.156599473-08:00","struct_type":"node"}
{"name":".gitignore","type":"file","path":"/.gitignore","uid":1000,"gid":1000,"size":22,"mode":436,"mtime":"2023-11-10T00:41:26.611346634-08:00","atime":"2023-11-10T00:41:26.611346634-08:00","ctime":"2023-11-10T00:41:26.611346634-08:00","struct_type":"node"}
{"name":"README.md","type":"file","path":"/README.md","uid":1000,"gid":1000,"size":762,"mode":436,"mtime":"2023-11-10T00:59:06.842538768-08:00","atime":"2023-11-10T00:59:06.842538768-08:00","ctime":"2023-11-10T00:59:06.842538768-08:00","struct_type":"node"}`
pkg/restic/restic.go
+88 -49
View File
@@ -19,22 +19,24 @@ type Repo struct {
mu sync.Mutex
cmd string
repo *v1.Repo
flags []string
env []string
initialized bool
extraArgs []string
extraEnv []string
}
func NewRepo(repo *v1.Repo, opts ...RepoOption) *Repo {
var opt RepoOpts
func NewRepo(repo *v1.Repo, opts ...GenericOption) *Repo {
opt := &GenericOpts{}
for _, o := range opts {
o(&opt)
o(opt)
}
return &Repo{
cmd: "restic", // TODO: configurable binary path
repo: repo,
flags: opt.flags,
env: opt.env,
initialized: false,
extraArgs: opt.extraArgs,
extraEnv: opt.extraEnv,
}
}
@@ -43,8 +45,8 @@ func (r *Repo) buildEnv() []string {
"RESTIC_REPOSITORY=" + r.repo.GetUri(),
"RESTIC_PASSWORD=" + r.repo.GetPassword(),
}
env = append(env, r.extraEnv...)
env = append(env, r.repo.GetEnv()...)
env = append(env, r.env...)
return env
}
@@ -55,7 +57,7 @@ func (r *Repo) init(ctx context.Context) error {
}
var args = []string{"init", "--json"}
args = append(args, r.flags...)
args = append(args, r.extraArgs...)
cmd := exec.CommandContext(ctx, r.cmd, args...)
cmd.Env = append(cmd.Env, r.buildEnv()...)
@@ -75,7 +77,7 @@ func (r *Repo) Init(ctx context.Context) error {
return r.init(ctx)
}
func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent), opts ...BackupOption) (*BackupEvent, error) {
func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupProgressEntry), opts ...BackupOption) (*BackupProgressEntry, error) {
r.mu.Lock()
defer r.mu.Unlock()
@@ -95,12 +97,9 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent),
}
args := []string{"backup", "--json", "--exclude-caches"}
args = append(args, r.flags...)
args = append(args, r.extraArgs...)
args = append(args, opt.paths...)
for _, e := range opt.excludes {
args = append(args, "--exclude", e)
}
args = append(args, opt.extraArgs...)
reader, writer := io.Pipe()
@@ -114,7 +113,7 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent),
}
var wg sync.WaitGroup
var summary *BackupEvent
var summary *BackupProgressEntry
var cmdErr error
var readErr error
@@ -122,7 +121,7 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent),
go func() {
defer wg.Done()
var err error
summary, err = readBackupEvents(cmd, reader, progressCallback)
summary, err = readBackupProgressEntries(cmd, reader, progressCallback)
if err != nil {
readErr = fmt.Errorf("processing command output: %w", err)
}
@@ -146,7 +145,7 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent),
return summary, err
}
func (r *Repo) Snapshots(ctx context.Context) ([]*Snapshot, error) {
func (r *Repo) Snapshots(ctx context.Context, opts ...GenericOption) ([]*Snapshot, error) {
r.mu.Lock()
defer r.mu.Unlock()
@@ -154,11 +153,15 @@ func (r *Repo) Snapshots(ctx context.Context) ([]*Snapshot, error) {
return nil, fmt.Errorf("failed to initialize repo: %w", err)
}
opt := resolveOpts(opts)
args := []string{"snapshots", "--json"}
args = append(args, r.flags...)
args = append(args, r.extraArgs...)
args = append(args, opt.extraArgs...)
cmd := exec.CommandContext(ctx, r.cmd, args...)
cmd.Env = append(cmd.Env, r.buildEnv()...)
cmd.Env = append(cmd.Env, opt.extraEnv...)
output, err := cmd.CombinedOutput()
if err != nil {
@@ -173,7 +176,7 @@ func (r *Repo) Snapshots(ctx context.Context) ([]*Snapshot, error) {
return snapshots, nil
}
func (r *Repo) ListDirectory(ctx context.Context, snapshot string, path string) (*Snapshot, []*LsEntry, error) {
func (r *Repo) ListDirectory(ctx context.Context, snapshot string, path string, opts ...GenericOption) (*Snapshot, []*LsEntry, error) {
r.mu.Lock()
defer r.mu.Unlock()
@@ -186,11 +189,15 @@ func (r *Repo) ListDirectory(ctx context.Context, snapshot string, path string)
return nil, nil, fmt.Errorf("failed to initialize repo: %w", err)
}
opt := resolveOpts(opts)
args := []string{"ls", "--json", snapshot, path}
args = append(args, r.flags...)
args = append(args, r.extraArgs...)
args = append(args, opt.extraArgs...)
cmd := exec.CommandContext(ctx, r.cmd, args...)
cmd.Env = append(cmd.Env, r.buildEnv()...)
cmd.Env = append(cmd.Env, opt.extraEnv...)
output, err := cmd.CombinedOutput()
if err != nil {
@@ -206,35 +213,9 @@ func (r *Repo) ListDirectory(ctx context.Context, snapshot string, path string)
return snapshots, entries, nil
}
type RepoOpts struct {
env []string // global env overrides
flags []string // global flags
}
type RepoOption func(opts *RepoOpts)
// WithHostEnv copies values from the host environment into the restic environment.
func WithRepoHostEnv() RepoOption {
return func(opts *RepoOpts) {
opts.env = append(opts.env, "HOME=" + os.Getenv("HOME"), "XDG_CACHE_HOME=" + os.Getenv("XDG_CACHE_HOME"))
}
}
func WithRepoEnv(env ...string) RepoOption {
return func(opts *RepoOpts) {
opts.env = append(opts.env, env...)
}
}
func WithRepoFlags(flags ...string) RepoOption {
return func(opts *RepoOpts) {
opts.flags = append(opts.flags, flags...)
}
}
type BackupOpts struct {
paths []string
excludes []string
extraArgs []string
}
type BackupOption func(opts *BackupOpts)
@@ -247,6 +228,64 @@ func WithBackupPaths(paths ...string) BackupOption {
func WithBackupExcludes(excludes ...string) BackupOption {
return func(opts *BackupOpts) {
opts.excludes = append(opts.excludes, excludes...)
for _, exclude := range excludes {
opts.extraArgs = append(opts.extraArgs, "--exclude", exclude)
}
}
}
func WithBackupTags(tags ...string) BackupOption {
return func(opts *BackupOpts) {
for _, tag := range tags {
opts.extraArgs = append(opts.extraArgs, "--tag", tag)
}
}
}
type GenericOpts struct {
extraArgs []string
extraEnv []string
}
func resolveOpts(opts []GenericOption) *GenericOpts {
opt := &GenericOpts{}
for _, o := range opts {
o(opt)
}
return opt
}
type GenericOption func(opts *GenericOpts)
func WithFlags(flags ...string) GenericOption {
return func(opts *GenericOpts) {
opts.extraArgs = append(opts.extraArgs, flags...)
}
}
func WithTags(tags ...string) GenericOption {
return func(opts *GenericOpts) {
for _, tag := range tags {
opts.extraArgs = append(opts.extraArgs, "--tag", tag)
}
}
}
func WithEnv(env ...string) GenericOption {
return func(opts *GenericOpts) {
opts.extraEnv = append(opts.extraEnv, env...)
}
}
var EnvToPropagate = []string{"PATH", "HOME", "XDG_CACHE_HOME"}
func WithPropagatedEnvVars(extras ...string) GenericOption {
var extension []string
for _, env := range EnvToPropagate {
if val, ok := os.LookupEnv(env); ok {
extension = append(extension, env + "=" + val)
}
}
return WithEnv(extension...)
}
pkg/restic/restic_test.go
+33 -16
View File
@@ -2,7 +2,7 @@ package restic
import (
"context"
"encoding/json"
"fmt"
"testing"
v1 "github.com/garethgeorge/resticui/gen/go/v1"
@@ -17,7 +17,7 @@ func TestResticInit(t *testing.T) {
Id: "test",
Uri: repo,
Password: "test",
}, WithRepoFlags("--no-cache"))
}, WithFlags("--no-cache"))
r.init(context.Background())
}
@@ -31,7 +31,7 @@ func TestResticBackup(t *testing.T) {
Id: "test",
Uri: repo,
Password: "test",
}, WithRepoFlags("--no-cache"))
}, WithFlags("--no-cache"))
testData := test.CreateTestData(t)
testData2 := test.CreateTestData(t)
@@ -71,7 +71,7 @@ func TestResticBackup(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
summary, err := r.Backup(context.Background(), func(event *BackupEvent) {
summary, err := r.Backup(context.Background(), func(event *BackupProgressEntry) {
t.Logf("backup event: %v", event)
}, tc.opts...)
if (err != nil) != tc.wantErr {
@@ -102,29 +102,46 @@ func TestSnapshot(t *testing.T) {
Id: "test",
Uri: repo,
Password: "test",
}, WithRepoFlags("--no-cache"))
}, WithFlags("--no-cache"))
testData := test.CreateTestData(t)
for i := 0; i < 10; i++ {
_, err := r.Backup(context.Background(), nil, WithBackupPaths(testData))
_, err := r.Backup(context.Background(), nil, WithBackupPaths(testData), WithBackupTags(fmt.Sprintf("tag%d", i)))
if err != nil {
t.Fatalf("failed to backup and create new snapshot: %v", err)
}
}
snapshots, err := r.Snapshots(context.Background())
if err != nil {
t.Fatalf("failed to list snapshots: %v", err)
var tests = []struct {
name string
opts []GenericOption
count int
}{
{
name: "no options",
opts: []GenericOption{},
count: 10,
},
{
name: "with tag",
opts: []GenericOption{WithTags("tag1")},
count: 1,
},
}
if len(snapshots) != 10 {
t.Errorf("wanted 10 snapshots, got: %d", len(snapshots))
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
snapshots, err := r.Snapshots(context.Background(), tc.opts...)
if err != nil {
t.Fatalf("failed to list snapshots: %v", err)
}
if len(snapshots) != tc.count {
t.Errorf("wanted %d snapshots, got: %d", tc.count, len(snapshots))
}
})
}
data, _ := json.Marshal(snapshots)
t.Logf("snapshots: %v", string(data))
}
func TestLs(t *testing.T) {
@@ -135,7 +152,7 @@ func TestLs(t *testing.T) {
Id: "test",
Uri: repo,
Password: "test",
}, WithRepoFlags("--no-cache"))
}, WithFlags("--no-cache"))
testData := test.CreateTestData(t)
proto/v1/config.proto
+6 -2
View File
@@ -11,6 +11,11 @@ message Config {
repeated Plan plans = 4 [json_name="plans"];
}
message User {
string name = 1;
string password = 2; // plaintext password
}
message Repo {
string id = 1 [json_name="id"];
string uri = 2 [json_name="uri"];
@@ -21,6 +26,5 @@ message Repo {
message Plan {
string id = 1 [json_name="id"];
string repo = 2 [json_name="repo"];
string repo_path = 3 [json_name="repo_path"]; // subpath of the repo to backup to
repeated string paths = 4 [json_name="paths"];
}
}
webui/package.json
+11
View File
@@ -0,0 +1,11 @@
{
"name": "webui",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC"
}