From beecfcf0016f0558794e4ba905fb87a06340620a Mon Sep 17 00:00:00 2001 From: Gareth George Date: Wed, 28 Feb 2024 20:05:36 -0800 Subject: [PATCH] fix: properly parse repo flags --- go.mod | 21 +++++++++++---------- go.sum | 22 ++++++++++++++++++++++ internal/orchestrator/orchestrator.go | 15 ++++----------- internal/orchestrator/repo.go | 21 +++++++++++++++++++-- internal/orchestrator/repo_test.go | 11 ++++++++--- internal/orchestrator/taskbackup.go | 2 +- 6 files changed, 65 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 73848024..90c6059b 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,21 @@ module github.com/garethgeorge/backrest go 1.21 require ( - connectrpc.com/connect v1.14.0 + connectrpc.com/connect v1.15.0 github.com/alessio/shellescape v1.4.2 github.com/gitploy-io/cronexpr v0.2.2 github.com/golang-jwt/jwt/v5 v5.2.0 + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/hashicorp/go-multierror v1.1.1 github.com/mattn/go-colorable v0.1.13 github.com/natefinch/atomic v1.0.1 - go.etcd.io/bbolt v1.3.8 - go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.18.0 - golang.org/x/net v0.20.0 + go.etcd.io/bbolt v1.3.9 + go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.20.0 + golang.org/x/net v0.21.0 golang.org/x/sync v0.6.0 - google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe - google.golang.org/grpc v1.61.0 + google.golang.org/genproto/googleapis/api v0.0.0-20240228224816-df926f6c8641 + google.golang.org/grpc v1.62.0 google.golang.org/protobuf v1.32.0 ) @@ -26,8 +27,8 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/stretchr/testify v1.8.4 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/sys v0.16.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect + google.golang.org/genproto v0.0.0-20240228224816-df926f6c8641 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240228224816-df926f6c8641 // indirect ) diff --git a/go.sum b/go.sum index 9d173f16..92b97795 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ connectrpc.com/connect v1.14.0 h1:PDS+J7uoz5Oui2VEOMcfz6Qft7opQM9hPiKvtGC01pA= connectrpc.com/connect v1.14.0/go.mod h1:uoAq5bmhhn43TwhaKdGKN/bZcGtzPW1v+ngDTn5u+8s= +connectrpc.com/connect v1.15.0 h1:lFdeCbZrVVDydAqwr4xGV2y+ULn+0Z73s5JBj2LikWo= +connectrpc.com/connect v1.15.0/go.mod h1:bQmjpDY8xItMnttnurVgOkHUBMRT9cpsNi2O4AjKhmA= github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -14,6 +16,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -32,33 +36,51 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= +go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= +golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe h1:USL2DhxfgRchafRvt/wYyyQNzwgL7ZiURcozOE/Pkvo= google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= +google.golang.org/genproto v0.0.0-20240228224816-df926f6c8641 h1:GihpvzHjeZHw+/mzsWpdxwr1LaG6E3ff/gyeZlVHbyc= +google.golang.org/genproto v0.0.0-20240228224816-df926f6c8641/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe h1:0poefMBYvYbs7g5UkjS6HcxBPaTRAmznle9jnxYoAI8= google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/api v0.0.0-20240228224816-df926f6c8641 h1:SO1wX9btGFrwj9EzH3ocqfwiPVOxfv4ggAJajzlHA5s= +google.golang.org/genproto/googleapis/api v0.0.0-20240228224816-df926f6c8641/go.mod h1:wLupoVsUfYPgOMwjzhYFbaVklw/INms+dqTp0tc1fv8= google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe h1:bQnxqljG/wqi4NTXu2+DJ3n7APcEA882QZ1JvhQAq9o= google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240228224816-df926f6c8641 h1:DKU1r6Tj5s1vlU/moGhuGz7E3xRfwjdAfDzbsaQJtEY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240228224816-df926f6c8641/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= +google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go index 80306e1d..9659183c 100644 --- a/internal/orchestrator/orchestrator.go +++ b/internal/orchestrator/orchestrator.go @@ -14,7 +14,6 @@ import ( "github.com/garethgeorge/backrest/internal/hook" "github.com/garethgeorge/backrest/internal/oplog" "github.com/garethgeorge/backrest/internal/rotatinglog" - "github.com/garethgeorge/backrest/pkg/restic" "go.uber.org/zap" "google.golang.org/protobuf/proto" ) @@ -317,17 +316,11 @@ func (rp *resticRepoPool) GetRepo(repoId string) (repo *RepoOrchestrator, err er } delete(rp.repos, repoId) - var opts []restic.GenericOption - opts = append(opts, restic.WithPropagatedEnvVars(restic.EnvToPropagate...)) - if len(repoProto.GetEnv()) > 0 { - opts = append(opts, restic.WithEnv(repoProto.GetEnv()...)) - } - if len(repoProto.GetFlags()) > 0 { - opts = append(opts, restic.WithFlags(repoProto.GetFlags()...)) - } - // Otherwise create a new repo. - repo = newRepoOrchestrator(repoProto, restic.NewRepo(rp.resticPath, repoProto, opts...)) + repo, err = newRepoOrchestrator(repoProto, rp.resticPath) + if err != nil { + return nil, err + } rp.repos[repoId] = repo return repo, nil } diff --git a/internal/orchestrator/repo.go b/internal/orchestrator/repo.go index a0a12179..ddbcd01f 100644 --- a/internal/orchestrator/repo.go +++ b/internal/orchestrator/repo.go @@ -11,6 +11,7 @@ import ( v1 "github.com/garethgeorge/backrest/gen/go/v1" "github.com/garethgeorge/backrest/internal/protoutil" "github.com/garethgeorge/backrest/pkg/restic" + "github.com/google/shlex" "go.uber.org/zap" "google.golang.org/protobuf/proto" ) @@ -26,12 +27,28 @@ type RepoOrchestrator struct { } // newRepoOrchestrator accepts a config and a repo that is configured with the properties of that config object. -func newRepoOrchestrator(repoConfig *v1.Repo, repo *restic.Repo) *RepoOrchestrator { +func newRepoOrchestrator(repoConfig *v1.Repo, resticPath string) (*RepoOrchestrator, error) { + var opts []restic.GenericOption + opts = append(opts, restic.WithPropagatedEnvVars(restic.EnvToPropagate...)) + if len(repoConfig.GetEnv()) > 0 { + opts = append(opts, restic.WithEnv(repoConfig.GetEnv()...)) + } + + for _, f := range repoConfig.GetFlags() { + args, err := shlex.Split(f) + if err != nil { + return nil, fmt.Errorf("failed to parse flag %q for repo %q: %w", f, repoConfig.Id, err) + } + opts = append(opts, restic.WithFlags(args...)) + } + + repo := restic.NewRepo(resticPath, repoConfig, opts...) + return &RepoOrchestrator{ repoConfig: repoConfig, repo: repo, l: zap.L().With(zap.String("repo", repoConfig.Id)), - } + }, nil } func (r *RepoOrchestrator) Snapshots(ctx context.Context) ([]*restic.Snapshot, error) { diff --git a/internal/orchestrator/repo_test.go b/internal/orchestrator/repo_test.go index 3efe6997..a2e1ee6d 100644 --- a/internal/orchestrator/repo_test.go +++ b/internal/orchestrator/repo_test.go @@ -6,7 +6,6 @@ import ( "testing" v1 "github.com/garethgeorge/backrest/gen/go/v1" - "github.com/garethgeorge/backrest/pkg/restic" "github.com/garethgeorge/backrest/test/helpers" test "github.com/garethgeorge/backrest/test/helpers" ) @@ -31,7 +30,10 @@ func TestBackup(t *testing.T) { Paths: []string{testData}, } - orchestrator := newRepoOrchestrator(r, restic.NewRepo(helpers.ResticBinary(t), r, restic.WithFlags("--no-cache"))) + orchestrator, err := newRepoOrchestrator(r, helpers.ResticBinary(t)) + if err != nil { + t.Fatalf("failed to create repo orchestrator: %v", err) + } summary, err := orchestrator.Backup(context.Background(), plan, nil) if err != nil { @@ -74,7 +76,10 @@ func TestSnapshotParenting(t *testing.T) { }, } - orchestrator := newRepoOrchestrator(r, restic.NewRepo(helpers.ResticBinary(t), r, restic.WithFlags("--no-cache"))) + orchestrator, err := newRepoOrchestrator(r, helpers.ResticBinary(t)) + if err != nil { + t.Fatalf("failed to create repo orchestrator: %v", err) + } for i := 0; i < 4; i++ { for _, plan := range plans { diff --git a/internal/orchestrator/taskbackup.go b/internal/orchestrator/taskbackup.go index e4f7678d..e92fc760 100644 --- a/internal/orchestrator/taskbackup.go +++ b/internal/orchestrator/taskbackup.go @@ -128,7 +128,7 @@ func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan backupOp.OperationBackup.LastStatus = protoutil.BackupProgressEntryToProto(entry) } else if entry.MessageType == "error" { - zap.S().Warnf("backup error: %v", entry.Item) + zap.S().Warnf("an unknown error was encountered in processing item: %v", entry.Item) backupError, err := protoutil.BackupProgressEntryToBackupError(entry) if err != nil { zap.S().Errorf("failed to convert backup progress entry to backup error: %v", err)