From 92d52bed8e84d6cd8dd331a1fa52a6e2d30cb7a7 Mon Sep 17 00:00:00 2001 From: garethgeorge Date: Sat, 6 Apr 2024 14:10:02 -0700 Subject: [PATCH] fix: limit cmd log length to 32KB per operation --- go.mod | 5 ++-- go.sum | 6 +++++ internal/fsutil/collapsedirs.go | 41 --------------------------------- internal/orchestrator/task.go | 6 ++--- pkg/restic/restic.go | 8 +++---- 5 files changed, 16 insertions(+), 50 deletions(-) delete mode 100644 internal/fsutil/collapsedirs.go diff --git a/go.mod b/go.mod index 8871699..09aa3bd 100644 --- a/go.mod +++ b/go.mod @@ -15,9 +15,10 @@ require ( go.etcd.io/bbolt v1.3.9 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.21.0 + golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 golang.org/x/net v0.22.0 golang.org/x/sync v0.6.0 - google.golang.org/genproto/googleapis/api v0.0.0-20240325203815-454cdb8f5daa + google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda google.golang.org/grpc v1.62.1 google.golang.org/protobuf v1.33.0 ) @@ -31,5 +32,5 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect ) diff --git a/go.sum b/go.sum index dfb3cdf..e227dc5 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= @@ -70,8 +72,12 @@ golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= google.golang.org/genproto/googleapis/api v0.0.0-20240325203815-454cdb8f5daa h1:Jt1XW5PaLXF1/ePZrznsh/aAUvI7Adfc3LY1dAKlzRs= google.golang.org/genproto/googleapis/api v0.0.0-20240325203815-454cdb8f5daa/go.mod h1:K4kfzHtI0kqWA79gecJarFtDn/Mls+GxQcg3Zox91Ac= +google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda h1:b6F6WIV4xHHD0FA4oIyzU6mHWg2WI2X1RBehwa5QN38= +google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda/go.mod h1:AHcE/gZH76Bk/ROZhQphlRoWo5xKDEtz3eVEO1LfA8c= google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa h1:RBgMaUMP+6soRkik4VoN8ojR2nex2TqZwjSSogic+eo= google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= diff --git a/internal/fsutil/collapsedirs.go b/internal/fsutil/collapsedirs.go deleted file mode 100644 index 056e1aa..0000000 --- a/internal/fsutil/collapsedirs.go +++ /dev/null @@ -1,41 +0,0 @@ -package fsutil - -import ( - "fmt" - "os" - "path" -) - -// CollapseSubdirectory recursively moves the subdir up towards parent and removes any (newly empty) directories on the way. -func CollapseSubdirectory(p, subdir string) (string, error) { - if p == subdir { - return p, nil - } - - dir := path.Dir(subdir) - ents, err := os.ReadDir(dir) - if err != nil { - return "", fmt.Errorf("read dir %q: %w", dir, err) - } - - base := path.Base(subdir) - for _, ent := range ents { - if ent.Name() != base && ent.Name() != "." && ent.Name() != ".." { - return subdir, nil - } - } - - if err := os.Rename(subdir, dir+".tmp"); err != nil { - return "", fmt.Errorf("rename %q to %q: %w", subdir, dir+".tmp", err) - } - - if err := os.Remove(dir); err != nil { - return "", fmt.Errorf("remove %q: %w", dir, err) - } - - if err := os.Rename(dir+".tmp", base); err != nil { - return "", fmt.Errorf("rename %q to %q: %w", dir+".tmp", base, err) - } - - return CollapseSubdirectory(p, dir) -} diff --git a/internal/orchestrator/task.go b/internal/orchestrator/task.go index b39d721..96441cf 100644 --- a/internal/orchestrator/task.go +++ b/internal/orchestrator/task.go @@ -1,7 +1,6 @@ package orchestrator import ( - "bytes" "context" "errors" "fmt" @@ -9,6 +8,7 @@ import ( "time" v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/ioutil" "github.com/garethgeorge/backrest/internal/oplog" "github.com/garethgeorge/backrest/pkg/restic" "github.com/hashicorp/go-multierror" @@ -62,8 +62,8 @@ func (t *TaskWithOperation) runWithOpAndContext(ctx context.Context, do func(ctx }() return WithOperation(t.orch.OpLog, t.op, func() error { - buf := bytes.NewBuffer(nil) - ctx = restic.ContextWithLogger(ctx, buf) + capture := ioutil.NewOutputCapturer(32_000) // 32k of logs + ctx = restic.ContextWithLogger(ctx, capture) err := do(ctx, t.op) diff --git a/pkg/restic/restic.go b/pkg/restic/restic.go index 289c4d8..e2223b5 100644 --- a/pkg/restic/restic.go +++ b/pkg/restic/restic.go @@ -12,6 +12,8 @@ import ( "slices" "strings" "sync" + + "github.com/garethgeorge/backrest/internal/ioutil" ) var errAlreadyInitialized = errors.New("repo already initialized") @@ -124,7 +126,7 @@ func (r *Repo) Backup(ctx context.Context, paths []string, progressCallback func args = append(args, paths...) cmd := r.commandWithContext(ctx, args, opts...) - capture := newOutputCapturer(outputBufferLimit) + capture := ioutil.NewOutputCapturer(outputBufferLimit) reader, writer := io.Pipe() r.pipeCmdOutputToWriter(cmd, writer, capture) @@ -209,7 +211,6 @@ func (r *Repo) Forget(ctx context.Context, policy *RetentionPolicy, opts ...Gene cmd := r.commandWithContext(ctx, args, opts...) output := bytes.NewBuffer(nil) r.pipeCmdOutputToWriter(cmd, output) - r.pipeCmdOutputToLogger(ctx, cmd) if err := cmd.Run(); err != nil { return nil, newCmdError(ctx, cmd, output.String(), err) } @@ -234,7 +235,6 @@ func (r *Repo) ForgetSnapshot(ctx context.Context, snapshotId string, opts ...Ge cmd := r.commandWithContext(ctx, args, opts...) output := bytes.NewBuffer(nil) r.pipeCmdOutputToWriter(cmd, output) - r.pipeCmdOutputToLogger(ctx, cmd) if err := cmd.Run(); err != nil { return newCmdError(ctx, cmd, output.String(), err) } @@ -259,7 +259,7 @@ func (r *Repo) Prune(ctx context.Context, pruneOutput io.Writer, opts ...Generic func (r *Repo) Restore(ctx context.Context, snapshot string, callback func(*RestoreProgressEntry), opts ...GenericOption) (*RestoreProgressEntry, error) { cmd := r.commandWithContext(ctx, []string{"restore", "--json", snapshot}, opts...) - output := newOutputCapturer(outputBufferLimit) + output := ioutil.NewOutputCapturer(outputBufferLimit) reader, writer := io.Pipe() r.pipeCmdOutputToWriter(cmd, output, writer) r.pipeCmdOutputToLogger(ctx, cmd)