lee/backrest
1
0
Fork 0
mirror of https://github.com/garethgeorge/backrest.git synced 2025-12-12 16:55:39 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix: limit cmd log length to 32KB per operation

Browse Source
This commit is contained in:
garethgeorge
2024-04-06 14:10:02 -07:00
parent 3770966111
commit 92d52bed8e
5 changed files with 16 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
go.mod
View File

@@ -15,9 +15,10 @@ require (
go.etcd.io/bbolt v1.3.9
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.21.0
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8
golang.org/x/net v0.22.0
golang.org/x/sync v0.6.0
google.golang.org/genproto/googleapis/api v0.0.0-20240325203815-454cdb8f5daa
google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda
google.golang.org/grpc v1.62.1
google.golang.org/protobuf v1.33.0
)
@@ -31,5 +32,5 @@ require (
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect
)

6
go.sum
View File

@@ -56,6 +56,8 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw=
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
@@ -70,8 +72,12 @@ golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
google.golang.org/genproto/googleapis/api v0.0.0-20240325203815-454cdb8f5daa h1:Jt1XW5PaLXF1/ePZrznsh/aAUvI7Adfc3LY1dAKlzRs=
google.golang.org/genproto/googleapis/api v0.0.0-20240325203815-454cdb8f5daa/go.mod h1:K4kfzHtI0kqWA79gecJarFtDn/Mls+GxQcg3Zox91Ac=
google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda h1:b6F6WIV4xHHD0FA4oIyzU6mHWg2WI2X1RBehwa5QN38=
google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda/go.mod h1:AHcE/gZH76Bk/ROZhQphlRoWo5xKDEtz3eVEO1LfA8c=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa h1:RBgMaUMP+6soRkik4VoN8ojR2nex2TqZwjSSogic+eo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk=
google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=

41
internal/fsutil/collapsedirs.go
View File

@@ -1,41 +0,0 @@
package fsutil
import (
"fmt"
"os"
"path"
)
// CollapseSubdirectory recursively moves the subdir up towards parent and removes any (newly empty) directories on the way.
func CollapseSubdirectory(p, subdir string) (string, error) {
if p == subdir {
return p, nil
}
dir := path.Dir(subdir)
ents, err := os.ReadDir(dir)
if err != nil {
return "", fmt.Errorf("read dir %q: %w", dir, err)
}
base := path.Base(subdir)
for _, ent := range ents {
if ent.Name() != base && ent.Name() != "." && ent.Name() != ".." {
return subdir, nil
}
}
if err := os.Rename(subdir, dir+".tmp"); err != nil {
return "", fmt.Errorf("rename %q to %q: %w", subdir, dir+".tmp", err)
}
if err := os.Remove(dir); err != nil {
return "", fmt.Errorf("remove %q: %w", dir, err)
}
if err := os.Rename(dir+".tmp", base); err != nil {
return "", fmt.Errorf("rename %q to %q: %w", dir+".tmp", base, err)
}
return CollapseSubdirectory(p, dir)
}

6
internal/orchestrator/task.go
View File

@@ -1,7 +1,6 @@
package orchestrator
import (
"bytes"
"context"
"errors"
"fmt"
@@ -9,6 +8,7 @@ import (
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/ioutil"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/pkg/restic"
"github.com/hashicorp/go-multierror"
@@ -62,8 +62,8 @@ func (t *TaskWithOperation) runWithOpAndContext(ctx context.Context, do func(ctx
}()
return WithOperation(t.orch.OpLog, t.op, func() error {
buf := bytes.NewBuffer(nil)
ctx = restic.ContextWithLogger(ctx, buf)
capture := ioutil.NewOutputCapturer(32_000) // 32k of logs
ctx = restic.ContextWithLogger(ctx, capture)
err := do(ctx, t.op)

8
pkg/restic/restic.go
View File

@@ -12,6 +12,8 @@ import (
"slices"
"strings"
"sync"
"github.com/garethgeorge/backrest/internal/ioutil"
)
var errAlreadyInitialized = errors.New("repo already initialized")
@@ -124,7 +126,7 @@ func (r *Repo) Backup(ctx context.Context, paths []string, progressCallback func
args = append(args, paths...)
cmd := r.commandWithContext(ctx, args, opts...)
capture := newOutputCapturer(outputBufferLimit)
capture := ioutil.NewOutputCapturer(outputBufferLimit)
reader, writer := io.Pipe()
r.pipeCmdOutputToWriter(cmd, writer, capture)
@@ -209,7 +211,6 @@ func (r *Repo) Forget(ctx context.Context, policy *RetentionPolicy, opts ...Gene
cmd := r.commandWithContext(ctx, args, opts...)
output := bytes.NewBuffer(nil)
r.pipeCmdOutputToWriter(cmd, output)
r.pipeCmdOutputToLogger(ctx, cmd)
if err := cmd.Run(); err != nil {
return nil, newCmdError(ctx, cmd, output.String(), err)
}
@@ -234,7 +235,6 @@ func (r *Repo) ForgetSnapshot(ctx context.Context, snapshotId string, opts ...Ge
cmd := r.commandWithContext(ctx, args, opts...)
output := bytes.NewBuffer(nil)
r.pipeCmdOutputToWriter(cmd, output)
r.pipeCmdOutputToLogger(ctx, cmd)
if err := cmd.Run(); err != nil {
return newCmdError(ctx, cmd, output.String(), err)
}
@@ -259,7 +259,7 @@ func (r *Repo) Prune(ctx context.Context, pruneOutput io.Writer, opts ...Generic
func (r *Repo) Restore(ctx context.Context, snapshot string, callback func(*RestoreProgressEntry), opts ...GenericOption) (*RestoreProgressEntry, error) {
cmd := r.commandWithContext(ctx, []string{"restore", "--json", snapshot}, opts...)
output := newOutputCapturer(outputBufferLimit)
output := ioutil.NewOutputCapturer(outputBufferLimit)
reader, writer := io.Pipe()
r.pipeCmdOutputToWriter(cmd, output, writer)
r.pipeCmdOutputToLogger(ctx, cmd)