diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 73f98b5..08d2aef 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -27,10 +27,10 @@ jobs: node-version: "20" - name: Install Deps - run: ./hack/install-deps.sh + run: ./scripts/install-deps.sh - name: Build - run: ./hack/build.sh + run: ./scripts/build.sh - name: Test run: PATH=$(pwd):$PATH go test ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4a5663b..41f48b6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,14 +25,10 @@ jobs: node-version: "20" - name: Install Deps - run: ./hack/install-deps.sh + run: ./scripts/install-deps.sh - name: Build - run: ./hack/build.sh - - - name: Rename Files - run: | - mv resticui resticui-linux-amd64 + run: ./scripts/build-all.sh - uses: "marvinpinto/action-automatic-releases@latest" with: @@ -41,3 +37,6 @@ jobs: files: | LICENSE resticui-linux-amd64 + resticui-linux-arm64 + resticui-darwin-amd64 + resticui-darwin-arm64 diff --git a/.gitignore b/.gitignore index 901c7fd..c15bb77 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ resticui +resticui-* diff --git a/README.md b/README.md index e923c1d..2b86f08 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ ResticUI is accessible from a web browser. By default it binds to `0.0.0.0:9898` * `RESTICUI_CONFIG_PATH` - the path to the config file. Defaults to `$HOME/.config/resticui/config.json` or if `$XDG_CONFIG_HOME` is set, `$XDG_CONFIG_HOME/resticui/config.json`. * `RESTICUI_DATA_DIR` - the path to the data directory. Defaults to `$HOME/.local/share/resticui` or if `$XDG_DATA_HOME` is set, `$XDG_DATA_HOME/resticui`. * `RESTICUI_RESTIC_BIN_PATH` - the path to the restic binary. Defaults managed version of restic which will be downloaded and installed in the data directory. + * `XDG_CACHE_HOME` -- the path to the cache directory. This is propagated to restic. ## Screenshots diff --git a/gen/go/v1/config.pb.go b/gen/go/v1/config.pb.go index f523e61..0581899 100644 --- a/gen/go/v1/config.pb.go +++ b/gen/go/v1/config.pb.go @@ -99,11 +99,12 @@ type Repo struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // unique but human readable ID for this repo. - Uri string `protobuf:"bytes,2,opt,name=uri,proto3" json:"uri,omitempty"` // restic repo URI - Password string `protobuf:"bytes,3,opt,name=password,proto3" json:"password,omitempty"` // plaintext password - Env []string `protobuf:"bytes,4,rep,name=env,proto3" json:"env,omitempty"` // extra environment variables to set for restic. - Flags []string `protobuf:"bytes,5,rep,name=flags,proto3" json:"flags,omitempty"` // extra flags set on the restic command. + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // unique but human readable ID for this repo. + Uri string `protobuf:"bytes,2,opt,name=uri,proto3" json:"uri,omitempty"` // restic repo URI + Password string `protobuf:"bytes,3,opt,name=password,proto3" json:"password,omitempty"` // plaintext password + Env []string `protobuf:"bytes,4,rep,name=env,proto3" json:"env,omitempty"` // extra environment variables to set for restic. + Flags []string `protobuf:"bytes,5,rep,name=flags,proto3" json:"flags,omitempty"` // extra flags set on the restic command. + PrunePolicy *PrunePolicy `protobuf:"bytes,6,opt,name=prune_policy,json=prunePolicy,proto3" json:"prune_policy,omitempty"` // policy for when to run prune. } func (x *Repo) Reset() { @@ -173,6 +174,13 @@ func (x *Repo) GetFlags() []string { return nil } +func (x *Repo) GetPrunePolicy() *PrunePolicy { + if x != nil { + return x.PrunePolicy + } + return nil +} + type Plan struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -274,7 +282,6 @@ type RetentionPolicy struct { KeepMonthly int32 `protobuf:"varint,6,opt,name=keep_monthly,json=keepMonthly,proto3" json:"keep_monthly,omitempty"` // keep the last n monthly snapshots. KeepYearly int32 `protobuf:"varint,7,opt,name=keep_yearly,json=keepYearly,proto3" json:"keep_yearly,omitempty"` // keep the last n yearly snapshots. KeepWithinDuration string `protobuf:"bytes,8,opt,name=keep_within_duration,json=keepWithinDuration,proto3" json:"keep_within_duration,omitempty"` // keep snapshots within a duration e.g. 1y2m3d4h5m6s - Prune bool `protobuf:"varint,9,opt,name=prune,proto3" json:"prune,omitempty"` // prune snapshots after forget. } func (x *RetentionPolicy) Reset() { @@ -365,13 +372,95 @@ func (x *RetentionPolicy) GetKeepWithinDuration() string { return "" } -func (x *RetentionPolicy) GetPrune() bool { - if x != nil { - return x.Prune - } - return false +type PrunePolicy struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + MaxFrequencyDays int32 `protobuf:"varint,1,opt,name=max_frequency_days,json=maxFrequencyDays,proto3" json:"max_frequency_days,omitempty"` // max frequency of prune runs in days. If 0, prune will be run on every backup. + // Types that are assignable to Policy: + // + // *PrunePolicy_MaxUnusedPercent + // *PrunePolicy_MaxUnusedBytes + Policy isPrunePolicy_Policy `protobuf_oneof:"policy"` } +func (x *PrunePolicy) Reset() { + *x = PrunePolicy{} + if protoimpl.UnsafeEnabled { + mi := &file_v1_config_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PrunePolicy) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PrunePolicy) ProtoMessage() {} + +func (x *PrunePolicy) ProtoReflect() protoreflect.Message { + mi := &file_v1_config_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PrunePolicy.ProtoReflect.Descriptor instead. +func (*PrunePolicy) Descriptor() ([]byte, []int) { + return file_v1_config_proto_rawDescGZIP(), []int{4} +} + +func (x *PrunePolicy) GetMaxFrequencyDays() int32 { + if x != nil { + return x.MaxFrequencyDays + } + return 0 +} + +func (m *PrunePolicy) GetPolicy() isPrunePolicy_Policy { + if m != nil { + return m.Policy + } + return nil +} + +func (x *PrunePolicy) GetMaxUnusedPercent() int32 { + if x, ok := x.GetPolicy().(*PrunePolicy_MaxUnusedPercent); ok { + return x.MaxUnusedPercent + } + return 0 +} + +func (x *PrunePolicy) GetMaxUnusedBytes() int32 { + if x, ok := x.GetPolicy().(*PrunePolicy_MaxUnusedBytes); ok { + return x.MaxUnusedBytes + } + return 0 +} + +type isPrunePolicy_Policy interface { + isPrunePolicy_Policy() +} + +type PrunePolicy_MaxUnusedPercent struct { + MaxUnusedPercent int32 `protobuf:"varint,100,opt,name=max_unused_percent,json=maxUnusedPercent,proto3,oneof"` // max percentage of repo size that can be unused before prune is run. +} + +type PrunePolicy_MaxUnusedBytes struct { + MaxUnusedBytes int32 `protobuf:"varint,101,opt,name=max_unused_bytes,json=maxUnusedBytes,proto3,oneof"` // max number of bytes that can be unused before prune is run. +} + +func (*PrunePolicy_MaxUnusedPercent) isPrunePolicy_Policy() {} + +func (*PrunePolicy_MaxUnusedBytes) isPrunePolicy_Policy() {} + var File_v1_config_proto protoreflect.FileDescriptor var file_v1_config_proto_rawDesc = []byte{ @@ -383,48 +472,61 @@ var file_v1_config_proto_rawDesc = []byte{ 0x6f, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x52, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x1e, 0x0a, 0x05, 0x70, 0x6c, 0x61, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6c, - 0x61, 0x6e, 0x52, 0x05, 0x70, 0x6c, 0x61, 0x6e, 0x73, 0x22, 0x6c, 0x0a, 0x04, 0x52, 0x65, 0x70, - 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, - 0x64, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x69, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x75, 0x72, 0x69, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, - 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x76, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x65, 0x6e, - 0x76, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x22, 0xa3, 0x01, 0x0a, 0x04, 0x50, 0x6c, 0x61, 0x6e, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x72, 0x65, 0x70, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x78, - 0x63, 0x6c, 0x75, 0x64, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x65, 0x78, - 0x63, 0x6c, 0x75, 0x64, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x72, 0x6f, 0x6e, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x72, 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x09, 0x72, 0x65, - 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x6c, 0x69, - 0x63, 0x79, 0x52, 0x09, 0x72, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc8, 0x02, - 0x0a, 0x0f, 0x52, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x6c, 0x69, 0x63, - 0x79, 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x61, 0x78, 0x5f, 0x75, 0x6e, 0x75, 0x73, 0x65, 0x64, 0x5f, - 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6d, 0x61, 0x78, - 0x55, 0x6e, 0x75, 0x73, 0x65, 0x64, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1e, 0x0a, 0x0b, 0x6b, - 0x65, 0x65, 0x70, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x09, 0x6b, 0x65, 0x65, 0x70, 0x4c, 0x61, 0x73, 0x74, 0x4e, 0x12, 0x1f, 0x0a, 0x0b, 0x6b, - 0x65, 0x65, 0x70, 0x5f, 0x68, 0x6f, 0x75, 0x72, 0x6c, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x0a, 0x6b, 0x65, 0x65, 0x70, 0x48, 0x6f, 0x75, 0x72, 0x6c, 0x79, 0x12, 0x1d, 0x0a, 0x0a, - 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x64, 0x61, 0x69, 0x6c, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x09, 0x6b, 0x65, 0x65, 0x70, 0x44, 0x61, 0x69, 0x6c, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x6b, - 0x65, 0x65, 0x70, 0x5f, 0x77, 0x65, 0x65, 0x6b, 0x6c, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x0a, 0x6b, 0x65, 0x65, 0x70, 0x57, 0x65, 0x65, 0x6b, 0x6c, 0x79, 0x12, 0x21, 0x0a, 0x0c, - 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x6d, 0x6f, 0x6e, 0x74, 0x68, 0x6c, 0x79, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x0b, 0x6b, 0x65, 0x65, 0x70, 0x4d, 0x6f, 0x6e, 0x74, 0x68, 0x6c, 0x79, 0x12, - 0x1f, 0x0a, 0x0b, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x79, 0x65, 0x61, 0x72, 0x6c, 0x79, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6b, 0x65, 0x65, 0x70, 0x59, 0x65, 0x61, 0x72, 0x6c, 0x79, - 0x12, 0x30, 0x0a, 0x14, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x77, 0x69, 0x74, 0x68, 0x69, 0x6e, 0x5f, - 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, - 0x6b, 0x65, 0x65, 0x70, 0x57, 0x69, 0x74, 0x68, 0x69, 0x6e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x72, 0x75, 0x6e, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x05, 0x70, 0x72, 0x75, 0x6e, 0x65, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, - 0x72, 0x67, 0x65, 0x2f, 0x72, 0x65, 0x73, 0x74, 0x69, 0x63, 0x75, 0x69, 0x2f, 0x67, 0x6f, 0x2f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x6e, 0x52, 0x05, 0x70, 0x6c, 0x61, 0x6e, 0x73, 0x22, 0xa0, 0x01, 0x0a, 0x04, 0x52, 0x65, + 0x70, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, + 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x69, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x75, 0x72, 0x69, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, + 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x76, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x65, + 0x6e, 0x76, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x32, 0x0a, 0x0c, 0x70, 0x72, 0x75, 0x6e, + 0x65, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, + 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, + 0x0b, 0x70, 0x72, 0x75, 0x6e, 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, 0xa3, 0x01, 0x0a, + 0x04, 0x50, 0x6c, 0x61, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, + 0x68, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x12, + 0x1a, 0x0a, 0x08, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x08, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, + 0x72, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x72, 0x6f, 0x6e, 0x12, + 0x31, 0x0a, 0x09, 0x72, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, + 0x6e, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x09, 0x72, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, + 0x6f, 0x6e, 0x22, 0xb2, 0x02, 0x0a, 0x0f, 0x52, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, + 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x61, 0x78, 0x5f, 0x75, 0x6e, + 0x75, 0x73, 0x65, 0x64, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x55, 0x6e, 0x75, 0x73, 0x65, 0x64, 0x4c, 0x69, 0x6d, 0x69, 0x74, + 0x12, 0x1e, 0x0a, 0x0b, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x6b, 0x65, 0x65, 0x70, 0x4c, 0x61, 0x73, 0x74, 0x4e, + 0x12, 0x1f, 0x0a, 0x0b, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x68, 0x6f, 0x75, 0x72, 0x6c, 0x79, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6b, 0x65, 0x65, 0x70, 0x48, 0x6f, 0x75, 0x72, 0x6c, + 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x64, 0x61, 0x69, 0x6c, 0x79, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x6b, 0x65, 0x65, 0x70, 0x44, 0x61, 0x69, 0x6c, 0x79, + 0x12, 0x1f, 0x0a, 0x0b, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x77, 0x65, 0x65, 0x6b, 0x6c, 0x79, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6b, 0x65, 0x65, 0x70, 0x57, 0x65, 0x65, 0x6b, 0x6c, + 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x6d, 0x6f, 0x6e, 0x74, 0x68, 0x6c, + 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x6b, 0x65, 0x65, 0x70, 0x4d, 0x6f, 0x6e, + 0x74, 0x68, 0x6c, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x79, 0x65, 0x61, + 0x72, 0x6c, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6b, 0x65, 0x65, 0x70, 0x59, + 0x65, 0x61, 0x72, 0x6c, 0x79, 0x12, 0x30, 0x0a, 0x14, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x77, 0x69, + 0x74, 0x68, 0x69, 0x6e, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x12, 0x6b, 0x65, 0x65, 0x70, 0x57, 0x69, 0x74, 0x68, 0x69, 0x6e, 0x44, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xa1, 0x01, 0x0a, 0x0b, 0x50, 0x72, 0x75, 0x6e, + 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x2c, 0x0a, 0x12, 0x6d, 0x61, 0x78, 0x5f, 0x66, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x64, 0x61, 0x79, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x10, 0x6d, 0x61, 0x78, 0x46, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, + 0x79, 0x44, 0x61, 0x79, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x6d, 0x61, 0x78, 0x5f, 0x75, 0x6e, 0x75, + 0x73, 0x65, 0x64, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x64, 0x20, 0x01, 0x28, + 0x05, 0x48, 0x00, 0x52, 0x10, 0x6d, 0x61, 0x78, 0x55, 0x6e, 0x75, 0x73, 0x65, 0x64, 0x50, 0x65, + 0x72, 0x63, 0x65, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x10, 0x6d, 0x61, 0x78, 0x5f, 0x75, 0x6e, 0x75, + 0x73, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x65, 0x20, 0x01, 0x28, 0x05, 0x48, + 0x00, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x55, 0x6e, 0x75, 0x73, 0x65, 0x64, 0x42, 0x79, 0x74, 0x65, + 0x73, 0x42, 0x08, 0x0a, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x42, 0x2e, 0x5a, 0x2c, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, + 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x72, 0x65, 0x73, 0x74, 0x69, 0x63, 0x75, 0x69, 0x2f, + 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -439,22 +541,24 @@ func file_v1_config_proto_rawDescGZIP() []byte { return file_v1_config_proto_rawDescData } -var file_v1_config_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_v1_config_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_v1_config_proto_goTypes = []interface{}{ (*Config)(nil), // 0: v1.Config (*Repo)(nil), // 1: v1.Repo (*Plan)(nil), // 2: v1.Plan (*RetentionPolicy)(nil), // 3: v1.RetentionPolicy + (*PrunePolicy)(nil), // 4: v1.PrunePolicy } var file_v1_config_proto_depIdxs = []int32{ 1, // 0: v1.Config.repos:type_name -> v1.Repo 2, // 1: v1.Config.plans:type_name -> v1.Plan - 3, // 2: v1.Plan.retention:type_name -> v1.RetentionPolicy - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 4, // 2: v1.Repo.prune_policy:type_name -> v1.PrunePolicy + 3, // 3: v1.Plan.retention:type_name -> v1.RetentionPolicy + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_v1_config_proto_init() } @@ -511,6 +615,22 @@ func file_v1_config_proto_init() { return nil } } + file_v1_config_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PrunePolicy); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_v1_config_proto_msgTypes[4].OneofWrappers = []interface{}{ + (*PrunePolicy_MaxUnusedPercent)(nil), + (*PrunePolicy_MaxUnusedBytes)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -518,7 +638,7 @@ func file_v1_config_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_v1_config_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 0, }, diff --git a/gen/go/v1/operations.pb.go b/gen/go/v1/operations.pb.go index dd9954c..91b5535 100644 --- a/gen/go/v1/operations.pb.go +++ b/gen/go/v1/operations.pb.go @@ -467,7 +467,9 @@ type OperationIndexSnapshot struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Snapshot *ResticSnapshot `protobuf:"bytes,2,opt,name=snapshot,proto3" json:"snapshot,omitempty"` + Snapshot *ResticSnapshot `protobuf:"bytes,2,opt,name=snapshot,proto3" json:"snapshot,omitempty"` // the snapshot that was indexed. + Forgot bool `protobuf:"varint,3,opt,name=forgot,proto3" json:"forgot,omitempty"` // tracks whether this snapshot is forgotten yet. + ForgotByOp int64 `protobuf:"varint,4,opt,name=forgot_by_op,json=forgotByOp,proto3" json:"forgot_by_op,omitempty"` // ID of a forget operation that removed this snapshot. } func (x *OperationIndexSnapshot) Reset() { @@ -509,6 +511,20 @@ func (x *OperationIndexSnapshot) GetSnapshot() *ResticSnapshot { return nil } +func (x *OperationIndexSnapshot) GetForgot() bool { + if x != nil { + return x.Forgot + } + return false +} + +func (x *OperationIndexSnapshot) GetForgotByOp() int64 { + if x != nil { + return x.ForgotByOp + } + return 0 +} + // OperationForget tracks a forget operation. type OperationForget struct { state protoimpl.MessageState @@ -662,37 +678,41 @@ var file_v1_operations_proto_rawDesc = []byte{ 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x22, 0x48, 0x0a, 0x16, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, - 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x08, 0x73, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, - 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x52, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x22, 0x3d, 0x0a, 0x0f, 0x4f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2a, 0x0a, - 0x06, 0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x22, 0x82, 0x01, 0x0a, 0x16, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, + 0x64, 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x08, 0x73, + 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, - 0x74, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x22, 0x28, 0x0a, 0x0e, 0x4f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, - 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x2a, 0x4d, 0x0a, 0x12, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, - 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x11, 0x0a, 0x0d, - 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, - 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, - 0x10, 0x02, 0x2a, 0xae, 0x01, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, - 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, - 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x15, - 0x0a, 0x11, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e, 0x50, 0x52, 0x4f, 0x47, 0x52, - 0x45, 0x53, 0x53, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x41, - 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x12, 0x1b, 0x0a, 0x17, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, 0x43, 0x41, 0x4e, - 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, - 0x44, 0x10, 0x06, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x72, - 0x65, 0x73, 0x74, 0x69, 0x63, 0x75, 0x69, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x52, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x66, + 0x6f, 0x72, 0x67, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x66, 0x6f, 0x72, + 0x67, 0x6f, 0x74, 0x12, 0x20, 0x0a, 0x0c, 0x66, 0x6f, 0x72, 0x67, 0x6f, 0x74, 0x5f, 0x62, 0x79, + 0x5f, 0x6f, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x66, 0x6f, 0x72, 0x67, 0x6f, + 0x74, 0x42, 0x79, 0x4f, 0x70, 0x22, 0x3d, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2a, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x67, + 0x65, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x06, 0x66, 0x6f, + 0x72, 0x67, 0x65, 0x74, 0x22, 0x28, 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x2a, 0x4d, + 0x0a, 0x12, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, + 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, + 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, + 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x02, 0x2a, 0xae, 0x01, + 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, + 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, + 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x54, 0x41, + 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x02, + 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, + 0x53, 0x53, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, + 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x5f, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, + 0x44, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x53, + 0x45, 0x52, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x06, 0x42, 0x2e, + 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, + 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x72, 0x65, 0x73, 0x74, 0x69, 0x63, + 0x75, 0x69, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/internal/api/api.go b/internal/api/api.go index 88ce13c..d25c270 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -42,7 +42,7 @@ func loggingFunc(l *zap.Logger) logging.Logger { case logging.LevelDebug: logger.Debug(msg) case logging.LevelInfo: - logger.Info(msg) + logger.Debug(msg) case logging.LevelWarn: logger.Warn(msg) case logging.LevelError: diff --git a/internal/api/server.go b/internal/api/server.go index 984e4c4..ee993de 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -194,22 +194,26 @@ func (s *Server) GetOperationEvents(_ *emptypb.Empty, stream v1.ResticUI_GetOper } func (s *Server) GetOperations(ctx context.Context, req *v1.GetOperationsRequest) (*v1.OperationList, error) { - collector := indexutil.CollectAll() + idCollector := indexutil.CollectAll() if req.LastN != 0 { - collector = indexutil.CollectLastN(int(req.LastN)) + idCollector = indexutil.CollectLastN(int(req.LastN)) } var err error var ops []*v1.Operation + opCollector := func(op *v1.Operation) error { + ops = append(ops, op) + return nil + } if req.RepoId != "" && req.PlanId != "" { return nil, errors.New("cannot specify both repoId and planId") } else if req.PlanId != "" { - ops, err = s.oplog.GetByPlan(req.PlanId, collector) + err = s.oplog.ForEachByPlan(req.PlanId, idCollector, opCollector) } else if req.RepoId != "" { - ops, err = s.oplog.GetByRepo(req.RepoId, collector) + err = s.oplog.ForEachByRepo(req.RepoId, idCollector, opCollector) } else if req.SnapshotId != "" { - ops, err = s.oplog.GetBySnapshotId(req.SnapshotId, collector) + err = s.oplog.ForEachBySnapshotId(req.SnapshotId, idCollector, opCollector) } else if len(req.Ids) > 0 { ops = make([]*v1.Operation, 0, len(req.Ids)) for i, id := range req.Ids { @@ -220,7 +224,7 @@ func (s *Server) GetOperations(ctx context.Context, req *v1.GetOperationsRequest ops = append(ops, op) } } else { - ops, err = s.oplog.GetAll() + err = s.oplog.ForAll(opCollector) } if err != nil { return nil, fmt.Errorf("failed to get operations: %w", err) diff --git a/internal/config/environment.go b/internal/config/environment.go index 86b7a19..bf8c2c3 100644 --- a/internal/config/environment.go +++ b/internal/config/environment.go @@ -41,7 +41,7 @@ func BindAddress() string { } return val } - return ":9898" + return "127.0.0.1:9898" } func ResticBinPath() string { diff --git a/internal/config/validate.go b/internal/config/validate.go index 731bc21..fd881e4 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -37,7 +37,7 @@ func ValidateConfig(c *v1.Config) error { err = multierror.Append(err, fmt.Errorf("plan %s: %w", plan.GetId(), e)) } } - } + } return err } @@ -78,14 +78,13 @@ func validatePlan(plan *v1.Plan, repos map[string]*v1.Repo) error { } if plan.Repo == "" { - err = multierror.Append(err,fmt.Errorf("repo is required")) + err = multierror.Append(err, fmt.Errorf("repo is required")) } if _, ok := repos[plan.Repo]; !ok { err = multierror.Append(err, fmt.Errorf("repo %q not found", plan.Repo)) } - if _, e := cronexpr.Parse(plan.Cron); e != nil { err = multierror.Append(err, fmt.Errorf("invalid cron %q: %w", plan.Cron, e)) } @@ -118,4 +117,4 @@ func validateRetention(policy *v1.RetentionPolicy) error { } } return err -} \ No newline at end of file +} diff --git a/internal/oplog/oplog.go b/internal/oplog/oplog.go index 30f942c..fb2d0d8 100644 --- a/internal/oplog/oplog.go +++ b/internal/oplog/oplog.go @@ -27,9 +27,7 @@ const ( EventTypeOpUpdated = EventType(iota) ) -const ( - schemaVersion int64 = 1 -) +var ErrNotExist = errors.New("operation does not exist") var ( SystemBucket = []byte("oplog.system") // system stores metadata @@ -37,7 +35,6 @@ var ( RepoIndexBucket = []byte("oplog.repo_idx") // repo_index tracks IDs of operations affecting a given repo PlanIndexBucket = []byte("oplog.plan_idx") // plan_index tracks IDs of operations affecting a given plan SnapshotIndexBucket = []byte("oplog.snapshot_idx") // snapshot_index tracks IDs of operations affecting a given snapshot - indexBuckets = [][]byte{RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket} ) // OpLog represents a log of operations performed. @@ -194,7 +191,7 @@ func (o *OpLog) notifyHelper(eventType EventType, op *v1.Operation) { func (o *OpLog) getOperationHelper(b *bolt.Bucket, id int64) (*v1.Operation, error) { bytes := b.Get(serializationutil.Itob(id)) if bytes == nil { - return nil, fmt.Errorf("operation with ID %d does not exist", id) + return nil, ErrNotExist } var op v1.Operation @@ -304,57 +301,45 @@ func (o *OpLog) Get(id int64) (*v1.Operation, error) { return op, nil } -func (o *OpLog) GetByRepo(repoId string, collector indexutil.Collector) ([]*v1.Operation, error) { - var err error - var ops []*v1.Operation - o.db.View(func(tx *bolt.Tx) error { +func (o *OpLog) ForEachByRepo(repoId string, collector indexutil.Collector, do func(op *v1.Operation) error) error { + return o.db.View(func(tx *bolt.Tx) error { ids := collector(indexutil.IndexSearchByteValue(tx.Bucket(RepoIndexBucket), []byte(repoId))) - ops, err = o.getOpsByIds(tx, ids) - return nil + return o.forOpsByIds(tx, ids, do) }) - return ops, err } -func (o *OpLog) GetByPlan(planId string, collector indexutil.Collector) ([]*v1.Operation, error) { - var err error - var ops []*v1.Operation - o.db.View(func(tx *bolt.Tx) error { +func (o *OpLog) ForEachByPlan(planId string, collector indexutil.Collector, do func(op *v1.Operation) error) error { + return o.db.View(func(tx *bolt.Tx) error { ids := collector(indexutil.IndexSearchByteValue(tx.Bucket(PlanIndexBucket), []byte(planId))) - ops, err = o.getOpsByIds(tx, ids) - return nil + return o.forOpsByIds(tx, ids, do) }) - return ops, err } -func (o *OpLog) GetBySnapshotId(snapshotId string, collector indexutil.Collector) ([]*v1.Operation, error) { +func (o *OpLog) ForEachBySnapshotId(snapshotId string, collector indexutil.Collector, do func(op *v1.Operation) error) error { if err := restic.ValidateSnapshotId(snapshotId); err != nil { - return nil, err - } - var err error - var ops []*v1.Operation - o.db.View(func(tx *bolt.Tx) error { - ids := collector(indexutil.IndexSearchByteValue(tx.Bucket(SnapshotIndexBucket), []byte(snapshotId))) - ops, err = o.getOpsByIds(tx, ids) return nil + } + return o.db.View(func(tx *bolt.Tx) error { + ids := collector(indexutil.IndexSearchByteValue(tx.Bucket(SnapshotIndexBucket), []byte(snapshotId))) + return o.forOpsByIds(tx, ids, do) }) - return ops, err } -func (o *OpLog) getOpsByIds(tx *bolt.Tx, ids []int64) ([]*v1.Operation, error) { +func (o *OpLog) forOpsByIds(tx *bolt.Tx, ids []int64, do func(*v1.Operation) error) error { b := tx.Bucket(OpLogBucket) - ops := make([]*v1.Operation, 0, len(ids)) for _, id := range ids { op, err := o.getOperationHelper(b, id) if err != nil { - return nil, err + return err + } + if err := do(op); err != nil { + return err } - ops = append(ops, op) } - return ops, nil + return nil } -func (o *OpLog) GetAll() ([]*v1.Operation, error) { - var ops []*v1.Operation +func (o *OpLog) ForAll(do func(op *v1.Operation) error) error { if err := o.db.View(func(tx *bolt.Tx) error { c := tx.Bucket(OpLogBucket).Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { @@ -362,13 +347,15 @@ func (o *OpLog) GetAll() ([]*v1.Operation, error) { if err := proto.Unmarshal(v, op); err != nil { return fmt.Errorf("error unmarshalling operation: %w", err) } - ops = append(ops, op) + if err := do(op); err != nil { + return err + } } return nil }); err != nil { - return nil, err + return nil } - return ops, nil + return nil } func (o *OpLog) Subscribe(callback *func(EventType, *v1.Operation)) { diff --git a/internal/oplog/oplog_test.go b/internal/oplog/oplog_test.go index 30db929..0829c34 100644 --- a/internal/oplog/oplog_test.go +++ b/internal/oplog/oplog_test.go @@ -194,10 +194,14 @@ func TestListOperation(t *testing.T) { // t.Parallel() var ops []*v1.Operation var err error + collect := func(op *v1.Operation) error { + ops = append(ops, op) + return nil + } if tc.byPlan { - ops, err = log.GetByPlan(tc.id, indexutil.CollectAll()) + err = log.ForEachByPlan(tc.id, indexutil.CollectAll(), collect) } else if tc.byRepo { - ops, err = log.GetByRepo(tc.id, indexutil.CollectAll()) + err = log.ForEachByRepo(tc.id, indexutil.CollectAll(), collect) } else { t.Fatalf("must specify byPlan or byRepo") } @@ -234,21 +238,8 @@ func TestBigIO(t *testing.T) { } } - ops, err := log.GetByPlan("plan1", indexutil.CollectAll()) - if err != nil { - t.Fatalf("error listing operations: %s", err) - } - if len(ops) != count { - t.Errorf("want %v operations, got %d", count, len(ops)) - } - - ops, err = log.GetByRepo("repo1", indexutil.CollectAll()) - if err != nil { - t.Fatalf("error listing operations: %s", err) - } - if len(ops) != count { - t.Errorf("want %v operations, got %d", count, len(ops)) - } + countByPlanHelper(t, log, "plan1", count) + countByRepoHelper(t, log, "repo1", count) } func TestIndexSnapshot(t *testing.T) { @@ -270,9 +261,12 @@ func TestIndexSnapshot(t *testing.T) { t.Fatalf("error adding operation: %s", err) } - ops, err := log.GetBySnapshotId(snapshotId, indexutil.CollectAll()) - if err != nil { - t.Fatalf("error checking for snapshot: %s", err) + var ops []*v1.Operation + if err := log.ForEachBySnapshotId(snapshotId, indexutil.CollectAll(), func(op *v1.Operation) error { + ops = append(ops, op) + return nil + }); err != nil { + t.Fatalf("error listing operations: %s", err) } if len(ops) != 1 { t.Fatalf("want 1 operation, got %d", len(ops)) @@ -303,21 +297,9 @@ func TestUpdateOperation(t *testing.T) { opId := op.Id // Validate initial values are indexed - if ops, err := log.GetByPlan("oldplan", indexutil.CollectAll()); err != nil { - t.Fatalf("error checking for plan: %s", err) - } else if len(ops) != 1 { - t.Fatalf("want 1 operation, got %d", len(ops)) - } - if ops, err := log.GetByRepo("oldrepo", indexutil.CollectAll()); err != nil { - t.Fatalf("error checking for repo: %s", err) - } else if len(ops) != 1 { - t.Fatalf("want 1 operation, got %d", len(ops)) - } - if ops, err := log.GetBySnapshotId(snapshotId, indexutil.CollectAll()); err != nil { - t.Fatalf("error checking for snapshot: %s", err) - } else if len(ops) != 1 { - t.Fatalf("want 1 operation, got %d", len(ops)) - } + countByPlanHelper(t, log, "oldplan", 1) + countByRepoHelper(t, log, "oldrepo", 1) + countBySnapshotIdHelper(t, log, snapshotId, 1) // Update indexed values op.SnapshotId = snapshotId2 @@ -331,40 +313,15 @@ func TestUpdateOperation(t *testing.T) { if opId != op.Id { t.Errorf("want operation ID %d, got %d", opId, op.Id) } - if ops, err := log.GetBySnapshotId(snapshotId2, indexutil.CollectAll()); err != nil { - t.Fatalf("error checking for snapshot: %s", err) - } else if len(ops) != 1 { - t.Fatalf("want 1 operation, got %d", len(ops)) - } - if ops, err := log.GetByPlan("myplan", indexutil.CollectAll()); err != nil { - t.Fatalf("error checking for plan: %s", err) - } else if len(ops) != 1 { - t.Fatalf("want 1 operation, got %d", len(ops)) - } - - if ops, err := log.GetByRepo("myrepo", indexutil.CollectAll()); err != nil { - t.Fatalf("error checking for repo: %s", err) - } else if len(ops) != 1 { - t.Fatalf("want 1 operation, got %d", len(ops)) - } + countByPlanHelper(t, log, "myplan", 1) + countByRepoHelper(t, log, "myrepo", 1) + countBySnapshotIdHelper(t, log, snapshotId2, 1) // Validate prior values are gone - if ops, err := log.GetByPlan("oldplan", indexutil.CollectAll()); err != nil { - t.Fatalf("error checking for plan: %s", err) - } else if len(ops) != 0 { - t.Fatalf("want 0 operations, got %d", len(ops)) - } - if ops, err := log.GetByRepo("oldrepo", indexutil.CollectAll()); err != nil { - t.Fatalf("error checking for repo: %s", err) - } else if len(ops) != 0 { - t.Fatalf("want 0 operations, got %d", len(ops)) - } - if ops, err := log.GetBySnapshotId(snapshotId, indexutil.CollectAll()); err != nil { - t.Fatalf("error checking for snapshot: %s", err) - } else if len(ops) != 0 { - t.Fatalf("want 0 operations, got %d", len(ops)) - } + countByPlanHelper(t, log, "oldplan", 0) + countByRepoHelper(t, log, "oldrepo", 0) + countBySnapshotIdHelper(t, log, snapshotId, 0) } func collectMessages(ops []*v1.Operation) []string { @@ -374,3 +331,45 @@ func collectMessages(ops []*v1.Operation) []string { } return messages } + +func countByRepoHelper(t *testing.T, log *OpLog, repo string, expected int) { + t.Helper() + count := 0 + if err := log.ForEachByRepo(repo, indexutil.CollectAll(), func(op *v1.Operation) error { + count += 1 + return nil + }); err != nil { + t.Fatalf("error listing operations: %s", err) + } + if count != expected { + t.Errorf("want %d operations, got %d", expected, count) + } +} + +func countByPlanHelper(t *testing.T, log *OpLog, plan string, expected int) { + t.Helper() + count := 0 + if err := log.ForEachByPlan(plan, indexutil.CollectAll(), func(op *v1.Operation) error { + count += 1 + return nil + }); err != nil { + t.Fatalf("error listing operations: %s", err) + } + if count != expected { + t.Errorf("want %d operations, got %d", expected, count) + } +} + +func countBySnapshotIdHelper(t *testing.T, log *OpLog, snapshotId string, expected int) { + t.Helper() + count := 0 + if err := log.ForEachBySnapshotId(snapshotId, indexutil.CollectAll(), func(op *v1.Operation) error { + count += 1 + return nil + }); err != nil { + t.Fatalf("error listing operations: %s", err) + } + if count != expected { + t.Errorf("want %d operations, got %d", expected, count) + } +} diff --git a/internal/orchestrator/backup.go b/internal/orchestrator/backup.go index 30ee9c0..b06e253 100644 --- a/internal/orchestrator/backup.go +++ b/internal/orchestrator/backup.go @@ -7,7 +7,6 @@ import ( "time" v1 "github.com/garethgeorge/resticui/gen/go/v1" - "github.com/garethgeorge/resticui/internal/oplog/indexutil" "github.com/garethgeorge/resticui/internal/protoutil" "github.com/garethgeorge/resticui/pkg/restic" "github.com/gitploy-io/cronexpr" @@ -161,72 +160,8 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan } if plan.Retention != nil { - orchestrator.ScheduleTask(NewOneofForgetTask(orchestrator, plan, time.Now())) + orchestrator.ScheduleTask(NewOneofForgetTask(orchestrator, plan, op.SnapshotId, time.Now())) } return nil } - -func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan) error { - repo, err := orchestrator.GetRepo(plan.Repo) - if err != nil { - return fmt.Errorf("couldn't get repo %q: %w", plan.Repo, err) - } - - snapshots, err := repo.SnapshotsForPlan(ctx, plan) - if err != nil { - return fmt.Errorf("get snapshots for plan %q: %w", plan.Id, err) - } - - startTime := time.Now() - alreadyIndexed := 0 - var indexOps []*v1.Operation - for _, snapshot := range snapshots { - ops, err := orchestrator.OpLog.GetBySnapshotId(snapshot.Id, indexutil.CollectAll()) - if err != nil { - return fmt.Errorf("HasIndexSnapshot for snapshot %q: %w", snapshot.Id, err) - } - - if containsSnapshotOperation(ops) { - alreadyIndexed += 1 - continue - } - - snapshotProto := protoutil.SnapshotToProto(snapshot) - indexOps = append(indexOps, &v1.Operation{ - RepoId: plan.Repo, - PlanId: plan.Id, - UnixTimeStartMs: snapshotProto.UnixTimeMs, - UnixTimeEndMs: snapshotProto.UnixTimeMs, - Status: v1.OperationStatus_STATUS_SUCCESS, - SnapshotId: snapshotProto.Id, - Op: &v1.Operation_OperationIndexSnapshot{ - OperationIndexSnapshot: &v1.OperationIndexSnapshot{ - Snapshot: snapshotProto, - }, - }, - }) - } - - if err := orchestrator.OpLog.BulkAdd(indexOps); err != nil { - return fmt.Errorf("BulkAdd snapshot operations: %w", err) - } - - zap.L().Debug("Indexed snapshots", - zap.String("plan", plan.Id), - zap.Duration("duration", time.Since(startTime)), - zap.Int("alreadyIndexed", alreadyIndexed), - zap.Int("newlyAdded", len(snapshots)-alreadyIndexed), - ) - - return err -} - -func containsSnapshotOperation(ops []*v1.Operation) bool { - for _, op := range ops { - if _, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok { - return true - } - } - return false -} diff --git a/internal/orchestrator/forget.go b/internal/orchestrator/forget.go index 20cd49d..2c59ecb 100644 --- a/internal/orchestrator/forget.go +++ b/internal/orchestrator/forget.go @@ -8,6 +8,7 @@ import ( "time" v1 "github.com/garethgeorge/resticui/gen/go/v1" + "go.uber.org/zap" ) // ForgetTask tracks a forget operation. @@ -15,6 +16,7 @@ type ForgetTask struct { name string orchestrator *Orchestrator // owning orchestrator plan *v1.Plan + linkSnapshot string // snapshot to link the task to. op *v1.Operation at *time.Time cancel atomic.Pointer[context.CancelFunc] // nil unless operation is running. @@ -22,11 +24,12 @@ type ForgetTask struct { var _ Task = &ForgetTask{} -func NewOneofForgetTask(orchestrator *Orchestrator, plan *v1.Plan, at time.Time) *ForgetTask { +func NewOneofForgetTask(orchestrator *Orchestrator, plan *v1.Plan, linkSnapshot string, at time.Time) *ForgetTask { return &ForgetTask{ orchestrator: orchestrator, plan: plan, at: &at, + linkSnapshot: linkSnapshot, } } @@ -41,6 +44,7 @@ func (t *ForgetTask) Next(now time.Time) *time.Time { t.op = &v1.Operation{ PlanId: t.plan.Id, RepoId: t.plan.Repo, + SnapshotId: t.linkSnapshot, UnixTimeStartMs: timeToUnixMillis(*ret), Status: v1.OperationStatus_STATUS_PENDING, Op: &v1.Operation_OperationForget{}, @@ -65,8 +69,10 @@ func (t *ForgetTask) Run(ctx context.Context) error { t.op.Op = forgetOp t.op.UnixTimeStartMs = curTimeMillis() + var repo *RepoOrchestrator if err := WithOperation(t.orchestrator.OpLog, t.op, func() error { - repo, err := t.orchestrator.GetRepo(t.plan.Repo) + var err error + repo, err = t.orchestrator.GetRepo(t.plan.Repo) if err != nil { return fmt.Errorf("get repo %q: %w", t.plan.Repo, err) } @@ -83,8 +89,9 @@ func (t *ForgetTask) Run(ctx context.Context) error { return err } - if t.plan.Retention.Prune { + if repo.repoConfig.PrunePolicy != nil { // TODO: schedule a prune task. + zap.S().Warn("repo specified a prune policy, automatic pruning is not yet implemented.") } return nil diff --git a/internal/orchestrator/indexsnapshots.go b/internal/orchestrator/indexsnapshots.go new file mode 100644 index 0000000..8cdcea7 --- /dev/null +++ b/internal/orchestrator/indexsnapshots.go @@ -0,0 +1,126 @@ +package orchestrator + +import ( + "context" + "fmt" + "time" + + v1 "github.com/garethgeorge/resticui/gen/go/v1" + "github.com/garethgeorge/resticui/internal/oplog" + "github.com/garethgeorge/resticui/internal/oplog/indexutil" + "github.com/garethgeorge/resticui/internal/protoutil" + "go.uber.org/zap" +) + +// indexSnapshotsHelper indexes all snapshots for a plan. +// - If the snapshot is already indexed, it is skipped. +// - If the snapshot is not indexed, an index snapshot operation with it's metadata is added. +// - If an index snapshot operation is found for a snapshot that is not returned by the repo, it is marked as forgotten. +func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan) error { + repo, err := orchestrator.GetRepo(plan.Repo) + if err != nil { + return fmt.Errorf("couldn't get repo %q: %w", plan.Repo, err) + } + + // collect all tracked snapshots for the plan. + snapshots, err := repo.SnapshotsForPlan(ctx, plan) + if err != nil { + return fmt.Errorf("get snapshots for plan %q: %w", plan.Id, err) + } + + // collect all current snapshot IDs. + currentIds, err := indexCurrentSnapshotIdsForPlan(orchestrator.OpLog, plan.Id) + if err != nil { + return fmt.Errorf("get known snapshot IDs for plan %q: %w", plan.Id, err) + } + + foundIds := make(map[string]bool) + + // Index newly found operations + startTime := time.Now() + var indexOps []*v1.Operation + for _, snapshot := range snapshots { + if _, ok := currentIds[snapshot.Id]; ok { + foundIds[snapshot.Id] = true + continue + } + + snapshotProto := protoutil.SnapshotToProto(snapshot) + indexOps = append(indexOps, &v1.Operation{ + RepoId: plan.Repo, + PlanId: plan.Id, + UnixTimeStartMs: snapshotProto.UnixTimeMs, + UnixTimeEndMs: snapshotProto.UnixTimeMs, + Status: v1.OperationStatus_STATUS_SUCCESS, + SnapshotId: snapshotProto.Id, + Op: &v1.Operation_OperationIndexSnapshot{ + OperationIndexSnapshot: &v1.OperationIndexSnapshot{ + Snapshot: snapshotProto, + }, + }, + }) + } + + if err := orchestrator.OpLog.BulkAdd(indexOps); err != nil { + return fmt.Errorf("BulkAdd snapshot operations: %w", err) + } + + // Mark missing operations as newly forgotten. + var forgetIds []int64 + for id, opId := range currentIds { + if _, ok := foundIds[id]; !ok { + forgetIds = append(forgetIds, opId) + } + } + + for _, opId := range forgetIds { + op, err := orchestrator.OpLog.Get(opId) + if err != nil { + // should only be possible in the case of a data race (e.g. operation was somehow deleted). + return fmt.Errorf("get operation %v: %w", opId, err) + } + + snapshotOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot) + if !ok { + return fmt.Errorf("operation %v is not an index snapshot operation", opId) + } + snapshotOp.OperationIndexSnapshot.Forgot = true + + if err := orchestrator.OpLog.Update(op); err != nil { + return fmt.Errorf("mark index snapshot operation %v as forgotten: %w", opId, err) + } + } + + // Print stats at the end of indexing. + zap.L().Debug("Indexed snapshots", + zap.String("plan", plan.Id), + zap.Duration("duration", time.Since(startTime)), + zap.Int("alreadyIndexed", len(foundIds)), + zap.Int("newlyAdded", len(indexOps)), + zap.Int("markedForgotten", len(currentIds)-len(foundIds)), + ) + + return err +} + +// returns a map of current (e.g. not forgotten) snapshot IDs for the plan. +func indexCurrentSnapshotIdsForPlan(log *oplog.OpLog, planId string) (map[string]int64, error) { + knownIds := make(map[string]int64) + + startTime := time.Now() + if err := log.ForEachByPlan(planId, indexutil.CollectAll(), func(op *v1.Operation) error { + if snapshotOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok { + if snapshotOp.OperationIndexSnapshot == nil { + return fmt.Errorf("operation %q has nil OperationIndexSnapshot, this shouldn't be possible.", op.Id) + } + if !snapshotOp.OperationIndexSnapshot.Forgot { + knownIds[snapshotOp.OperationIndexSnapshot.Snapshot.Id] = op.Id + } + } + return nil + }); err != nil { + return nil, err + } + zap.S().Debugf("Indexed known (and not forgotten) snapshot IDs for plan %v in %v", planId, time.Since(startTime)) + return knownIds, nil +} diff --git a/internal/orchestrator/prune.go b/internal/orchestrator/prune.go new file mode 100644 index 0000000..4501191 --- /dev/null +++ b/internal/orchestrator/prune.go @@ -0,0 +1,31 @@ +package orchestrator + +import ( + "bytes" + "sync" +) + +func pruneHelper() { + // TODO: This is a stub. + +} + +// synchronizedBuffer is used for collecting prune command's output +type synchronizedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (w *synchronizedBuffer) Write(p []byte) (n int, err error) { + w.mu.Lock() + defer w.mu.Unlock() + + return w.buf.Write(p) +} + +func (w *synchronizedBuffer) String() string { + w.mu.Lock() + defer w.mu.Unlock() + + return w.buf.String() +} diff --git a/internal/orchestrator/repo.go b/internal/orchestrator/repo.go index dc82bcd..736a416 100644 --- a/internal/orchestrator/repo.go +++ b/internal/orchestrator/repo.go @@ -3,6 +3,7 @@ package orchestrator import ( "context" "fmt" + "io" "sort" "sync" "time" @@ -114,7 +115,7 @@ func (r *RepoOrchestrator) Forget(ctx context.Context, plan *v1.Plan) ([]*v1.Res l := zap.L().With(zap.String("repo", r.repoConfig.Id), zap.String("plan", plan.Id)) l.Debug("Forget snapshots", zap.Any("policy", policy)) - result, err := r.repo.Forget(ctx, protoutil.RetentionPolicyFromProto(plan.Retention)) + result, err := r.repo.Forget(ctx, protoutil.RetentionPolicyFromProto(plan.Retention), restic.WithFlags("--tag", tagForPlan(plan), "--group-by", "")) if err != nil { return nil, fmt.Errorf("get snapshots for repo %v: %w", r.repoConfig.Id, err) } @@ -128,6 +129,20 @@ func (r *RepoOrchestrator) Forget(ctx context.Context, plan *v1.Plan) ([]*v1.Res return forgotten, nil } +func (r *RepoOrchestrator) Prune(ctx context.Context, output io.Writer) error { + r.mu.Lock() + defer r.mu.Unlock() + + l := zap.L().With(zap.String("repo", r.repoConfig.Id)) + + l.Debug("Prune snapshots") + err := r.repo.Prune(ctx, output) + if err != nil { + return fmt.Errorf("prune snapshots for repo %v: %w", r.repoConfig.Id, err) + } + return nil +} + func tagForPlan(plan *v1.Plan) string { return fmt.Sprintf("plan:%s", plan.Id) } diff --git a/internal/orchestrator/repo_test.go b/internal/orchestrator/repo_test.go index 1bb8128..8bb108a 100644 --- a/internal/orchestrator/repo_test.go +++ b/internal/orchestrator/repo_test.go @@ -35,7 +35,7 @@ func TestBackup(t *testing.T) { summary, err := orchestrator.Backup(context.Background(), plan, nil) if err != nil { - t.Fatal(err) + t.Fatalf("backup error: %v", err) } if summary.SnapshotId == "" { diff --git a/pkg/restic/restic.go b/pkg/restic/restic.go index bc3d4c1..b841b34 100644 --- a/pkg/restic/restic.go +++ b/pkg/restic/restic.go @@ -15,6 +15,8 @@ import ( v1 "github.com/garethgeorge/resticui/gen/go/v1" ) +var errAlreadyInitialized = errors.New("repo already initialized") + type Repo struct { mu sync.Mutex cmd string @@ -64,9 +66,10 @@ func (r *Repo) init(ctx context.Context) error { cmd.Env = append(cmd.Env, r.buildEnv()...) if output, err := cmd.CombinedOutput(); err != nil { - if !strings.Contains(string(output), "config file already exists") { - return NewCmdError(cmd, output, err) + if strings.Contains(string(output), "config file already exists") || strings.Contains(string(output), "already initialized") { + return errAlreadyInitialized } + return NewCmdError(cmd, output, err) } r.initialized = true @@ -76,7 +79,10 @@ func (r *Repo) init(ctx context.Context) error { func (r *Repo) Init(ctx context.Context) error { r.mu.Lock() defer r.mu.Unlock() - return r.init(ctx) + if err := r.init(ctx); err != nil && !errors.Is(err, errAlreadyInitialized) { + return fmt.Errorf("init failed: %w", err) + } + return nil } func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupProgressEntry), opts ...BackupOption) (*BackupProgressEntry, error) { @@ -259,10 +265,6 @@ func (r *Repo) ListDirectory(ctx context.Context, snapshot string, path string, return nil, nil, errors.New("path must not be empty") } - if err := r.init(ctx); err != nil { - return nil, nil, fmt.Errorf("failed to initialize repo: %w", err) - } - opt := resolveOpts(opts) args := []string{"ls", "--json", snapshot, path} @@ -401,7 +403,7 @@ func WithEnv(env ...string) GenericOption { } } -var EnvToPropagate = []string{"PATH", "HOME", "XDG_CACHE_HOME"} +var EnvToPropagate = []string{"PATH", "HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME"} func WithPropagatedEnvVars(extras ...string) GenericOption { var extension []string diff --git a/proto/v1/config.proto b/proto/v1/config.proto index a1518e3..10cd82e 100644 --- a/proto/v1/config.proto +++ b/proto/v1/config.proto @@ -22,6 +22,7 @@ message Repo { string password = 3 [json_name="password"]; // plaintext password repeated string env = 4 [json_name="env"]; // extra environment variables to set for restic. repeated string flags = 5 [json_name="flags"]; // extra flags set on the restic command. + PrunePolicy prune_policy = 6 [json_name="prunePolicy"]; // policy for when to run prune. } message Plan { @@ -35,15 +36,22 @@ message Plan { message RetentionPolicy { // max_unused_limit is used to decide when forget should be run. - string max_unused_limit = 1; // e.g. a percentage i.e. 25% or a number of megabytes. + string max_unused_limit = 1 [json_name="maxUnusedLimit"]; // e.g. a percentage i.e. 25% or a number of megabytes. - int32 keep_last_n = 2; // keep the last n snapshots. - int32 keep_hourly = 3; // keep the last n hourly snapshots. - int32 keep_daily = 4; // keep the last n daily snapshots. - int32 keep_weekly = 5; // keep the last n weekly snapshots. - int32 keep_monthly = 6; // keep the last n monthly snapshots. - int32 keep_yearly = 7; // keep the last n yearly snapshots. - string keep_within_duration = 8; // keep snapshots within a duration e.g. 1y2m3d4h5m6s - - bool prune = 9; // prune snapshots after forget. + int32 keep_last_n = 2 [json_name="keepLastN"]; // keep the last n snapshots. + int32 keep_hourly = 3 [json_name="keepHourly"]; // keep the last n hourly snapshots. + int32 keep_daily = 4 [json_name="keepDaily"]; // keep the last n daily snapshots. + int32 keep_weekly = 5 [json_name="keepWeekly"]; // keep the last n weekly snapshots. + int32 keep_monthly = 6 [json_name="keepMonthly"]; // keep the last n monthly snapshots. + int32 keep_yearly = 7 [json_name="keepYearly"]; // keep the last n yearly snapshots. + string keep_within_duration = 8 [json_name="keepWithinDuration"]; // keep snapshots within a duration e.g. 1y2m3d4h5m6s } + +message PrunePolicy { + int32 max_frequency_days = 1; // max frequency of prune runs in days. If 0, prune will be run on every backup. + + oneof policy { + int32 max_unused_percent = 100; // max percentage of repo size that can be unused before prune is run. + int32 max_unused_bytes = 101; // max number of bytes that can be unused before prune is run. + } +} \ No newline at end of file diff --git a/proto/v1/operations.proto b/proto/v1/operations.proto index 2db26a4..e8d3729 100644 --- a/proto/v1/operations.proto +++ b/proto/v1/operations.proto @@ -64,7 +64,9 @@ message OperationBackup { // OperationIndexSnapshot tracks that a snapshot was detected by resticui. message OperationIndexSnapshot { - ResticSnapshot snapshot = 2; + ResticSnapshot snapshot = 2; // the snapshot that was indexed. + bool forgot = 3; // tracks whether this snapshot is forgotten yet. + int64 forgot_by_op = 4; // ID of a forget operation that removed this snapshot. } // OperationForget tracks a forget operation. diff --git a/resticui.go b/resticui.go index cd9664f..5a0adfa 100644 --- a/resticui.go +++ b/resticui.go @@ -3,11 +3,11 @@ package main import ( "context" "errors" - "flag" "net/http" "os" "os/signal" "path" + "strings" "sync" "syscall" @@ -18,12 +18,11 @@ import ( "github.com/garethgeorge/resticui/internal/orchestrator" "github.com/garethgeorge/resticui/internal/resticinstaller" "github.com/mattn/go-colorable" + "go.etcd.io/bbolt" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) -var flagAutoInstallRestic = flag.Bool("auto-install-restic", true, "Automatically install restic if $RESTICUI_") - func main() { ctx := context.Background() ctx, cancel := context.WithCancel(ctx) @@ -51,6 +50,9 @@ func main() { oplogFile := path.Join(config.DataDir(), "oplog.boltdb") oplog, err := oplog.NewOpLog(oplogFile) if err != nil { + if !errors.Is(err, bbolt.ErrTimeout) { + zap.S().Fatalf("Timeout while waiting to open database, is the database open elsewhere?") + } zap.S().Warnf("Operation log may be corrupted, if errors recur delete the file %q and restart. Your backups stored in your repos are safe.", oplogFile) zap.S().Fatalf("Error creating oplog : %v", err) } @@ -95,7 +97,7 @@ func main() { wg.Add(1) go func() { defer wg.Done() - zap.S().Infof("HTTP binding to address %v", server.Addr) + zap.S().Infof("Starting web server %v", server.Addr) go func() { <-ctx.Done() server.Shutdown(context.Background()) @@ -112,7 +114,7 @@ func main() { func init() { zap.ReplaceGlobals(zap.Must(zap.NewProduction())) - if os.Getenv("DEBUG") != "" { + if !strings.HasPrefix(os.Getenv("ENV"), "prod") { c := zap.NewDevelopmentEncoderConfig() c.EncodeLevel = zapcore.CapitalColorLevelEncoder c.EncodeTime = zapcore.ISO8601TimeEncoder diff --git a/scripts/build-all.sh b/scripts/build-all.sh new file mode 100755 index 0000000..d2ef14e --- /dev/null +++ b/scripts/build-all.sh @@ -0,0 +1,18 @@ +#! /bin/sh + +(cd webui && npm i && npm run build) + +for bin in resticui-*; do + rm -f $bin +done + +find webui/dist -name '*.map' -exec rm ./{} \; + +GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o resticui-linux-amd64 +GOOS=linux GOARCH=arm64 go build -ldflags="-s -w" -o resticui-linux-arm64 +GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -o resticui-darwin-amd64 +GOOS=darwin GOARCH=arm64 go build -ldflags="-s -w" -o resticui-darwin-arm64 + +for bin in resticui-*; do + rice append --exec $bin +done diff --git a/hack/build.sh b/scripts/build.sh similarity index 100% rename from hack/build.sh rename to scripts/build.sh diff --git a/hack/install-deps.sh b/scripts/install-deps.sh similarity index 100% rename from hack/install-deps.sh rename to scripts/install-deps.sh diff --git a/hack/run.sh b/scripts/run.sh similarity index 100% rename from hack/run.sh rename to scripts/run.sh diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..ffa1a35 --- /dev/null +++ b/test.txt @@ -0,0 +1,18 @@ +loading indexes... +loading all snapshots... +finding data that is still in use for 2 snapshots +[0:00] 100.00% 2 / 2 snapshots + +searching used packs... +collecting packs for deletion and repacking +[0:00] 100.00% 219 / 219 packs processed + + +to repack: 0 blobs / 0 B +this removes 0 blobs / 0 B +to delete: 0 blobs / 0 B +total prune: 0 blobs / 0 B +remaining: 22570 blobs / 999.479 MiB +unused size after prune: 0 B (0.00% of remaining size) + +done diff --git a/webui/gen/ts/v1/config.pb.ts b/webui/gen/ts/v1/config.pb.ts index 94f4482..98b2379 100644 --- a/webui/gen/ts/v1/config.pb.ts +++ b/webui/gen/ts/v1/config.pb.ts @@ -3,6 +3,15 @@ /* * This file is a generated Typescript file for GRPC Gateway, DO NOT MODIFY */ + +type Absent = { [k in Exclude]?: undefined }; +type OneOf = + | { [k in keyof T]?: undefined } + | ( + keyof T extends infer K ? + (K extends string & keyof T ? { [k in K]: T[K] } & Absent + : never) + : never); export type Config = { modno?: number host?: string @@ -16,6 +25,7 @@ export type Repo = { password?: string env?: string[] flags?: string[] + prunePolicy?: PrunePolicy } export type Plan = { @@ -36,5 +46,12 @@ export type RetentionPolicy = { keepMonthly?: number keepYearly?: number keepWithinDuration?: string - prune?: boolean -} \ No newline at end of file +} + + +type BasePrunePolicy = { + maxFrequencyDays?: number +} + +export type PrunePolicy = BasePrunePolicy + & OneOf<{ maxUnusedPercent: number; maxUnusedBytes: number }> \ No newline at end of file diff --git a/webui/gen/ts/v1/operations.pb.ts b/webui/gen/ts/v1/operations.pb.ts index 937d0e9..c82f397 100644 --- a/webui/gen/ts/v1/operations.pb.ts +++ b/webui/gen/ts/v1/operations.pb.ts @@ -61,6 +61,8 @@ export type OperationBackup = { export type OperationIndexSnapshot = { snapshot?: V1Restic.ResticSnapshot + forgot?: boolean + forgotByOp?: string } export type OperationForget = { diff --git a/webui/src/components/OperationList.tsx b/webui/src/components/OperationList.tsx index 2f2820c..42ac9f9 100644 --- a/webui/src/components/OperationList.tsx +++ b/webui/src/components/OperationList.tsx @@ -37,6 +37,7 @@ export const OperationList = ({ req, }: React.PropsWithoutRef<{ req: GetOperationsRequest }>) => { const [operations, setOperations] = useState([]); + console.log("operation list with req: ", req); useEffect(() => { const lis = buildOperationListListener(req, (event, operation, opList) => { diff --git a/webui/src/components/OperationTree.tsx b/webui/src/components/OperationTree.tsx index 62dcdb4..95d152c 100644 --- a/webui/src/components/OperationTree.tsx +++ b/webui/src/components/OperationTree.tsx @@ -2,11 +2,9 @@ import React, { useEffect, useState } from "react"; import { BackupInfo, BackupInfoCollector, - EOperation, getOperations, shouldHideStatus, subscribeToOperations, - toEop, unsubscribeFromOperations, } from "../state/oplog"; import { Col, Divider, Empty, Row, Tree } from "antd"; @@ -36,7 +34,6 @@ type OpTreeNode = DataNode & { export const OperationTree = ({ req, }: React.PropsWithoutRef<{ req: GetOperationsRequest }>) => { - console.log("Loading operation tree with req: ", req); const alertApi = useAlertApi(); const [backups, setBackups] = useState([]); const [selectedBackupId, setSelectedBackupId] = useState(null); @@ -132,8 +129,14 @@ export const OperationTree = ({ const b = node.backup; const details: string[] = []; + if (b.operations.length === 1) { + if (b.operations[0].operationForget) { + return <>Forget {formatTime(b.displayTime)}; + } + } + if (b.status === OperationStatus.STATUS_PENDING) { - details.push("pending"); + details.push("scheduled, waiting"); } else if (b.status === OperationStatus.STATUS_SYSTEM_CANCELLED) { details.push("system cancel"); } else if (b.status === OperationStatus.STATUS_USER_CANCELLED) { diff --git a/webui/src/state/oplog.ts b/webui/src/state/oplog.ts index 98c649b..35f6255 100644 --- a/webui/src/state/oplog.ts +++ b/webui/src/state/oplog.ts @@ -129,9 +129,27 @@ export const toEop = (op: Operation): EOperation => { }; }; +enum DisplayType { + BACKUP, + SNAPSHOT, + FORGET, + PRUNE, +} + +const getTypeForDisplay = (op: Operation) => { + if (op.operationForget) { + return DisplayType.FORGET; + } else if (op.operationPrune) { + return DisplayType.PRUNE; + } else { + return DisplayType.BACKUP; + } +}; + export interface BackupInfo { id: string; // id of the first operation that generated this backup. displayTime: Date; + displayType: DisplayType; startTimeMs: number; endTimeMs: number; status: OperationStatus; @@ -157,6 +175,7 @@ export class BackupInfoCollector { existing.startTimeMs = Math.min(existing.startTimeMs, newInfo.startTimeMs); existing.endTimeMs = Math.max(existing.endTimeMs, newInfo.endTimeMs); existing.displayTime = new Date(existing.startTimeMs); + existing.displayType = DisplayType.SNAPSHOT; if (newInfo.startTimeMs >= existing.startTimeMs) { existing.status = newInfo.status; // use the latest status } @@ -183,6 +202,7 @@ export class BackupInfoCollector { endTimeMs, status: op.status!, displayTime: new Date(startTimeMs), + displayType: getTypeForDisplay(op), repoId: op.repoId, planId: op.planId, snapshotId: op.snapshotId, @@ -241,7 +261,16 @@ export class BackupInfoCollector { public getAll(): BackupInfo[] { const arr = []; arr.push(...Object.values(this.backupByOpId)); - arr.push(...Object.values(this.backupBySnapshotId)); + arr.push( + ...Object.values(this.backupBySnapshotId).filter((b) => { + for (const op of b.operations) { + if (op.operationIndexSnapshot && op.operationIndexSnapshot.forgot) { + return false; + } + } + return true; + }) + ); return arr; } diff --git a/webui/src/views/AddPlanModel.tsx b/webui/src/views/AddPlanModal.tsx similarity index 67% rename from webui/src/views/AddPlanModel.tsx rename to webui/src/views/AddPlanModal.tsx index 8070fbf..076e7e2 100644 --- a/webui/src/views/AddPlanModel.tsx +++ b/webui/src/views/AddPlanModal.tsx @@ -5,19 +5,22 @@ import { Typography, Select, Button, - Divider, Tooltip, + Radio, + InputNumber, + Row, + Card, + Col, } from "antd"; import React, { useState } from "react"; import { useShowModal } from "../components/ModalManager"; -import { Plan } from "../../gen/ts/v1/config.pb"; +import { Plan, RetentionPolicy } from "../../gen/ts/v1/config.pb"; import { useRecoilState } from "recoil"; import { configState, fetchConfig, updateConfig } from "../state/config"; import { nameRegex } from "../lib/patterns"; import { MinusCircleOutlined, PlusOutlined } from "@ant-design/icons"; import { URIAutocomplete } from "../components/URIAutocomplete"; import { useAlertApi } from "../components/Alerts"; -import { ResticUI } from "../../gen/ts/v1/service.pb"; import { Cron } from "react-js-cron"; import { validateForm } from "../lib/formutil"; @@ -65,7 +68,8 @@ export const AddPlanModal = ({ showModal(null); alertsApi.success( - "Plan deleted from config, but not from restic repo. Snapshots will remain in storage and operations will be tracked until manually deleted. Reusing a deleted plan ID is not recommended if backups have already been performed." + "Plan deleted from config, but not from restic repo. Snapshots will remain in storage and operations will be tracked until manually deleted. Reusing a deleted plan ID is not recommended if backups have already been performed.", + 30 ); } catch (e: any) { alertsApi.error("Operation failed: " + e.message, 15); @@ -116,6 +120,7 @@ export const AddPlanModal = ({ open={true} onCancel={handleCancel} title={template ? "Update Plan" : "Add Plan"} + width="40vw" footer={[ , ]} > -
+ {/* Plan.id */} hasFeedback @@ -207,9 +217,10 @@ export const AddPlanModal = ({ <> {fields.map((field, index) => ( form.validateFields()} /> @@ -234,7 +245,10 @@ export const AddPlanModal = ({ /> ))} - + - + + + + + +