From 0cf01e020640b0145bcd0d25a38cde1fce940aff Mon Sep 17 00:00:00 2001 From: garethgeorge Date: Fri, 2 Feb 2024 02:51:39 -0800 Subject: [PATCH] fix: store large log outputs in tar bundles of logs --- backrest.go | 10 +- gen/go/v1/operations.pb.go | 60 +++--- gen/go/v1/service.pb.go | 210 ++++++++++---------- gen/go/v1/service_grpc.pb.go | 62 +++--- gen/go/v1/v1connect/service.connect.go | 111 +++++------ internal/api/backresthandler.go | 11 +- internal/hook/hook.go | 33 ++- internal/oplog/bigopdatastore.go | 62 ------ internal/oplog/bigopdatastore_test.go | 35 ---- internal/oplog/oplog.go | 10 - internal/orchestrator/orchestrator.go | 16 +- internal/orchestrator/orchestrator_test.go | 8 +- internal/orchestrator/{tasks.go => task.go} | 0 internal/orchestrator/taskbackup.go | 6 +- internal/orchestrator/taskforget.go | 2 +- internal/orchestrator/taskindexsnapshots.go | 2 +- internal/orchestrator/taskprune.go | 2 +- internal/orchestrator/taskrestore.go | 2 +- internal/orchestrator/taskstats.go | 5 +- internal/rotatinglog/rotatinglog.go | 195 ++++++++++++++++++ internal/rotatinglog/rotatinglog_test.go | 61 ++++++ proto/v1/operations.proto | 2 +- proto/v1/service.proto | 7 +- webui/gen/ts/v1/operations_pb.ts | 8 +- webui/gen/ts/v1/service_connect.ts | 10 +- webui/gen/ts/v1/service_pb.ts | 40 ++-- webui/src/components/OperationList.tsx | 15 +- 27 files changed, 572 insertions(+), 413 deletions(-) delete mode 100644 internal/oplog/bigopdatastore.go delete mode 100644 internal/oplog/bigopdatastore_test.go rename internal/orchestrator/{tasks.go => task.go} (100%) create mode 100644 internal/rotatinglog/rotatinglog.go create mode 100644 internal/rotatinglog/rotatinglog_test.go diff --git a/backrest.go b/backrest.go index 79e5cd5..8df8b67 100644 --- a/backrest.go +++ b/backrest.go @@ -21,6 +21,7 @@ import ( "github.com/garethgeorge/backrest/internal/oplog" "github.com/garethgeorge/backrest/internal/orchestrator" "github.com/garethgeorge/backrest/internal/resticinstaller" + "github.com/garethgeorge/backrest/internal/rotatinglog" "github.com/mattn/go-colorable" "go.etcd.io/bbolt" "go.uber.org/zap" @@ -63,8 +64,14 @@ func main() { } defer oplog.Close() + // Create rotating log storage + logStore := rotatinglog.NewRotatingLog(path.Join(config.DataDir(), "rotatinglogs"), 30) + if err != nil { + zap.S().Fatalf("Error creating rotating log storage: %v", err) + } + // Create orchestrator and start task loop. - orchestrator, err := orchestrator.NewOrchestrator(resticPath, cfg, oplog) + orchestrator, err := orchestrator.NewOrchestrator(resticPath, cfg, oplog, logStore) if err != nil { zap.S().Fatalf("Error creating orchestrator: %v", err) } @@ -80,6 +87,7 @@ func main() { configStore, orchestrator, oplog, + logStore, ) apiAuthenticationHandler := api.NewAuthenticationHandler(authenticator) diff --git a/gen/go/v1/operations.pb.go b/gen/go/v1/operations.pb.go index a8c7981..4e44924 100644 --- a/gen/go/v1/operations.pb.go +++ b/gen/go/v1/operations.pb.go @@ -792,8 +792,8 @@ type OperationRunHook struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // description of the hook that was run. typically repo/hook_idx or plan/hook_idx. - OutputRef string `protobuf:"bytes,2,opt,name=output_ref,json=outputRef,proto3" json:"output_ref,omitempty"` // reference to the output of the hook. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // description of the hook that was run. typically repo/hook_idx or plan/hook_idx. + OutputLogref string `protobuf:"bytes,2,opt,name=output_logref,json=outputLogref,proto3" json:"output_logref,omitempty"` // logref of the hook's output. } func (x *OperationRunHook) Reset() { @@ -835,9 +835,9 @@ func (x *OperationRunHook) GetName() string { return "" } -func (x *OperationRunHook) GetOutputRef() string { +func (x *OperationRunHook) GetOutputLogref() string { if x != nil { - return x.OutputRef + return x.OutputLogref } return "" } @@ -941,33 +941,33 @@ var file_v1_operations_proto_rawDesc = []byte{ 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, - 0x45, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x48, + 0x4b, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x75, 0x74, 0x70, 0x75, - 0x74, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x52, 0x65, 0x66, 0x2a, 0x60, 0x0a, 0x12, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x11, 0x0a, 0x0d, - 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, - 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, - 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, - 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x44, - 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x03, 0x2a, 0xc2, 0x01, 0x0a, 0x0f, 0x4f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, - 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, - 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, - 0x4e, 0x47, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, - 0x4e, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x03, 0x12, - 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, - 0x47, 0x10, 0x07, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, - 0x52, 0x4f, 0x52, 0x10, 0x04, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, - 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x53, 0x45, - 0x52, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x06, 0x42, 0x2c, 0x5a, - 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, - 0x74, 0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, - 0x74, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x75, 0x74, 0x70, 0x75, + 0x74, 0x5f, 0x6c, 0x6f, 0x67, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x4c, 0x6f, 0x67, 0x72, 0x65, 0x66, 0x2a, 0x60, 0x0a, 0x12, + 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, + 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x43, + 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, + 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x45, + 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x03, 0x2a, 0xc2, + 0x01, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x4b, + 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x54, + 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, + 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x55, 0x43, 0x43, + 0x45, 0x53, 0x53, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, + 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x07, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x41, + 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x12, 0x1b, 0x0a, 0x17, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, 0x43, 0x41, 0x4e, + 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x41, 0x54, + 0x55, 0x53, 0x5f, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, + 0x44, 0x10, 0x06, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62, + 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76, + 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/gen/go/v1/service.pb.go b/gen/go/v1/service.pb.go index 1303b6e..66a68f5 100644 --- a/gen/go/v1/service.pb.go +++ b/gen/go/v1/service.pb.go @@ -409,17 +409,16 @@ func (x *ListSnapshotFilesResponse) GetEntries() []*LsEntry { return nil } -type OperationDataRequest struct { +type LogDataRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` // operation id - Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` // data key + Ref string `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` } -func (x *OperationDataRequest) Reset() { - *x = OperationDataRequest{} +func (x *LogDataRequest) Reset() { + *x = LogDataRequest{} if protoimpl.UnsafeEnabled { mi := &file_v1_service_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -427,13 +426,13 @@ func (x *OperationDataRequest) Reset() { } } -func (x *OperationDataRequest) String() string { +func (x *LogDataRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OperationDataRequest) ProtoMessage() {} +func (*LogDataRequest) ProtoMessage() {} -func (x *OperationDataRequest) ProtoReflect() protoreflect.Message { +func (x *LogDataRequest) ProtoReflect() protoreflect.Message { mi := &file_v1_service_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -445,21 +444,14 @@ func (x *OperationDataRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OperationDataRequest.ProtoReflect.Descriptor instead. -func (*OperationDataRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use LogDataRequest.ProtoReflect.Descriptor instead. +func (*LogDataRequest) Descriptor() ([]byte, []int) { return file_v1_service_proto_rawDescGZIP(), []int{6} } -func (x *OperationDataRequest) GetId() int64 { +func (x *LogDataRequest) GetRef() string { if x != nil { - return x.Id - } - return 0 -} - -func (x *OperationDataRequest) GetKey() string { - if x != nil { - return x.Key + return x.Ref } return "" } @@ -636,95 +628,93 @@ var file_v1_service_proto_rawDesc = []byte{ 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x25, 0x0a, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x65, - 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0x38, 0x0a, 0x14, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x22, 0xd3, 0x01, 0x0a, 0x07, 0x4c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, - 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x67, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, - 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, - 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x74, - 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x61, 0x74, 0x69, 0x6d, 0x65, - 0x12, 0x14, 0x0a, 0x05, 0x63, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x63, 0x74, 0x69, 0x6d, 0x65, 0x32, 0xba, 0x08, 0x0a, 0x08, 0x42, 0x61, 0x63, 0x6b, 0x72, - 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0x22, 0x0a, 0x0e, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, + 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0xd3, 0x01, 0x0a, 0x07, 0x4c, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x03, 0x67, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, + 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, + 0x74, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x61, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x74, + 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x63, 0x74, 0x69, 0x6d, 0x65, + 0x32, 0xa8, 0x08, 0x0a, 0x08, 0x42, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, + 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, + 0x12, 0x25, 0x0a, 0x09, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x0a, 0x2e, + 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, 0x12, 0x21, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x52, 0x65, + 0x70, 0x6f, 0x12, 0x08, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x1a, 0x0a, 0x2e, 0x76, + 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, 0x12, 0x44, 0x0a, 0x12, 0x47, 0x65, + 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, 0x12, 0x25, 0x0a, 0x09, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, - 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, 0x12, 0x21, 0x0a, - 0x07, 0x41, 0x64, 0x64, 0x52, 0x65, 0x70, 0x6f, 0x12, 0x08, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x70, 0x6f, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, - 0x12, 0x44, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x12, - 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x3e, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, - 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0d, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, - 0x70, 0x73, 0x68, 0x6f, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x11, 0x4c, - 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, - 0x12, 0x1c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, - 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, - 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x3e, 0x0a, 0x0e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x73, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, - 0x36, 0x0a, 0x06, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, - 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x00, 0x30, 0x01, + 0x12, 0x3e, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x76, 0x31, + 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, + 0x12, 0x43, 0x0a, 0x0d, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, + 0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, + 0x68, 0x6f, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x76, 0x31, + 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x4c, + 0x69, 0x73, 0x74, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, + 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x76, 0x31, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3e, 0x0a, 0x0e, 0x49, 0x6e, 0x64, + 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x12, 0x12, 0x2e, 0x74, 0x79, + 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, + 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, + 0x00, 0x12, 0x35, 0x0a, 0x05, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, + 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x46, 0x6f, 0x72, 0x67, + 0x65, 0x74, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, + 0x12, 0x3f, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1a, 0x2e, 0x76, 0x31, + 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, + 0x00, 0x12, 0x36, 0x0a, 0x06, 0x55, 0x6e, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x12, 0x2e, 0x74, 0x79, + 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x05, 0x53, 0x74, 0x61, + 0x74, 0x73, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, + 0x12, 0x35, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x11, 0x2e, 0x74, 0x79, 0x70, + 0x65, 0x73, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x05, 0x50, 0x72, 0x75, 0x6e, 0x65, - 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x36, - 0x0a, 0x06, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, - 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x12, 0x1a, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x55, 0x6e, 0x6c, 0x6f, 0x63, - 0x6b, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, - 0x35, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, - 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, - 0x12, 0x11, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x44, 0x0a, - 0x13, 0x47, 0x65, 0x74, 0x42, 0x69, 0x67, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x44, 0x61, 0x74, 0x61, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, - 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0c, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x48, 0x69, 0x73, 0x74, - 0x6f, 0x72, 0x79, 0x12, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x48, 0x69, - 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x10, 0x50, 0x61, 0x74, 0x68, 0x41, 0x75, - 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, - 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x11, - 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, - 0x74, 0x22, 0x00, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62, - 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76, - 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, + 0x67, 0x73, 0x12, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0c, 0x43, + 0x6c, 0x65, 0x61, 0x72, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x17, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3b, + 0x0a, 0x10, 0x50, 0x61, 0x74, 0x68, 0x41, 0x75, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, + 0x74, 0x65, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x11, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, + 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x42, 0x2c, 0x5a, 0x2a, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, + 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, + 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -747,7 +737,7 @@ var file_v1_service_proto_goTypes = []interface{}{ (*RestoreSnapshotRequest)(nil), // 3: v1.RestoreSnapshotRequest (*ListSnapshotFilesRequest)(nil), // 4: v1.ListSnapshotFilesRequest (*ListSnapshotFilesResponse)(nil), // 5: v1.ListSnapshotFilesResponse - (*OperationDataRequest)(nil), // 6: v1.OperationDataRequest + (*LogDataRequest)(nil), // 6: v1.LogDataRequest (*LsEntry)(nil), // 7: v1.LsEntry (*emptypb.Empty)(nil), // 8: google.protobuf.Empty (*Config)(nil), // 9: v1.Config @@ -777,7 +767,7 @@ var file_v1_service_proto_depIdxs = []int32{ 11, // 13: v1.Backrest.Unlock:input_type -> types.StringValue 11, // 14: v1.Backrest.Stats:input_type -> types.StringValue 12, // 15: v1.Backrest.Cancel:input_type -> types.Int64Value - 6, // 16: v1.Backrest.GetBigOperationData:input_type -> v1.OperationDataRequest + 6, // 16: v1.Backrest.GetLogs:input_type -> v1.LogDataRequest 0, // 17: v1.Backrest.ClearHistory:input_type -> v1.ClearHistoryRequest 11, // 18: v1.Backrest.PathAutocomplete:input_type -> types.StringValue 9, // 19: v1.Backrest.GetConfig:output_type -> v1.Config @@ -795,7 +785,7 @@ var file_v1_service_proto_depIdxs = []int32{ 8, // 31: v1.Backrest.Unlock:output_type -> google.protobuf.Empty 8, // 32: v1.Backrest.Stats:output_type -> google.protobuf.Empty 8, // 33: v1.Backrest.Cancel:output_type -> google.protobuf.Empty - 16, // 34: v1.Backrest.GetBigOperationData:output_type -> types.BytesValue + 16, // 34: v1.Backrest.GetLogs:output_type -> types.BytesValue 8, // 35: v1.Backrest.ClearHistory:output_type -> google.protobuf.Empty 17, // 36: v1.Backrest.PathAutocomplete:output_type -> types.StringList 19, // [19:37] is the sub-list for method output_type @@ -887,7 +877,7 @@ func file_v1_service_proto_init() { } } file_v1_service_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OperationDataRequest); i { + switch v := v.(*LogDataRequest); i { case 0: return &v.state case 1: diff --git a/gen/go/v1/service_grpc.pb.go b/gen/go/v1/service_grpc.pb.go index 311a0d8..08abf12 100644 --- a/gen/go/v1/service_grpc.pb.go +++ b/gen/go/v1/service_grpc.pb.go @@ -21,24 +21,24 @@ import ( const _ = grpc.SupportPackageIsVersion7 const ( - Backrest_GetConfig_FullMethodName = "/v1.Backrest/GetConfig" - Backrest_SetConfig_FullMethodName = "/v1.Backrest/SetConfig" - Backrest_AddRepo_FullMethodName = "/v1.Backrest/AddRepo" - Backrest_GetOperationEvents_FullMethodName = "/v1.Backrest/GetOperationEvents" - Backrest_GetOperations_FullMethodName = "/v1.Backrest/GetOperations" - Backrest_ListSnapshots_FullMethodName = "/v1.Backrest/ListSnapshots" - Backrest_ListSnapshotFiles_FullMethodName = "/v1.Backrest/ListSnapshotFiles" - Backrest_IndexSnapshots_FullMethodName = "/v1.Backrest/IndexSnapshots" - Backrest_Backup_FullMethodName = "/v1.Backrest/Backup" - Backrest_Prune_FullMethodName = "/v1.Backrest/Prune" - Backrest_Forget_FullMethodName = "/v1.Backrest/Forget" - Backrest_Restore_FullMethodName = "/v1.Backrest/Restore" - Backrest_Unlock_FullMethodName = "/v1.Backrest/Unlock" - Backrest_Stats_FullMethodName = "/v1.Backrest/Stats" - Backrest_Cancel_FullMethodName = "/v1.Backrest/Cancel" - Backrest_GetBigOperationData_FullMethodName = "/v1.Backrest/GetBigOperationData" - Backrest_ClearHistory_FullMethodName = "/v1.Backrest/ClearHistory" - Backrest_PathAutocomplete_FullMethodName = "/v1.Backrest/PathAutocomplete" + Backrest_GetConfig_FullMethodName = "/v1.Backrest/GetConfig" + Backrest_SetConfig_FullMethodName = "/v1.Backrest/SetConfig" + Backrest_AddRepo_FullMethodName = "/v1.Backrest/AddRepo" + Backrest_GetOperationEvents_FullMethodName = "/v1.Backrest/GetOperationEvents" + Backrest_GetOperations_FullMethodName = "/v1.Backrest/GetOperations" + Backrest_ListSnapshots_FullMethodName = "/v1.Backrest/ListSnapshots" + Backrest_ListSnapshotFiles_FullMethodName = "/v1.Backrest/ListSnapshotFiles" + Backrest_IndexSnapshots_FullMethodName = "/v1.Backrest/IndexSnapshots" + Backrest_Backup_FullMethodName = "/v1.Backrest/Backup" + Backrest_Prune_FullMethodName = "/v1.Backrest/Prune" + Backrest_Forget_FullMethodName = "/v1.Backrest/Forget" + Backrest_Restore_FullMethodName = "/v1.Backrest/Restore" + Backrest_Unlock_FullMethodName = "/v1.Backrest/Unlock" + Backrest_Stats_FullMethodName = "/v1.Backrest/Stats" + Backrest_Cancel_FullMethodName = "/v1.Backrest/Cancel" + Backrest_GetLogs_FullMethodName = "/v1.Backrest/GetLogs" + Backrest_ClearHistory_FullMethodName = "/v1.Backrest/ClearHistory" + Backrest_PathAutocomplete_FullMethodName = "/v1.Backrest/PathAutocomplete" ) // BackrestClient is the client API for Backrest service. @@ -69,7 +69,7 @@ type BackrestClient interface { // Cancel attempts to cancel a task with the given operation ID. Not guaranteed to succeed. Cancel(ctx context.Context, in *types.Int64Value, opts ...grpc.CallOption) (*emptypb.Empty, error) // GetBigOperationData returns the keyed large data for the given operation. - GetBigOperationData(ctx context.Context, in *OperationDataRequest, opts ...grpc.CallOption) (*types.BytesValue, error) + GetLogs(ctx context.Context, in *LogDataRequest, opts ...grpc.CallOption) (*types.BytesValue, error) // Clears the history of operations ClearHistory(ctx context.Context, in *ClearHistoryRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) // PathAutocomplete provides path autocompletion options for a given filesystem path. @@ -242,9 +242,9 @@ func (c *backrestClient) Cancel(ctx context.Context, in *types.Int64Value, opts return out, nil } -func (c *backrestClient) GetBigOperationData(ctx context.Context, in *OperationDataRequest, opts ...grpc.CallOption) (*types.BytesValue, error) { +func (c *backrestClient) GetLogs(ctx context.Context, in *LogDataRequest, opts ...grpc.CallOption) (*types.BytesValue, error) { out := new(types.BytesValue) - err := c.cc.Invoke(ctx, Backrest_GetBigOperationData_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, Backrest_GetLogs_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -297,7 +297,7 @@ type BackrestServer interface { // Cancel attempts to cancel a task with the given operation ID. Not guaranteed to succeed. Cancel(context.Context, *types.Int64Value) (*emptypb.Empty, error) // GetBigOperationData returns the keyed large data for the given operation. - GetBigOperationData(context.Context, *OperationDataRequest) (*types.BytesValue, error) + GetLogs(context.Context, *LogDataRequest) (*types.BytesValue, error) // Clears the history of operations ClearHistory(context.Context, *ClearHistoryRequest) (*emptypb.Empty, error) // PathAutocomplete provides path autocompletion options for a given filesystem path. @@ -354,8 +354,8 @@ func (UnimplementedBackrestServer) Stats(context.Context, *types.StringValue) (* func (UnimplementedBackrestServer) Cancel(context.Context, *types.Int64Value) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Cancel not implemented") } -func (UnimplementedBackrestServer) GetBigOperationData(context.Context, *OperationDataRequest) (*types.BytesValue, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetBigOperationData not implemented") +func (UnimplementedBackrestServer) GetLogs(context.Context, *LogDataRequest) (*types.BytesValue, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetLogs not implemented") } func (UnimplementedBackrestServer) ClearHistory(context.Context, *ClearHistoryRequest) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method ClearHistory not implemented") @@ -649,20 +649,20 @@ func _Backrest_Cancel_Handler(srv interface{}, ctx context.Context, dec func(int return interceptor(ctx, in, info, handler) } -func _Backrest_GetBigOperationData_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(OperationDataRequest) +func _Backrest_GetLogs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LogDataRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(BackrestServer).GetBigOperationData(ctx, in) + return srv.(BackrestServer).GetLogs(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: Backrest_GetBigOperationData_FullMethodName, + FullMethod: Backrest_GetLogs_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(BackrestServer).GetBigOperationData(ctx, req.(*OperationDataRequest)) + return srv.(BackrestServer).GetLogs(ctx, req.(*LogDataRequest)) } return interceptor(ctx, in, info, handler) } @@ -767,8 +767,8 @@ var Backrest_ServiceDesc = grpc.ServiceDesc{ Handler: _Backrest_Cancel_Handler, }, { - MethodName: "GetBigOperationData", - Handler: _Backrest_GetBigOperationData_Handler, + MethodName: "GetLogs", + Handler: _Backrest_GetLogs_Handler, }, { MethodName: "ClearHistory", diff --git a/gen/go/v1/v1connect/service.connect.go b/gen/go/v1/v1connect/service.connect.go index f18d457..5158de4 100644 --- a/gen/go/v1/v1connect/service.connect.go +++ b/gen/go/v1/v1connect/service.connect.go @@ -67,9 +67,8 @@ const ( BackrestStatsProcedure = "/v1.Backrest/Stats" // BackrestCancelProcedure is the fully-qualified name of the Backrest's Cancel RPC. BackrestCancelProcedure = "/v1.Backrest/Cancel" - // BackrestGetBigOperationDataProcedure is the fully-qualified name of the Backrest's - // GetBigOperationData RPC. - BackrestGetBigOperationDataProcedure = "/v1.Backrest/GetBigOperationData" + // BackrestGetLogsProcedure is the fully-qualified name of the Backrest's GetLogs RPC. + BackrestGetLogsProcedure = "/v1.Backrest/GetLogs" // BackrestClearHistoryProcedure is the fully-qualified name of the Backrest's ClearHistory RPC. BackrestClearHistoryProcedure = "/v1.Backrest/ClearHistory" // BackrestPathAutocompleteProcedure is the fully-qualified name of the Backrest's PathAutocomplete @@ -79,25 +78,25 @@ const ( // These variables are the protoreflect.Descriptor objects for the RPCs defined in this package. var ( - backrestServiceDescriptor = v1.File_v1_service_proto.Services().ByName("Backrest") - backrestGetConfigMethodDescriptor = backrestServiceDescriptor.Methods().ByName("GetConfig") - backrestSetConfigMethodDescriptor = backrestServiceDescriptor.Methods().ByName("SetConfig") - backrestAddRepoMethodDescriptor = backrestServiceDescriptor.Methods().ByName("AddRepo") - backrestGetOperationEventsMethodDescriptor = backrestServiceDescriptor.Methods().ByName("GetOperationEvents") - backrestGetOperationsMethodDescriptor = backrestServiceDescriptor.Methods().ByName("GetOperations") - backrestListSnapshotsMethodDescriptor = backrestServiceDescriptor.Methods().ByName("ListSnapshots") - backrestListSnapshotFilesMethodDescriptor = backrestServiceDescriptor.Methods().ByName("ListSnapshotFiles") - backrestIndexSnapshotsMethodDescriptor = backrestServiceDescriptor.Methods().ByName("IndexSnapshots") - backrestBackupMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Backup") - backrestPruneMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Prune") - backrestForgetMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Forget") - backrestRestoreMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Restore") - backrestUnlockMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Unlock") - backrestStatsMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Stats") - backrestCancelMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Cancel") - backrestGetBigOperationDataMethodDescriptor = backrestServiceDescriptor.Methods().ByName("GetBigOperationData") - backrestClearHistoryMethodDescriptor = backrestServiceDescriptor.Methods().ByName("ClearHistory") - backrestPathAutocompleteMethodDescriptor = backrestServiceDescriptor.Methods().ByName("PathAutocomplete") + backrestServiceDescriptor = v1.File_v1_service_proto.Services().ByName("Backrest") + backrestGetConfigMethodDescriptor = backrestServiceDescriptor.Methods().ByName("GetConfig") + backrestSetConfigMethodDescriptor = backrestServiceDescriptor.Methods().ByName("SetConfig") + backrestAddRepoMethodDescriptor = backrestServiceDescriptor.Methods().ByName("AddRepo") + backrestGetOperationEventsMethodDescriptor = backrestServiceDescriptor.Methods().ByName("GetOperationEvents") + backrestGetOperationsMethodDescriptor = backrestServiceDescriptor.Methods().ByName("GetOperations") + backrestListSnapshotsMethodDescriptor = backrestServiceDescriptor.Methods().ByName("ListSnapshots") + backrestListSnapshotFilesMethodDescriptor = backrestServiceDescriptor.Methods().ByName("ListSnapshotFiles") + backrestIndexSnapshotsMethodDescriptor = backrestServiceDescriptor.Methods().ByName("IndexSnapshots") + backrestBackupMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Backup") + backrestPruneMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Prune") + backrestForgetMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Forget") + backrestRestoreMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Restore") + backrestUnlockMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Unlock") + backrestStatsMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Stats") + backrestCancelMethodDescriptor = backrestServiceDescriptor.Methods().ByName("Cancel") + backrestGetLogsMethodDescriptor = backrestServiceDescriptor.Methods().ByName("GetLogs") + backrestClearHistoryMethodDescriptor = backrestServiceDescriptor.Methods().ByName("ClearHistory") + backrestPathAutocompleteMethodDescriptor = backrestServiceDescriptor.Methods().ByName("PathAutocomplete") ) // BackrestClient is a client for the v1.Backrest service. @@ -126,7 +125,7 @@ type BackrestClient interface { // Cancel attempts to cancel a task with the given operation ID. Not guaranteed to succeed. Cancel(context.Context, *connect.Request[types.Int64Value]) (*connect.Response[emptypb.Empty], error) // GetBigOperationData returns the keyed large data for the given operation. - GetBigOperationData(context.Context, *connect.Request[v1.OperationDataRequest]) (*connect.Response[types.BytesValue], error) + GetLogs(context.Context, *connect.Request[v1.LogDataRequest]) (*connect.Response[types.BytesValue], error) // Clears the history of operations ClearHistory(context.Context, *connect.Request[v1.ClearHistoryRequest]) (*connect.Response[emptypb.Empty], error) // PathAutocomplete provides path autocompletion options for a given filesystem path. @@ -233,10 +232,10 @@ func NewBackrestClient(httpClient connect.HTTPClient, baseURL string, opts ...co connect.WithSchema(backrestCancelMethodDescriptor), connect.WithClientOptions(opts...), ), - getBigOperationData: connect.NewClient[v1.OperationDataRequest, types.BytesValue]( + getLogs: connect.NewClient[v1.LogDataRequest, types.BytesValue]( httpClient, - baseURL+BackrestGetBigOperationDataProcedure, - connect.WithSchema(backrestGetBigOperationDataMethodDescriptor), + baseURL+BackrestGetLogsProcedure, + connect.WithSchema(backrestGetLogsMethodDescriptor), connect.WithClientOptions(opts...), ), clearHistory: connect.NewClient[v1.ClearHistoryRequest, emptypb.Empty]( @@ -256,24 +255,24 @@ func NewBackrestClient(httpClient connect.HTTPClient, baseURL string, opts ...co // backrestClient implements BackrestClient. type backrestClient struct { - getConfig *connect.Client[emptypb.Empty, v1.Config] - setConfig *connect.Client[v1.Config, v1.Config] - addRepo *connect.Client[v1.Repo, v1.Config] - getOperationEvents *connect.Client[emptypb.Empty, v1.OperationEvent] - getOperations *connect.Client[v1.GetOperationsRequest, v1.OperationList] - listSnapshots *connect.Client[v1.ListSnapshotsRequest, v1.ResticSnapshotList] - listSnapshotFiles *connect.Client[v1.ListSnapshotFilesRequest, v1.ListSnapshotFilesResponse] - indexSnapshots *connect.Client[types.StringValue, emptypb.Empty] - backup *connect.Client[types.StringValue, emptypb.Empty] - prune *connect.Client[types.StringValue, emptypb.Empty] - forget *connect.Client[types.StringValue, emptypb.Empty] - restore *connect.Client[v1.RestoreSnapshotRequest, emptypb.Empty] - unlock *connect.Client[types.StringValue, emptypb.Empty] - stats *connect.Client[types.StringValue, emptypb.Empty] - cancel *connect.Client[types.Int64Value, emptypb.Empty] - getBigOperationData *connect.Client[v1.OperationDataRequest, types.BytesValue] - clearHistory *connect.Client[v1.ClearHistoryRequest, emptypb.Empty] - pathAutocomplete *connect.Client[types.StringValue, types.StringList] + getConfig *connect.Client[emptypb.Empty, v1.Config] + setConfig *connect.Client[v1.Config, v1.Config] + addRepo *connect.Client[v1.Repo, v1.Config] + getOperationEvents *connect.Client[emptypb.Empty, v1.OperationEvent] + getOperations *connect.Client[v1.GetOperationsRequest, v1.OperationList] + listSnapshots *connect.Client[v1.ListSnapshotsRequest, v1.ResticSnapshotList] + listSnapshotFiles *connect.Client[v1.ListSnapshotFilesRequest, v1.ListSnapshotFilesResponse] + indexSnapshots *connect.Client[types.StringValue, emptypb.Empty] + backup *connect.Client[types.StringValue, emptypb.Empty] + prune *connect.Client[types.StringValue, emptypb.Empty] + forget *connect.Client[types.StringValue, emptypb.Empty] + restore *connect.Client[v1.RestoreSnapshotRequest, emptypb.Empty] + unlock *connect.Client[types.StringValue, emptypb.Empty] + stats *connect.Client[types.StringValue, emptypb.Empty] + cancel *connect.Client[types.Int64Value, emptypb.Empty] + getLogs *connect.Client[v1.LogDataRequest, types.BytesValue] + clearHistory *connect.Client[v1.ClearHistoryRequest, emptypb.Empty] + pathAutocomplete *connect.Client[types.StringValue, types.StringList] } // GetConfig calls v1.Backrest.GetConfig. @@ -351,9 +350,9 @@ func (c *backrestClient) Cancel(ctx context.Context, req *connect.Request[types. return c.cancel.CallUnary(ctx, req) } -// GetBigOperationData calls v1.Backrest.GetBigOperationData. -func (c *backrestClient) GetBigOperationData(ctx context.Context, req *connect.Request[v1.OperationDataRequest]) (*connect.Response[types.BytesValue], error) { - return c.getBigOperationData.CallUnary(ctx, req) +// GetLogs calls v1.Backrest.GetLogs. +func (c *backrestClient) GetLogs(ctx context.Context, req *connect.Request[v1.LogDataRequest]) (*connect.Response[types.BytesValue], error) { + return c.getLogs.CallUnary(ctx, req) } // ClearHistory calls v1.Backrest.ClearHistory. @@ -392,7 +391,7 @@ type BackrestHandler interface { // Cancel attempts to cancel a task with the given operation ID. Not guaranteed to succeed. Cancel(context.Context, *connect.Request[types.Int64Value]) (*connect.Response[emptypb.Empty], error) // GetBigOperationData returns the keyed large data for the given operation. - GetBigOperationData(context.Context, *connect.Request[v1.OperationDataRequest]) (*connect.Response[types.BytesValue], error) + GetLogs(context.Context, *connect.Request[v1.LogDataRequest]) (*connect.Response[types.BytesValue], error) // Clears the history of operations ClearHistory(context.Context, *connect.Request[v1.ClearHistoryRequest]) (*connect.Response[emptypb.Empty], error) // PathAutocomplete provides path autocompletion options for a given filesystem path. @@ -495,10 +494,10 @@ func NewBackrestHandler(svc BackrestHandler, opts ...connect.HandlerOption) (str connect.WithSchema(backrestCancelMethodDescriptor), connect.WithHandlerOptions(opts...), ) - backrestGetBigOperationDataHandler := connect.NewUnaryHandler( - BackrestGetBigOperationDataProcedure, - svc.GetBigOperationData, - connect.WithSchema(backrestGetBigOperationDataMethodDescriptor), + backrestGetLogsHandler := connect.NewUnaryHandler( + BackrestGetLogsProcedure, + svc.GetLogs, + connect.WithSchema(backrestGetLogsMethodDescriptor), connect.WithHandlerOptions(opts...), ) backrestClearHistoryHandler := connect.NewUnaryHandler( @@ -545,8 +544,8 @@ func NewBackrestHandler(svc BackrestHandler, opts ...connect.HandlerOption) (str backrestStatsHandler.ServeHTTP(w, r) case BackrestCancelProcedure: backrestCancelHandler.ServeHTTP(w, r) - case BackrestGetBigOperationDataProcedure: - backrestGetBigOperationDataHandler.ServeHTTP(w, r) + case BackrestGetLogsProcedure: + backrestGetLogsHandler.ServeHTTP(w, r) case BackrestClearHistoryProcedure: backrestClearHistoryHandler.ServeHTTP(w, r) case BackrestPathAutocompleteProcedure: @@ -620,8 +619,8 @@ func (UnimplementedBackrestHandler) Cancel(context.Context, *connect.Request[typ return nil, connect.NewError(connect.CodeUnimplemented, errors.New("v1.Backrest.Cancel is not implemented")) } -func (UnimplementedBackrestHandler) GetBigOperationData(context.Context, *connect.Request[v1.OperationDataRequest]) (*connect.Response[types.BytesValue], error) { - return nil, connect.NewError(connect.CodeUnimplemented, errors.New("v1.Backrest.GetBigOperationData is not implemented")) +func (UnimplementedBackrestHandler) GetLogs(context.Context, *connect.Request[v1.LogDataRequest]) (*connect.Response[types.BytesValue], error) { + return nil, connect.NewError(connect.CodeUnimplemented, errors.New("v1.Backrest.GetLogs is not implemented")) } func (UnimplementedBackrestHandler) ClearHistory(context.Context, *connect.Request[v1.ClearHistoryRequest]) (*connect.Response[emptypb.Empty], error) { diff --git a/internal/api/backresthandler.go b/internal/api/backresthandler.go index dfc8f13..88ce968 100644 --- a/internal/api/backresthandler.go +++ b/internal/api/backresthandler.go @@ -19,6 +19,7 @@ import ( "github.com/garethgeorge/backrest/internal/orchestrator" "github.com/garethgeorge/backrest/internal/protoutil" "github.com/garethgeorge/backrest/internal/resticinstaller" + "github.com/garethgeorge/backrest/internal/rotatinglog" "github.com/garethgeorge/backrest/pkg/restic" "go.uber.org/zap" "google.golang.org/protobuf/proto" @@ -30,15 +31,17 @@ type BackrestHandler struct { config config.ConfigStore orchestrator *orchestrator.Orchestrator oplog *oplog.OpLog + logStore *rotatinglog.RotatingLog } var _ v1connect.BackrestHandler = &BackrestHandler{} -func NewBackrestHandler(config config.ConfigStore, orchestrator *orchestrator.Orchestrator, oplog *oplog.OpLog) *BackrestHandler { +func NewBackrestHandler(config config.ConfigStore, orchestrator *orchestrator.Orchestrator, oplog *oplog.OpLog, logStore *rotatinglog.RotatingLog) *BackrestHandler { s := &BackrestHandler{ config: config, orchestrator: orchestrator, oplog: oplog, + logStore: logStore, } return s @@ -397,10 +400,10 @@ func (s *BackrestHandler) ClearHistory(ctx context.Context, req *connect.Request return connect.NewResponse(&emptypb.Empty{}), err } -func (s *BackrestHandler) GetBigOperationData(ctx context.Context, req *connect.Request[v1.OperationDataRequest]) (*connect.Response[types.BytesValue], error) { - data, err := s.oplog.GetBigData(req.Msg.Id, req.Msg.Key) +func (s *BackrestHandler) GetLogs(ctx context.Context, req *connect.Request[v1.LogDataRequest]) (*connect.Response[types.BytesValue], error) { + data, err := s.logStore.Read(req.Msg.GetRef()) if err != nil { - return nil, fmt.Errorf("get operation data: %w", err) + return nil, fmt.Errorf("get log data %v: %w", req.Msg.GetRef(), err) } return connect.NewResponse(&types.BytesValue{Value: data}), nil } diff --git a/internal/hook/hook.go b/internal/hook/hook.go index 9de6b7a..44203ae 100644 --- a/internal/hook/hook.go +++ b/internal/hook/hook.go @@ -12,6 +12,7 @@ import ( v1 "github.com/garethgeorge/backrest/gen/go/v1" "github.com/garethgeorge/backrest/internal/oplog" + "github.com/garethgeorge/backrest/internal/rotatinglog" "go.uber.org/zap" "google.golang.org/protobuf/proto" ) @@ -20,9 +21,21 @@ var ( defaultTemplate = `{{ .Summary }}` ) +type HookExecutor struct { + oplog *oplog.OpLog + logStore *rotatinglog.RotatingLog +} + +func NewHookExecutor(oplog *oplog.OpLog, bigOutputStore *rotatinglog.RotatingLog) *HookExecutor { + return &HookExecutor{ + oplog: oplog, + logStore: bigOutputStore, + } +} + // ExecuteHooks schedules tasks for the hooks subscribed to the given event. The vars map is used to substitute variables // Hooks are pulled both from the provided plan and from the repo config. -func ExecuteHooks(oplog *oplog.OpLog, repo *v1.Repo, plan *v1.Plan, snapshotId string, events []v1.Hook_Condition, vars HookVars) { +func (e *HookExecutor) ExecuteHooks(repo *v1.Repo, plan *v1.Plan, snapshotId string, events []v1.Hook_Condition, vars HookVars) { operationBase := v1.Operation{ Status: v1.OperationStatus_STATUS_INPROGRESS, PlanId: plan.Id, @@ -51,7 +64,7 @@ func ExecuteHooks(oplog *oplog.OpLog, repo *v1.Repo, plan *v1.Plan, snapshotId s }, } zap.L().Info("Running hook", zap.String("plan", plan.Id), zap.Int64("opId", operation.Id), zap.String("hook", name)) - executeHook(oplog, operation, h, event, vars) + e.executeHook(operation, h, event, vars) } for idx, hook := range plan.GetHooks() { @@ -70,7 +83,7 @@ func ExecuteHooks(oplog *oplog.OpLog, repo *v1.Repo, plan *v1.Plan, snapshotId s }, } zap.L().Info("Running hook", zap.String("plan", plan.Id), zap.Int64("opId", operation.Id), zap.String("hook", name)) - executeHook(oplog, operation, (*Hook)(hook), event, vars) + e.executeHook(operation, h, event, vars) } } @@ -83,8 +96,8 @@ func firstMatchingCondition(hook *Hook, events []v1.Hook_Condition) v1.Hook_Cond return v1.Hook_CONDITION_UNKNOWN } -func executeHook(oplog *oplog.OpLog, op *v1.Operation, hook *Hook, event v1.Hook_Condition, vars HookVars) { - if err := oplog.Add(op); err != nil { +func (e *HookExecutor) executeHook(op *v1.Operation, hook *Hook, event v1.Hook_Condition, vars HookVars) { + if err := e.oplog.Add(op); err != nil { zap.S().Errorf("execute hook: add operation: %v", err) return } @@ -108,13 +121,15 @@ func executeHook(oplog *oplog.OpLog, op *v1.Operation, hook *Hook, event v1.Hook op.Status = v1.OperationStatus_STATUS_SUCCESS } - if err := oplog.SetBigData(op.Id, "hook.log", output.Bytes()); err != nil { - zap.S().Errorf("execute hook: set big data %q: %v", "hook.log", err) + outputRef, err := e.logStore.Write(output.Bytes()) + if err != nil { + zap.S().Errorf("execute hook: write log: %v", err) + return } - op.Op.(*v1.Operation_OperationRunHook).OperationRunHook.OutputRef = "hook.log" + op.Op.(*v1.Operation_OperationRunHook).OperationRunHook.OutputLogref = outputRef op.UnixTimeEndMs = curTimeMs() - if err := oplog.Update(op); err != nil { + if err := e.oplog.Update(op); err != nil { zap.S().Errorf("execute hook: update operation: %v", err) return } diff --git a/internal/oplog/bigopdatastore.go b/internal/oplog/bigopdatastore.go deleted file mode 100644 index 6e3166a..0000000 --- a/internal/oplog/bigopdatastore.go +++ /dev/null @@ -1,62 +0,0 @@ -package oplog - -import ( - "bytes" - "os" - "strconv" - - "github.com/natefinch/atomic" - "go.uber.org/zap" -) - -type BigOpDataStore struct { - path string -} - -func NewBigOpDataStore(path string) *BigOpDataStore { - return &BigOpDataStore{ - path: path, - } -} - -func (s *BigOpDataStore) resolvePath(opId int64) string { - return s.path + "/" + strconv.FormatInt(opId&0xFF, 16) + "/" + strconv.FormatInt(opId, 16) -} - -func (s *BigOpDataStore) DeleteOperationData(opId int64) error { - dir := s.resolvePath(opId) - files, err := os.ReadDir(dir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - - for _, dirent := range files { - if dirent.Name() == "." || dirent.Name() == ".." { - continue - } - if err := os.Remove(dir + "/" + dirent.Name()); err != nil { - zap.S().Errorf("deleting big operation data for operation %v failed to delete %v: %v", opId, dirent.Name(), err) - continue - } - } - return os.Remove(dir) -} - -func (s *BigOpDataStore) SetBigData(opId int64, key string, data []byte) error { - dir := s.resolvePath(opId) - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - return atomic.WriteFile(dir+"/"+key, bytes.NewReader(data)) -} - -func (s *BigOpDataStore) GetBigData(opId int64, key string) ([]byte, error) { - return os.ReadFile(s.resolvePath(opId) + "/" + key) -} - -func (s *BigOpDataStore) DeleteBigData(opId int64, key string) error { - return os.Remove(s.resolvePath(opId) + "/" + key) -} diff --git a/internal/oplog/bigopdatastore_test.go b/internal/oplog/bigopdatastore_test.go deleted file mode 100644 index dddc74a..0000000 --- a/internal/oplog/bigopdatastore_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package oplog - -import ( - "testing" -) - -func TestBigOpLogDataStore(t *testing.T) { - store := &BigOpDataStore{ - path: t.TempDir(), - } - - if err := store.SetBigData(1, "test", []byte("hello world")); err != nil { - t.Fatal(err) - } - - if byte, err := store.GetBigData(1, "test"); err != nil || string(byte) != "hello world" { - t.Fatalf("expected %q, got %q", "hello world", string(byte)) - } - - if err := store.SetBigData(1, "test", []byte("hello world 2")); err != nil { - t.Fatal(err) - } - - if byte, err := store.GetBigData(1, "test"); err != nil || string(byte) != "hello world 2" { - t.Fatalf("expected %q, got %q", "hello world 2", string(byte)) - } - - if err := store.DeleteOperationData(1); err != nil { - t.Fatal(err) - } - - if _, err := store.GetBigData(1, "test"); err == nil { - t.Fatalf("expected error, got nil") - } -} diff --git a/internal/oplog/oplog.go b/internal/oplog/oplog.go index d962554..cf2a4cf 100644 --- a/internal/oplog/oplog.go +++ b/internal/oplog/oplog.go @@ -13,7 +13,6 @@ import ( "github.com/garethgeorge/backrest/internal/oplog/serializationutil" "github.com/garethgeorge/backrest/internal/protoutil" "github.com/garethgeorge/backrest/pkg/restic" - "github.com/hashicorp/go-multierror" bolt "go.etcd.io/bbolt" "go.uber.org/zap" "google.golang.org/protobuf/proto" @@ -42,7 +41,6 @@ var ( // Operations are indexed by repo and plan. type OpLog struct { db *bolt.DB - *BigOpDataStore subscribersMu sync.RWMutex subscribers []*func(*v1.Operation, *v1.Operation) @@ -60,9 +58,6 @@ func NewOpLog(databasePath string) (*OpLog, error) { o := &OpLog{ db: db, - BigOpDataStore: &BigOpDataStore{ - path: path.Dir(databasePath) + "/opdatav1", - }, } if err := db.Update(func(tx *bolt.Tx) error { @@ -215,11 +210,6 @@ func (o *OpLog) Delete(ids ...int64) error { for _, op := range removedOps { o.notifyHelper(op, nil) } - for _, id := range ids { - if e := o.DeleteOperationData(id); e != nil { - err = multierror.Append(err, fmt.Errorf("deleting big data for operation %v: %w", id, e)) - } - } } return err } diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go index a115178..80306e1 100644 --- a/internal/orchestrator/orchestrator.go +++ b/internal/orchestrator/orchestrator.go @@ -11,7 +11,9 @@ import ( v1 "github.com/garethgeorge/backrest/gen/go/v1" "github.com/garethgeorge/backrest/internal/config" + "github.com/garethgeorge/backrest/internal/hook" "github.com/garethgeorge/backrest/internal/oplog" + "github.com/garethgeorge/backrest/internal/rotatinglog" "github.com/garethgeorge/backrest/pkg/restic" "go.uber.org/zap" "google.golang.org/protobuf/proto" @@ -33,11 +35,12 @@ const ( // Orchestrator is responsible for managing repos and backups. type Orchestrator struct { - mu sync.Mutex - config *v1.Config - OpLog *oplog.OpLog - repoPool *resticRepoPool - taskQueue taskQueue + mu sync.Mutex + config *v1.Config + OpLog *oplog.OpLog + repoPool *resticRepoPool + taskQueue taskQueue + hookExecutor *hook.HookExecutor // now for the purpose of testing; used by Run() to get the current time. now func() time.Time @@ -45,7 +48,7 @@ type Orchestrator struct { runningTask atomic.Pointer[taskExecutionInfo] } -func NewOrchestrator(resticBin string, cfg *v1.Config, oplog *oplog.OpLog) (*Orchestrator, error) { +func NewOrchestrator(resticBin string, cfg *v1.Config, oplog *oplog.OpLog, logStore *rotatinglog.RotatingLog) (*Orchestrator, error) { cfg = proto.Clone(cfg).(*v1.Config) // create the orchestrator. @@ -58,6 +61,7 @@ func NewOrchestrator(resticBin string, cfg *v1.Config, oplog *oplog.OpLog) (*Orc taskQueue: newTaskQueue(func() time.Time { return o.curTime() }), + hookExecutor: hook.NewHookExecutor(oplog, logStore), } // verify the operation log and mark any incomplete operations as failed. diff --git a/internal/orchestrator/orchestrator_test.go b/internal/orchestrator/orchestrator_test.go index 81dbbe1..0e6a1dd 100644 --- a/internal/orchestrator/orchestrator_test.go +++ b/internal/orchestrator/orchestrator_test.go @@ -42,7 +42,7 @@ func TestTaskScheduling(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - orch, err := NewOrchestrator("", config.NewDefaultConfig(), nil) + orch, err := NewOrchestrator("", config.NewDefaultConfig(), nil, nil) if err != nil { t.Fatalf("failed to create orchestrator: %v", err) } @@ -81,7 +81,7 @@ func TestTaskRescheduling(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - orch, err := NewOrchestrator("", config.NewDefaultConfig(), nil) + orch, err := NewOrchestrator("", config.NewDefaultConfig(), nil, nil) if err != nil { t.Fatalf("failed to create orchestrator: %v", err) } @@ -130,7 +130,7 @@ func TestGracefulShutdown(t *testing.T) { t.Parallel() // Arrange - orch, err := NewOrchestrator("", config.NewDefaultConfig(), nil) + orch, err := NewOrchestrator("", config.NewDefaultConfig(), nil, nil) if err != nil { t.Fatalf("failed to create orchestrator: %v", err) } @@ -150,7 +150,7 @@ func TestSchedulerWait(t *testing.T) { // Arrange curTime := time.Now() - orch, err := NewOrchestrator("", config.NewDefaultConfig(), nil) + orch, err := NewOrchestrator("", config.NewDefaultConfig(), nil, nil) if err != nil { t.Fatalf("failed to create orchestrator: %v", err) } diff --git a/internal/orchestrator/tasks.go b/internal/orchestrator/task.go similarity index 100% rename from internal/orchestrator/tasks.go rename to internal/orchestrator/task.go diff --git a/internal/orchestrator/taskbackup.go b/internal/orchestrator/taskbackup.go index 446a7ea..0863e63 100644 --- a/internal/orchestrator/taskbackup.go +++ b/internal/orchestrator/taskbackup.go @@ -104,7 +104,7 @@ func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan return fmt.Errorf("couldn't get repo %q: %w", plan.Repo, err) } - hook.ExecuteHooks(orchestrator.OpLog, repo.Config(), plan, "", []v1.Hook_Condition{ + orchestrator.hookExecutor.ExecuteHooks(repo.Config(), plan, "", []v1.Hook_Condition{ v1.Hook_CONDITION_SNAPSHOT_START, }, hook.HookVars{ Task: t.Name(), @@ -137,7 +137,7 @@ func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan } if err != nil { vars.Error = err.Error() - hook.ExecuteHooks(orchestrator.OpLog, repo.Config(), plan, "", []v1.Hook_Condition{ + orchestrator.hookExecutor.ExecuteHooks(repo.Config(), plan, "", []v1.Hook_Condition{ v1.Hook_CONDITION_SNAPSHOT_ERROR, v1.Hook_CONDITION_ANY_ERROR, }, vars) @@ -148,7 +148,7 @@ func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan op.DisplayMessage = "Partial backup, some files may not have been read completely." } - hook.ExecuteHooks(orchestrator.OpLog, repo.Config(), plan, summary.SnapshotId, []v1.Hook_Condition{ + orchestrator.hookExecutor.ExecuteHooks(repo.Config(), plan, summary.SnapshotId, []v1.Hook_Condition{ v1.Hook_CONDITION_SNAPSHOT_END, }, vars) diff --git a/internal/orchestrator/taskforget.go b/internal/orchestrator/taskforget.go index 5d87596..803b802 100644 --- a/internal/orchestrator/taskforget.go +++ b/internal/orchestrator/taskforget.go @@ -109,7 +109,7 @@ func (t *ForgetTask) Run(ctx context.Context) error { return err }); err != nil { repo, _ := t.orch.GetRepo(t.plan.Repo) - hook.ExecuteHooks(t.orch.OpLog, repo.Config(), t.plan, t.linkSnapshot, []v1.Hook_Condition{ + t.orch.hookExecutor.ExecuteHooks(repo.Config(), t.plan, t.linkSnapshot, []v1.Hook_Condition{ v1.Hook_CONDITION_ANY_ERROR, }, hook.HookVars{ Task: t.Name(), diff --git a/internal/orchestrator/taskindexsnapshots.go b/internal/orchestrator/taskindexsnapshots.go index 5b27f8c..a0e2334 100644 --- a/internal/orchestrator/taskindexsnapshots.go +++ b/internal/orchestrator/taskindexsnapshots.go @@ -48,7 +48,7 @@ func (t *IndexSnapshotsTask) Next(now time.Time) *time.Time { func (t *IndexSnapshotsTask) Run(ctx context.Context) error { if err := indexSnapshotsHelper(ctx, t.orchestrator, t.repoId); err != nil { repo, _ := t.orchestrator.GetRepo(t.repoId) - hook.ExecuteHooks(t.orchestrator.OpLog, repo.Config(), nil, "", []v1.Hook_Condition{ + t.orchestrator.hookExecutor.ExecuteHooks(repo.Config(), nil, "", []v1.Hook_Condition{ v1.Hook_CONDITION_ANY_ERROR, }, hook.HookVars{ Task: t.Name(), diff --git a/internal/orchestrator/taskprune.go b/internal/orchestrator/taskprune.go index db1d7ce..140b675 100644 --- a/internal/orchestrator/taskprune.go +++ b/internal/orchestrator/taskprune.go @@ -163,7 +163,7 @@ func (t *PruneTask) Run(ctx context.Context) error { return nil }); err != nil { repo, _ := t.orch.GetRepo(t.plan.Repo) - hook.ExecuteHooks(t.orch.OpLog, repo.Config(), t.plan, "", []v1.Hook_Condition{ + t.orch.hookExecutor.ExecuteHooks(repo.Config(), t.plan, "", []v1.Hook_Condition{ v1.Hook_CONDITION_ANY_ERROR, }, hook.HookVars{ Task: t.Name(), diff --git a/internal/orchestrator/taskrestore.go b/internal/orchestrator/taskrestore.go index 5786721..c945467 100644 --- a/internal/orchestrator/taskrestore.go +++ b/internal/orchestrator/taskrestore.go @@ -96,7 +96,7 @@ func (t *RestoreTask) Run(ctx context.Context) error { return nil }); err != nil { repo, _ := t.orch.GetRepo(t.restoreOpts.Plan.Repo) - hook.ExecuteHooks(t.orch.OpLog, repo.Config(), nil, t.restoreOpts.SnapshotId, []v1.Hook_Condition{ + t.orch.hookExecutor.ExecuteHooks(repo.Config(), nil, t.restoreOpts.SnapshotId, []v1.Hook_Condition{ v1.Hook_CONDITION_ANY_ERROR, }, hook.HookVars{ Task: t.Name(), diff --git a/internal/orchestrator/taskstats.go b/internal/orchestrator/taskstats.go index 84cc15a..8099d8a 100644 --- a/internal/orchestrator/taskstats.go +++ b/internal/orchestrator/taskstats.go @@ -47,6 +47,9 @@ func (t *StatsTask) shouldRun() (bool, error) { return oplog.ErrStopIteration } else if backup, ok := op.Op.(*v1.Operation_OperationBackup); ok && backup.OperationBackup.LastStatus != nil { if summary, ok := backup.OperationBackup.LastStatus.Entry.(*v1.BackupProgressEntry_Summary); ok { + if bytesSinceLastStat == -1 { + bytesSinceLastStat = 0 + } bytesSinceLastStat += summary.Summary.DataAdded } } @@ -115,7 +118,7 @@ func (t *StatsTask) Run(ctx context.Context) error { return err }); err != nil { repo, _ := t.orch.GetRepo(t.plan.Repo) - hook.ExecuteHooks(t.orch.OpLog, repo.Config(), t.plan, "", []v1.Hook_Condition{ + t.orch.hookExecutor.ExecuteHooks(repo.Config(), t.plan, "", []v1.Hook_Condition{ v1.Hook_CONDITION_ANY_ERROR, }, hook.HookVars{ Task: t.Name(), diff --git a/internal/rotatinglog/rotatinglog.go b/internal/rotatinglog/rotatinglog.go new file mode 100644 index 0000000..d82dfbc --- /dev/null +++ b/internal/rotatinglog/rotatinglog.go @@ -0,0 +1,195 @@ +package rotatinglog + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +var ErrFileNotFound = errors.New("file not found") +var ErrNotFound = errors.New("entry not found") +var ErrBadName = errors.New("bad name") + +type RotatingLog struct { + mu sync.Mutex + dir string + lastFile string + maxLogFiles int +} + +func NewRotatingLog(dir string, maxLogFiles int) *RotatingLog { + return &RotatingLog{dir: dir, maxLogFiles: maxLogFiles} +} + +func (r *RotatingLog) curfile() string { + return path.Join(r.dir, time.Now().Format("2006-01-02.tar.gz")) +} + +func (r *RotatingLog) removeExpiredFiles() error { + if r.maxLogFiles < 0 { + return nil + } + files, err := os.ReadDir(r.dir) + if err != nil { + return err + } + files = slices.DeleteFunc(files, func(f fs.DirEntry) bool { + return f.IsDir() + }) + sort.Slice(files, func(i, j int) bool { + return files[i].Name() < files[j].Name() + }) + if len(files) >= r.maxLogFiles { + for i := 0; i < len(files)-r.maxLogFiles+1; i++ { + if err := os.Remove(path.Join(r.dir, files[i].Name())); err != nil { + return err + } + } + } + return nil +} + +func (r *RotatingLog) Write(data []byte) (string, error) { + r.mu.Lock() + defer r.mu.Unlock() + + data, err := compress(data) + if err != nil { + return "", err + } + + file := r.curfile() + if file != r.lastFile { + if err := os.MkdirAll(r.dir, os.ModePerm); err != nil { + return "", err + } + r.lastFile = file + } + f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, os.ModePerm) + if err != nil { + return "", err + } + defer f.Close() + + size, err := f.Seek(0, io.SeekEnd) + if err != nil { + return "", err + } + pos := int64(0) + if size != 0 { + pos, err = f.Seek(-1024, io.SeekEnd) + if err != nil { + return "", err + } + } + tw := tar.NewWriter(f) + defer tw.Close() + name := fmt.Sprintf("%s/%d", path.Base(file), pos) + tw.WriteHeader(&tar.Header{ + Name: name, + Size: int64(len(data)), + Mode: 0600, + Typeflag: tar.TypeReg, + ModTime: time.Now(), + }) + + _, err = tw.Write(data) + if err != nil { + return "", err + } + + return name, nil +} + +func (r *RotatingLog) Read(name string) ([]byte, error) { + r.mu.Lock() + defer r.mu.Unlock() + + // parse name e.g. of the form "2006-01-02-15-04-05.tar/1234" + splitAt := strings.Index(name, "/") + if splitAt == -1 { + return nil, ErrBadName + } + + offset, err := strconv.Atoi(name[splitAt+1:]) + if err != nil { + return nil, ErrBadName + } + + // open file and seek to the offset where the tarball segment should start + f, err := os.Open(path.Join(r.dir, name[:splitAt])) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrFileNotFound + } + return nil, fmt.Errorf("open failed: %w", err) + } + defer f.Close() + f.Seek(int64(offset), io.SeekStart) + + // search for the tarball segment in the tarball and read + decompress it if found + tr := tar.NewReader(f) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("next failed: %v", err) + } + if hdr.Name == name { + buf := make([]byte, hdr.Size) + _, err := io.ReadFull(tr, buf) + if err != nil { + return nil, fmt.Errorf("read failed: %v", err) + } + return decompress(buf) + } + } + return nil, ErrNotFound +} + +func compress(data []byte) ([]byte, error) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + + if _, err := zw.Write(data); err != nil { + return nil, err + } + + if err := zw.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func decompress(compressedData []byte) ([]byte, error) { + var buf bytes.Buffer + zr, err := gzip.NewReader(bytes.NewReader(compressedData)) + if err != nil { + return nil, err + } + + if _, err := io.Copy(&buf, zr); err != nil { + return nil, err + } + + if err := zr.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/internal/rotatinglog/rotatinglog_test.go b/internal/rotatinglog/rotatinglog_test.go new file mode 100644 index 0000000..bb8432f --- /dev/null +++ b/internal/rotatinglog/rotatinglog_test.go @@ -0,0 +1,61 @@ +package rotatinglog + +import ( + "fmt" + "strings" + "testing" +) + +func TestRotatingLog(t *testing.T) { + log := NewRotatingLog(t.TempDir()+"/rotatinglog", 10) + name, err := log.Write([]byte("test")) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + data, err := log.Read(name) + if err != nil { + t.Fatalf("Read failed: %v", err) + } + if string(data) != "test" { + t.Fatalf("Read failed: expected test, got %s", string(data)) + } +} + +func TestRotatingLogMultipleEntries(t *testing.T) { + log := NewRotatingLog(t.TempDir()+"/rotatinglog", 10) + for i := 0; i < 10; i++ { + name, err := log.Write([]byte(fmt.Sprintf("%d", i))) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + data, err := log.Read(name) + if err != nil { + t.Fatalf("Read failed: %v", err) + } + if fmt.Sprintf("%d", i) != string(data) { + t.Fatalf("Read failed: expected %d, got %s", i, string(data)) + } + } +} + +func TestBigEntries(t *testing.T) { + log := NewRotatingLog(t.TempDir()+"/rotatinglog", 10) + for size := range []int{10, 100, 1234, 5938, 1023, 1025} { + data := genstr(size) + name, err := log.Write([]byte(data)) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + read, err := log.Read(name) + if err != nil { + t.Fatalf("Read failed: %v", err) + } + if string(read) != data { + t.Fatalf("Read failed: expected %s, got %s", data, string(read)) + } + } +} + +func genstr(size int) string { + return strings.Repeat("a", size) +} diff --git a/proto/v1/operations.proto b/proto/v1/operations.proto index 2416c0b..7d4143b 100644 --- a/proto/v1/operations.proto +++ b/proto/v1/operations.proto @@ -98,5 +98,5 @@ message OperationStats { message OperationRunHook { string name = 1; // description of the hook that was run. typically repo/hook_idx or plan/hook_idx. - string output_ref = 2; // reference to the output of the hook. + string output_logref = 2; // logref of the hook's output. } diff --git a/proto/v1/service.proto b/proto/v1/service.proto index 9b1338a..fb0a740 100644 --- a/proto/v1/service.proto +++ b/proto/v1/service.proto @@ -51,7 +51,7 @@ service Backrest { rpc Cancel(types.Int64Value) returns (google.protobuf.Empty) {} // GetBigOperationData returns the keyed large data for the given operation. - rpc GetBigOperationData(OperationDataRequest) returns (types.BytesValue) {} + rpc GetLogs(LogDataRequest) returns (types.BytesValue) {} // Clears the history of operations rpc ClearHistory(ClearHistoryRequest) returns (google.protobuf.Empty) {} @@ -97,9 +97,8 @@ message ListSnapshotFilesResponse { repeated LsEntry entries = 2; } -message OperationDataRequest { - int64 id = 1; // operation id - string key = 2; // data key +message LogDataRequest { + string ref = 1; } message LsEntry { diff --git a/webui/gen/ts/v1/operations_pb.ts b/webui/gen/ts/v1/operations_pb.ts index f755e8c..0c3a0e3 100644 --- a/webui/gen/ts/v1/operations_pb.ts +++ b/webui/gen/ts/v1/operations_pb.ts @@ -627,11 +627,11 @@ export class OperationRunHook extends Message { name = ""; /** - * reference to the output of the hook. + * logref of the hook's output. * - * @generated from field: string output_ref = 2; + * @generated from field: string output_logref = 2; */ - outputRef = ""; + outputLogref = ""; constructor(data?: PartialMessage) { super(); @@ -642,7 +642,7 @@ export class OperationRunHook extends Message { static readonly typeName = "v1.OperationRunHook"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "name", kind: "scalar", T: 9 /* ScalarType.STRING */ }, - { no: 2, name: "output_ref", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 2, name: "output_logref", kind: "scalar", T: 9 /* ScalarType.STRING */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): OperationRunHook { diff --git a/webui/gen/ts/v1/service_connect.ts b/webui/gen/ts/v1/service_connect.ts index b65df58..2d0997f 100644 --- a/webui/gen/ts/v1/service_connect.ts +++ b/webui/gen/ts/v1/service_connect.ts @@ -6,7 +6,7 @@ import { Empty, MethodKind } from "@bufbuild/protobuf"; import { Config, Repo } from "./config_pb.js"; import { OperationEvent, OperationList } from "./operations_pb.js"; -import { ClearHistoryRequest, GetOperationsRequest, ListSnapshotFilesRequest, ListSnapshotFilesResponse, ListSnapshotsRequest, OperationDataRequest, RestoreSnapshotRequest } from "./service_pb.js"; +import { ClearHistoryRequest, GetOperationsRequest, ListSnapshotFilesRequest, ListSnapshotFilesResponse, ListSnapshotsRequest, LogDataRequest, RestoreSnapshotRequest } from "./service_pb.js"; import { ResticSnapshotList } from "./restic_pb.js"; import { BytesValue, Int64Value, StringList, StringValue } from "../types/value_pb.js"; @@ -170,11 +170,11 @@ export const Backrest = { /** * GetBigOperationData returns the keyed large data for the given operation. * - * @generated from rpc v1.Backrest.GetBigOperationData + * @generated from rpc v1.Backrest.GetLogs */ - getBigOperationData: { - name: "GetBigOperationData", - I: OperationDataRequest, + getLogs: { + name: "GetLogs", + I: LogDataRequest, O: BytesValue, kind: MethodKind.Unary, }, diff --git a/webui/gen/ts/v1/service_pb.ts b/webui/gen/ts/v1/service_pb.ts index de8a745..3742d7e 100644 --- a/webui/gen/ts/v1/service_pb.ts +++ b/webui/gen/ts/v1/service_pb.ts @@ -309,49 +309,39 @@ export class ListSnapshotFilesResponse extends Message { +export class LogDataRequest extends Message { /** - * operation id - * - * @generated from field: int64 id = 1; + * @generated from field: string ref = 1; */ - id = protoInt64.zero; + ref = ""; - /** - * data key - * - * @generated from field: string key = 2; - */ - key = ""; - - constructor(data?: PartialMessage) { + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); } static readonly runtime: typeof proto3 = proto3; - static readonly typeName = "v1.OperationDataRequest"; + static readonly typeName = "v1.LogDataRequest"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "id", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, - { no: 2, name: "key", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 1, name: "ref", kind: "scalar", T: 9 /* ScalarType.STRING */ }, ]); - static fromBinary(bytes: Uint8Array, options?: Partial): OperationDataRequest { - return new OperationDataRequest().fromBinary(bytes, options); + static fromBinary(bytes: Uint8Array, options?: Partial): LogDataRequest { + return new LogDataRequest().fromBinary(bytes, options); } - static fromJson(jsonValue: JsonValue, options?: Partial): OperationDataRequest { - return new OperationDataRequest().fromJson(jsonValue, options); + static fromJson(jsonValue: JsonValue, options?: Partial): LogDataRequest { + return new LogDataRequest().fromJson(jsonValue, options); } - static fromJsonString(jsonString: string, options?: Partial): OperationDataRequest { - return new OperationDataRequest().fromJsonString(jsonString, options); + static fromJsonString(jsonString: string, options?: Partial): LogDataRequest { + return new LogDataRequest().fromJsonString(jsonString, options); } - static equals(a: OperationDataRequest | PlainMessage | undefined, b: OperationDataRequest | PlainMessage | undefined): boolean { - return proto3.util.equals(OperationDataRequest, a, b); + static equals(a: LogDataRequest | PlainMessage | undefined, b: LogDataRequest | PlainMessage | undefined): boolean { + return proto3.util.equals(LogDataRequest, a, b); } } diff --git a/webui/src/components/OperationList.tsx b/webui/src/components/OperationList.tsx index 258d0be..ead4ef7 100644 --- a/webui/src/components/OperationList.tsx +++ b/webui/src/components/OperationList.tsx @@ -44,7 +44,7 @@ import { normalizeSnapshotId, } from "../lib/formatting"; import _ from "lodash"; -import { GetOperationsRequest, OperationDataRequest } from "../../gen/ts/v1/service_pb"; +import { GetOperationsRequest, LogDataRequest } from "../../gen/ts/v1/service_pb"; import { useAlertApi } from "./Alerts"; import { MessageInstance } from "antd/es/message/interface"; import { backrestService } from "../api"; @@ -494,7 +494,7 @@ const RunHookOperationStatus = ({ op }: { op: Operation }) => { key: 1, label: "Logs for hook " + hook.name, children: <> - + }, ]} /> @@ -502,22 +502,21 @@ const RunHookOperationStatus = ({ op }: { op: Operation }) => { } // TODO: refactor this to use the provider pattern -const BigOperationDataVerbatim = ({ id, outputRef }: { id: bigint, outputRef: string }) => { +const BigOperationDataVerbatim = ({ logref }: { logref: string }) => { const [output, setOutput] = useState(undefined); useEffect(() => { - if (!outputRef) { + if (!logref) { return; } - backrestService.getBigOperationData(new OperationDataRequest({ - id: id, - key: outputRef, + backrestService.getLogs(new LogDataRequest({ + ref: logref, })).then((resp) => { setOutput(new TextDecoder("utf-8").decode(resp.value)); }).catch((e) => { console.error("Failed to fetch hook output: ", e); }); - }, [id, outputRef]); + }, [logref]); return 
{output}
; } \ No newline at end of file