diff --git a/gen/go/v1/operations.pb.go b/gen/go/v1/operations.pb.go index 8c060c36..cbe08c26 100644 --- a/gen/go/v1/operations.pb.go +++ b/gen/go/v1/operations.pb.go @@ -191,6 +191,8 @@ type Operation struct { // required, primary ID of the operation. ID is sequential based on creation time of the operation. Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + // flow id groups operations together, e.g. by an execution of a plan. + FlowId int64 `protobuf:"varint,10,opt,name=flow_id,json=flowId,proto3" json:"flow_id,omitempty"` // optional, flow id if associated with a flow // required, repo id if associated with a repo RepoId string `protobuf:"bytes,2,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"` // required, plan id if associated with a plan @@ -257,6 +259,13 @@ func (x *Operation) GetId() int64 { return 0 } +func (x *Operation) GetFlowId() int64 { + if x != nil { + return x.FlowId + } + return 0 +} + func (x *Operation) GetRepoId() string { if x != nil { return x.RepoId @@ -809,8 +818,9 @@ type OperationRunHook struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // description of the hook that was run. typically repo/hook_idx or plan/hook_idx. - OutputLogref string `protobuf:"bytes,2,opt,name=output_logref,json=outputLogref,proto3" json:"output_logref,omitempty"` // logref of the hook's output. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // description of the hook that was run. typically repo/hook_idx or plan/hook_idx. + OutputLogref string `protobuf:"bytes,2,opt,name=output_logref,json=outputLogref,proto3" json:"output_logref,omitempty"` // logref of the hook's output. DEPRECATED. + Condition Hook_Condition `protobuf:"varint,3,opt,name=condition,proto3,enum=v1.Hook_Condition" json:"condition,omitempty"` // triggering condition of the hook. } func (x *OperationRunHook) Reset() { @@ -859,6 +869,13 @@ func (x *OperationRunHook) GetOutputLogref() string { return "" } +func (x *OperationRunHook) GetCondition() Hook_Condition { + if x != nil { + return x.Condition + } + return Hook_CONDITION_UNKNOWN +} + var File_v1_operations_proto protoreflect.FileDescriptor var file_v1_operations_proto_rawDesc = []byte{ @@ -869,127 +886,131 @@ var file_v1_operations_proto_rawDesc = []byte{ 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x0a, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x0a, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x9d, 0x06, 0x0a, 0x09, + 0x0a, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xb6, 0x06, 0x0a, 0x09, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, - 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, - 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, - 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x76, - 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x12, 0x75, 0x6e, 0x69, - 0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6d, 0x73, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x75, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x4d, 0x73, 0x12, 0x27, 0x0a, 0x10, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x6d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x0d, 0x75, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x45, 0x6e, 0x64, 0x4d, 0x73, 0x12, - 0x27, 0x0a, 0x0f, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, - 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x6f, 0x67, 0x72, - 0x65, 0x66, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x6f, 0x67, 0x72, 0x65, 0x66, - 0x12, 0x40, 0x0a, 0x10, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x62, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x18, 0x64, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, - 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x48, - 0x00, 0x52, 0x0f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, - 0x75, 0x70, 0x12, 0x56, 0x0a, 0x18, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x65, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x48, 0x00, 0x52, 0x16, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, - 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x40, 0x0a, 0x10, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x18, 0x66, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x3d, 0x0a, 0x0f, - 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x75, 0x6e, 0x65, 0x18, - 0x67, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, 0x43, 0x0a, 0x11, 0x6f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x18, 0x68, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x48, 0x00, 0x52, 0x10, - 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x12, 0x3d, 0x0a, 0x0f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, - 0x61, 0x74, 0x73, 0x18, 0x69, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x48, 0x00, 0x52, - 0x0e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, - 0x44, 0x0a, 0x12, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6e, - 0x5f, 0x68, 0x6f, 0x6f, 0x6b, 0x18, 0x6a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x76, 0x31, - 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x48, 0x6f, 0x6f, - 0x6b, 0x48, 0x00, 0x52, 0x10, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, - 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x42, 0x04, 0x0a, 0x02, 0x6f, 0x70, 0x22, 0x69, 0x0a, 0x0e, 0x4f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x2a, 0x0a, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x76, 0x31, - 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x2b, 0x0a, 0x09, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, - 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x7c, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b, 0x6c, 0x61, 0x73, - 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, - 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, - 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x06, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x73, 0x22, 0x82, 0x01, 0x0a, 0x16, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, - 0x2e, 0x0a, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, - 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, - 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x67, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x06, 0x66, 0x6f, 0x72, 0x67, 0x6f, 0x74, 0x12, 0x20, 0x0a, 0x0c, 0x66, 0x6f, 0x72, 0x67, 0x6f, - 0x74, 0x5f, 0x62, 0x79, 0x5f, 0x6f, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x66, - 0x6f, 0x72, 0x67, 0x6f, 0x74, 0x42, 0x79, 0x4f, 0x70, 0x22, 0x6a, 0x0a, 0x0f, 0x4f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2a, 0x0a, 0x06, - 0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, - 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x52, 0x06, 0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2b, 0x0a, 0x06, 0x70, 0x6f, 0x6c, 0x69, - 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x06, 0x70, - 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, 0x28, 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x22, - 0x70, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, - 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x67, - 0x72, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x22, 0x35, 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x53, 0x74, 0x61, 0x74, - 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, 0x4b, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x6c, 0x6f, 0x67, 0x72, 0x65, - 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x4c, - 0x6f, 0x67, 0x72, 0x65, 0x66, 0x2a, 0x60, 0x0a, 0x12, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x45, - 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x11, - 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, - 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, - 0x45, 0x44, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x44, 0x45, - 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x03, 0x2a, 0xc2, 0x01, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, - 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, - 0x47, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e, - 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, - 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x03, 0x12, 0x12, - 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, 0x47, - 0x10, 0x07, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, - 0x4f, 0x52, 0x10, 0x04, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, - 0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, - 0x05, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x53, 0x45, 0x52, - 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x06, 0x42, 0x2c, 0x5a, 0x2a, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, - 0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, - 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x6c, 0x6f, + 0x77, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x66, 0x6c, 0x6f, 0x77, + 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x70, + 0x6c, 0x61, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, + 0x61, 0x6e, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, + 0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, + 0x68, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x12, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, + 0x75, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x72, 0x74, 0x4d, 0x73, 0x12, + 0x27, 0x0a, 0x10, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6e, 0x64, + 0x5f, 0x6d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x75, 0x6e, 0x69, 0x78, 0x54, + 0x69, 0x6d, 0x65, 0x45, 0x6e, 0x64, 0x4d, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x64, 0x69, 0x73, 0x70, + 0x6c, 0x61, 0x79, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0e, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x6f, 0x67, 0x72, 0x65, 0x66, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x6c, 0x6f, 0x67, 0x72, 0x65, 0x66, 0x12, 0x40, 0x0a, 0x10, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x64, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x56, 0x0a, 0x18, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x73, + 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x65, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x48, 0x00, 0x52, 0x16, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, + 0x68, 0x6f, 0x74, 0x12, 0x40, 0x0a, 0x10, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x18, 0x66, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, + 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x72, 0x67, + 0x65, 0x74, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, + 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x3d, 0x0a, 0x0f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x75, 0x6e, 0x65, 0x18, 0x67, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x75, + 0x6e, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, + 0x72, 0x75, 0x6e, 0x65, 0x12, 0x43, 0x0a, 0x11, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x68, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x14, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x48, 0x00, 0x52, 0x10, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x3d, 0x0a, 0x0f, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x69, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x48, 0x00, 0x52, 0x0e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x44, 0x0a, 0x12, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x68, 0x6f, 0x6f, 0x6b, 0x18, 0x6a, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x48, 0x00, 0x52, 0x10, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x42, 0x04, + 0x0a, 0x02, 0x6f, 0x70, 0x22, 0x69, 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x12, 0x2b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x7c, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, + 0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x06, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, + 0x31, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0x82, 0x01, + 0x0a, 0x16, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78, + 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x08, 0x73, 0x6e, 0x61, 0x70, + 0x73, 0x68, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x08, + 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x67, + 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x67, 0x6f, 0x74, + 0x12, 0x20, 0x0a, 0x0c, 0x66, 0x6f, 0x72, 0x67, 0x6f, 0x74, 0x5f, 0x62, 0x79, 0x5f, 0x6f, 0x70, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x66, 0x6f, 0x72, 0x67, 0x6f, 0x74, 0x42, 0x79, + 0x4f, 0x70, 0x22, 0x6a, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, + 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2a, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, + 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x67, 0x65, + 0x74, 0x12, 0x2b, 0x0a, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, + 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, 0x28, + 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x75, 0x6e, 0x65, + 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x22, 0x70, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x35, 0x0a, 0x0e, 0x4f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x05, + 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x31, + 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, + 0x73, 0x22, 0x7d, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, + 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x75, 0x74, + 0x70, 0x75, 0x74, 0x5f, 0x6c, 0x6f, 0x67, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0c, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x4c, 0x6f, 0x67, 0x72, 0x65, 0x66, 0x12, 0x30, + 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x6f, 0x6f, 0x6b, 0x2e, 0x43, 0x6f, 0x6e, 0x64, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x2a, 0x60, 0x0a, 0x12, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, + 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, + 0x4e, 0x54, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, + 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, + 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, + 0x10, 0x03, 0x2a, 0xc2, 0x01, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, + 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x15, + 0x0a, 0x11, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e, 0x50, 0x52, 0x4f, 0x47, 0x52, + 0x45, 0x53, 0x53, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, + 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, + 0x54, 0x55, 0x53, 0x5f, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x07, 0x12, 0x10, 0x0a, + 0x0c, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x12, + 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, + 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, + 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x43, 0x41, 0x4e, 0x43, + 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x06, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, 0x72, + 0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, 0x67, 0x65, 0x6e, 0x2f, + 0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1025,6 +1046,7 @@ var file_v1_operations_proto_goTypes = []interface{}{ (*RetentionPolicy)(nil), // 15: v1.RetentionPolicy (*RestoreProgressEntry)(nil), // 16: v1.RestoreProgressEntry (*RepoStats)(nil), // 17: v1.RepoStats + (Hook_Condition)(0), // 18: v1.Hook.Condition } var file_v1_operations_proto_depIdxs = []int32{ 3, // 0: v1.OperationList.operations:type_name -> v1.Operation @@ -1045,11 +1067,12 @@ var file_v1_operations_proto_depIdxs = []int32{ 15, // 15: v1.OperationForget.policy:type_name -> v1.RetentionPolicy 16, // 16: v1.OperationRestore.status:type_name -> v1.RestoreProgressEntry 17, // 17: v1.OperationStats.stats:type_name -> v1.RepoStats - 18, // [18:18] is the sub-list for method output_type - 18, // [18:18] is the sub-list for method input_type - 18, // [18:18] is the sub-list for extension type_name - 18, // [18:18] is the sub-list for extension extendee - 0, // [0:18] is the sub-list for field type_name + 18, // 18: v1.OperationRunHook.condition:type_name -> v1.Hook.Condition + 19, // [19:19] is the sub-list for method output_type + 19, // [19:19] is the sub-list for method input_type + 19, // [19:19] is the sub-list for extension type_name + 19, // [19:19] is the sub-list for extension extendee + 0, // [0:19] is the sub-list for field type_name } func init() { file_v1_operations_proto_init() } diff --git a/gen/go/v1/service.pb.go b/gen/go/v1/service.pb.go index 42912e0c..95a88d29 100644 --- a/gen/go/v1/service.pb.go +++ b/gen/go/v1/service.pb.go @@ -220,6 +220,7 @@ type GetOperationsRequest struct { RepoId string `protobuf:"bytes,1,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"` PlanId string `protobuf:"bytes,2,opt,name=plan_id,json=planId,proto3" json:"plan_id,omitempty"` SnapshotId string `protobuf:"bytes,4,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"` + FlowId int64 `protobuf:"varint,6,opt,name=flow_id,json=flowId,proto3" json:"flow_id,omitempty"` Ids []int64 `protobuf:"varint,5,rep,packed,name=ids,proto3" json:"ids,omitempty"` LastN int64 `protobuf:"varint,3,opt,name=last_n,json=lastN,proto3" json:"last_n,omitempty"` // limit to the last n operations } @@ -277,6 +278,13 @@ func (x *GetOperationsRequest) GetSnapshotId() string { return "" } +func (x *GetOperationsRequest) GetFlowId() int64 { + if x != nil { + return x.FlowId + } + return 0 +} + func (x *GetOperationsRequest) GetIds() []int64 { if x != nil { return x.Ids @@ -685,123 +693,125 @@ var file_v1_service_proto_rawDesc = []byte{ 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, - 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x22, 0x92, 0x01, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, + 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x22, 0xab, 0x01, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, - 0x64, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x03, 0x52, 0x03, - 0x69, 0x64, 0x73, 0x12, 0x15, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6e, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x05, 0x6c, 0x61, 0x73, 0x74, 0x4e, 0x22, 0x97, 0x01, 0x0a, 0x16, 0x52, - 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x17, - 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, - 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x22, 0x68, 0x0a, 0x18, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, - 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, - 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, - 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x56, - 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, - 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, - 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, - 0x25, 0x0a, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x0b, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x65, - 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0x22, 0x0a, 0x0e, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, - 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0xd3, 0x01, 0x0a, 0x07, 0x4c, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, - 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, - 0x74, 0x68, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x03, 0x67, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, - 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, - 0x74, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x61, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x74, - 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x63, 0x74, 0x69, 0x6d, 0x65, - 0x32, 0xa7, 0x08, 0x0a, 0x08, 0x42, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, - 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, - 0x12, 0x25, 0x0a, 0x09, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x0a, 0x2e, - 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, 0x12, 0x21, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x52, 0x65, - 0x70, 0x6f, 0x12, 0x08, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x1a, 0x0a, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, 0x12, 0x44, 0x0a, 0x12, 0x47, 0x65, - 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, - 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x00, 0x30, 0x01, - 0x12, 0x3e, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x76, 0x31, - 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, - 0x12, 0x43, 0x0a, 0x0d, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, - 0x68, 0x6f, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x76, 0x31, - 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x4c, - 0x69, 0x73, 0x74, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, - 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x76, 0x31, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3e, 0x0a, 0x0e, 0x49, 0x6e, 0x64, - 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x12, 0x12, 0x2e, 0x74, 0x79, - 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, - 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x42, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, + 0x64, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x06, 0x66, 0x6c, 0x6f, 0x77, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x64, + 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x03, 0x52, 0x03, 0x69, 0x64, 0x73, 0x12, 0x15, 0x0a, 0x06, + 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6c, 0x61, + 0x73, 0x74, 0x4e, 0x22, 0x97, 0x01, 0x0a, 0x16, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, + 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, + 0x0a, 0x07, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x70, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, + 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, + 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0x68, 0x0a, + 0x18, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, + 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, + 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, + 0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x56, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x53, + 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x25, 0x0a, 0x07, 0x65, 0x6e, 0x74, 0x72, + 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x76, 0x31, 0x2e, 0x4c, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, + 0x22, 0x0a, 0x0e, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x72, 0x65, 0x66, 0x22, 0xd3, 0x01, 0x0a, 0x07, 0x4c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x10, 0x0a, 0x03, 0x75, + 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, + 0x03, 0x67, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x67, 0x69, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, + 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, + 0x05, 0x61, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x61, 0x74, + 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x63, 0x74, 0x69, 0x6d, 0x65, 0x32, 0xa7, 0x08, 0x0a, 0x08, 0x42, 0x61, + 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x0a, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, 0x12, 0x25, 0x0a, 0x09, 0x53, 0x65, 0x74, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, + 0x12, 0x21, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x52, 0x65, 0x70, 0x6f, 0x12, 0x08, 0x2e, 0x76, 0x31, + 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x22, 0x00, 0x12, 0x44, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x1a, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x3e, 0x0a, 0x0d, 0x47, 0x65, 0x74, + 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, + 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0d, 0x4c, 0x69, 0x73, + 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, + 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x12, 0x52, + 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, + 0x6c, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, + 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1d, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, + 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x3e, 0x0a, 0x0e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, + 0x68, 0x6f, 0x74, 0x73, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x12, 0x2e, 0x74, + 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x05, 0x50, 0x72, + 0x75, 0x6e, 0x65, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, - 0x00, 0x12, 0x35, 0x0a, 0x05, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, + 0x00, 0x12, 0x35, 0x0a, 0x06, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x11, 0x2e, 0x76, 0x31, + 0x2e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x12, 0x1a, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, + 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x55, 0x6e, 0x6c, + 0x6f, 0x63, 0x6b, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, + 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, + 0x00, 0x12, 0x35, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x06, 0x46, 0x6f, 0x72, 0x67, - 0x65, 0x74, 0x12, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, - 0x3f, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1a, 0x2e, 0x76, 0x31, 0x2e, - 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, - 0x12, 0x36, 0x0a, 0x06, 0x55, 0x6e, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, - 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, - 0x73, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, + 0x65, 0x6c, 0x12, 0x11, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, - 0x35, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x11, 0x2e, 0x74, 0x79, 0x70, 0x65, - 0x73, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, + 0x32, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x12, 0x2e, 0x76, 0x31, 0x2e, + 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, + 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0c, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x48, 0x69, 0x73, 0x74, + 0x6f, 0x72, 0x79, 0x12, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x48, 0x69, + 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, - 0x73, 0x12, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x42, 0x79, - 0x74, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0c, 0x43, 0x6c, - 0x65, 0x61, 0x72, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x17, 0x2e, 0x76, 0x31, 0x2e, - 0x43, 0x6c, 0x65, 0x61, 0x72, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3b, 0x0a, - 0x10, 0x50, 0x61, 0x74, 0x68, 0x41, 0x75, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, - 0x65, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x11, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, - 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, - 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, 0x67, - 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x10, 0x50, 0x61, 0x74, 0x68, 0x41, 0x75, + 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, + 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x11, + 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, + 0x74, 0x22, 0x00, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62, + 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76, + 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/internal/api/backresthandler.go b/internal/api/backresthandler.go index e45f1021..4a936f45 100644 --- a/internal/api/backresthandler.go +++ b/internal/api/backresthandler.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "path" - "sync" "time" "connectrpc.com/connect" @@ -17,6 +16,8 @@ import ( "github.com/garethgeorge/backrest/internal/oplog" "github.com/garethgeorge/backrest/internal/oplog/indexutil" "github.com/garethgeorge/backrest/internal/orchestrator" + "github.com/garethgeorge/backrest/internal/orchestrator/repo" + "github.com/garethgeorge/backrest/internal/orchestrator/tasks" "github.com/garethgeorge/backrest/internal/protoutil" "github.com/garethgeorge/backrest/internal/resticinstaller" "github.com/garethgeorge/backrest/internal/rotatinglog" @@ -107,7 +108,7 @@ func (s *BackrestHandler) AddRepo(ctx context.Context, req *connect.Request[v1.R return nil, fmt.Errorf("failed to find or install restic binary: %w", err) } - r, err := orchestrator.NewRepoOrchestrator(req.Msg, bin) + r, err := repo.NewRepoOrchestrator(req.Msg, bin) if err != nil { return nil, fmt.Errorf("failed to configure repo: %w", err) } @@ -127,7 +128,7 @@ func (s *BackrestHandler) AddRepo(ctx context.Context, req *connect.Request[v1.R // index snapshots for the newly added repository. zap.L().Debug("scheduling index snapshots task") - s.orchestrator.ScheduleTask(orchestrator.NewOneoffIndexSnapshotsTask(s.orchestrator, req.Msg.Id, time.Now()), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityIndexSnapshots) + s.orchestrator.ScheduleTask(tasks.NewOneoffIndexSnapshotsTask(req.Msg.Id, time.Now()), tasks.TaskPriorityInteractive+tasks.TaskPriorityIndexSnapshots) zap.L().Debug("done add repo") return connect.NewResponse(c), nil @@ -136,7 +137,7 @@ func (s *BackrestHandler) AddRepo(ctx context.Context, req *connect.Request[v1.R // ListSnapshots implements POST /v1/snapshots func (s *BackrestHandler) ListSnapshots(ctx context.Context, req *connect.Request[v1.ListSnapshotsRequest]) (*connect.Response[v1.ResticSnapshotList], error) { query := req.Msg - repo, err := s.orchestrator.GetRepo(query.RepoId) + repo, err := s.orchestrator.GetRepoOrchestrator(query.RepoId) if err != nil { return nil, fmt.Errorf("failed to get repo: %w", err) } @@ -170,7 +171,7 @@ func (s *BackrestHandler) ListSnapshots(ctx context.Context, req *connect.Reques func (s *BackrestHandler) ListSnapshotFiles(ctx context.Context, req *connect.Request[v1.ListSnapshotFilesRequest]) (*connect.Response[v1.ListSnapshotFilesResponse], error) { query := req.Msg - repo, err := s.orchestrator.GetRepo(query.RepoId) + repo, err := s.orchestrator.GetRepoOrchestrator(query.RepoId) if err != nil { return nil, fmt.Errorf("failed to get repo: %w", err) } @@ -258,6 +259,8 @@ func (s *BackrestHandler) GetOperations(ctx context.Context, req *connect.Reques err = s.oplog.ForEachByRepo(req.Msg.RepoId, idCollector, opCollector) } else if req.Msg.SnapshotId != "" { err = s.oplog.ForEachBySnapshotId(req.Msg.SnapshotId, idCollector, opCollector) + } else if req.Msg.FlowId != 0 { + err = s.oplog.ForEachByFlowId(req.Msg.FlowId, idCollector, opCollector) } else if len(req.Msg.Ids) > 0 { ops = make([]*v1.Operation, 0, len(req.Msg.Ids)) for i, id := range req.Msg.Ids { @@ -285,7 +288,7 @@ func (s *BackrestHandler) IndexSnapshots(ctx context.Context, req *connect.Reque return nil, fmt.Errorf("failed to get repo %q: %w", req.Msg.Value, err) } - s.orchestrator.ScheduleTask(orchestrator.NewOneoffIndexSnapshotsTask(s.orchestrator, req.Msg.Value, time.Now()), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityIndexSnapshots) + s.orchestrator.ScheduleTask(tasks.NewOneoffIndexSnapshotsTask(req.Msg.Value, time.Now()), tasks.TaskPriorityInteractive+tasks.TaskPriorityIndexSnapshots) return connect.NewResponse(&emptypb.Empty{}), nil } @@ -295,38 +298,36 @@ func (s *BackrestHandler) Backup(ctx context.Context, req *connect.Request[types if err != nil { return nil, fmt.Errorf("failed to get plan %q: %w", req.Msg.Value, err) } - var wg sync.WaitGroup - wg.Add(1) - s.orchestrator.ScheduleTask(orchestrator.NewOneoffBackupTask(s.orchestrator, plan, time.Now()), orchestrator.TaskPriorityInteractive, func(e error) { + wait := make(chan struct{}) + s.orchestrator.ScheduleTask(tasks.NewOneoffBackupTask(plan, time.Now()), tasks.TaskPriorityInteractive, func(e error) { err = e - wg.Done() + close(wait) }) - wg.Wait() + <-wait return connect.NewResponse(&emptypb.Empty{}), err } func (s *BackrestHandler) Forget(ctx context.Context, req *connect.Request[v1.ForgetRequest]) (*connect.Response[emptypb.Empty], error) { at := time.Now() - var err error + _, err := s.orchestrator.GetPlan(req.Msg.PlanId) + if err != nil { + return nil, fmt.Errorf("failed to get plan %q: %w", req.Msg.PlanId, err) + } + if req.Msg.SnapshotId != "" && req.Msg.PlanId != "" && req.Msg.RepoId != "" { wait := make(chan struct{}) s.orchestrator.ScheduleTask( - orchestrator.NewOneoffForgetSnapshotTask(s.orchestrator, req.Msg.RepoId, req.Msg.PlanId, req.Msg.SnapshotId, at), - orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityForget, func(e error) { + tasks.NewOneoffForgetSnapshotTask(req.Msg.RepoId, req.Msg.PlanId, 0, at, req.Msg.SnapshotId), + tasks.TaskPriorityInteractive+tasks.TaskPriorityForget, func(e error) { err = e close(wait) }) <-wait } else if req.Msg.RepoId != "" && req.Msg.PlanId != "" { - plan, err := s.orchestrator.GetPlan(req.Msg.PlanId) - if err != nil { - return nil, fmt.Errorf("failed to get plan %q: %w", req.Msg.PlanId, err) - } - wait := make(chan struct{}) s.orchestrator.ScheduleTask( - orchestrator.NewOneoffForgetTask(s.orchestrator, plan, "", at), - orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityForget, func(e error) { + tasks.NewOneoffForgetTask(req.Msg.RepoId, req.Msg.PlanId, 0, at), + tasks.TaskPriorityInteractive+tasks.TaskPriorityForget, func(e error) { err = e close(wait) }) @@ -348,13 +349,12 @@ func (s *BackrestHandler) Prune(ctx context.Context, req *connect.Request[types. } at := time.Now() - var wg sync.WaitGroup - wg.Add(1) - s.orchestrator.ScheduleTask(orchestrator.NewOneoffPruneTask(s.orchestrator, plan, at, true), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityPrune, func(e error) { + wait := make(chan struct{}) + s.orchestrator.ScheduleTask(tasks.NewOneoffPruneTask(plan.Repo, plan.Id, 0, at, true), tasks.TaskPriorityInteractive+tasks.TaskPriorityPrune, func(e error) { err = e - wg.Done() + close(wait) }) - wg.Wait() + <-wait return connect.NewResponse(&emptypb.Empty{}), nil } @@ -375,19 +375,17 @@ func (s *BackrestHandler) Restore(ctx context.Context, req *connect.Request[v1.R at := time.Now() - s.orchestrator.ScheduleTask(orchestrator.NewOneoffRestoreTask(s.orchestrator, orchestrator.RestoreTaskOpts{ - RepoId: req.Msg.RepoId, - PlanId: req.Msg.PlanId, - SnapshotId: req.Msg.SnapshotId, - Path: req.Msg.Path, - Target: target, - }, at), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityDefault) + flowID, err := tasks.FlowIDForSnapshotID(s.oplog, req.Msg.SnapshotId) + if err != nil { + return nil, fmt.Errorf("failed to get flow ID for snapshot %q: %w", req.Msg.SnapshotId, err) + } + s.orchestrator.ScheduleTask(tasks.NewOneoffRestoreTask(req.Msg.RepoId, req.Msg.PlanId, flowID, at, req.Msg.SnapshotId, req.Msg.Path, target), tasks.TaskPriorityInteractive+tasks.TaskPriorityDefault) return connect.NewResponse(&emptypb.Empty{}), nil } func (s *BackrestHandler) Unlock(ctx context.Context, req *connect.Request[types.StringValue]) (*connect.Response[emptypb.Empty], error) { - repo, err := s.orchestrator.GetRepo(req.Msg.Value) + repo, err := s.orchestrator.GetRepoOrchestrator(req.Msg.Value) if err != nil { return nil, fmt.Errorf("failed to get repo %q: %w", req.Msg.Value, err) } @@ -402,13 +400,12 @@ func (s *BackrestHandler) Unlock(ctx context.Context, req *connect.Request[types func (s *BackrestHandler) Stats(ctx context.Context, req *connect.Request[types.StringValue]) (*connect.Response[emptypb.Empty], error) { at := time.Now() var err error - var wg sync.WaitGroup - wg.Add(1) - s.orchestrator.ScheduleTask(orchestrator.NewOneoffStatsTask(s.orchestrator, req.Msg.Value, orchestrator.PlanForUnassociatedOperations, at), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityStats, func(e error) { + wait := make(chan struct{}) + s.orchestrator.ScheduleTask(tasks.NewOneoffStatsTask(req.Msg.Value, tasks.PlanForUnassociatedOperations, at), tasks.TaskPriorityInteractive+tasks.TaskPriorityStats, func(e error) { err = e - wg.Done() + close(wait) }) - wg.Wait() + <-wait return connect.NewResponse(&emptypb.Empty{}), err } diff --git a/internal/api/backresthandler_test.go b/internal/api/backresthandler_test.go index ad5152f8..b6aafdf8 100644 --- a/internal/api/backresthandler_test.go +++ b/internal/api/backresthandler_test.go @@ -134,15 +134,14 @@ func TestBackup(t *testing.T) { } // Wait for the index snapshot operation to appear in the oplog. - var snapshotId string + var snapshotOp *v1.Operation if err := retry(t, 10, 2*time.Second, func() error { operations := getOperations(t, sut.oplog) if index := slices.IndexFunc(operations, func(op *v1.Operation) bool { _, ok := op.GetOp().(*v1.Operation_OperationIndexSnapshot) return op.Status == v1.OperationStatus_STATUS_SUCCESS && ok }); index != -1 { - op := operations[index] - snapshotId = op.SnapshotId + snapshotOp = operations[index] return nil } return errors.New("snapshot not indexed") @@ -150,7 +149,7 @@ func TestBackup(t *testing.T) { t.Fatalf("Couldn't find snapshot in oplog") } - if snapshotId == "" { + if snapshotOp.SnapshotId == "" { t.Fatalf("snapshotId must be set") } @@ -162,8 +161,8 @@ func TestBackup(t *testing.T) { return op.Status == v1.OperationStatus_STATUS_SUCCESS && ok }); index != -1 { op := operations[index] - if op.SnapshotId != snapshotId { - t.Fatalf("Snapshot ID mismatch on forget operation") + if op.FlowId != snapshotOp.FlowId { + t.Fatalf("Flow ID mismatch on forget operation") } return nil } @@ -460,15 +459,14 @@ func TestRestore(t *testing.T) { } // Wait for the index snapshot operation to appear in the oplog. - var snapshotId string + var snapshotOp *v1.Operation if err := retry(t, 10, 2*time.Second, func() error { operations := getOperations(t, sut.oplog) if index := slices.IndexFunc(operations, func(op *v1.Operation) bool { _, ok := op.GetOp().(*v1.Operation_OperationIndexSnapshot) return op.Status == v1.OperationStatus_STATUS_SUCCESS && ok }); index != -1 { - op := operations[index] - snapshotId = op.SnapshotId + snapshotOp = operations[index] return nil } return errors.New("snapshot not indexed") @@ -476,14 +474,14 @@ func TestRestore(t *testing.T) { t.Fatalf("Couldn't find snapshot in oplog") } - if snapshotId == "" { + if snapshotOp.SnapshotId == "" { t.Fatalf("snapshotId must be set") } restoreTarget := t.TempDir() _, err = sut.handler.Restore(context.Background(), connect.NewRequest(&v1.RestoreSnapshotRequest{ - SnapshotId: snapshotId, + SnapshotId: snapshotOp.SnapshotId, PlanId: "test", RepoId: "local", Target: restoreTarget, @@ -500,8 +498,11 @@ func TestRestore(t *testing.T) { return op.Status == v1.OperationStatus_STATUS_SUCCESS && ok }); index != -1 { op := operations[index] - if op.SnapshotId != snapshotId { - t.Fatalf("Snapshot ID mismatch on restore operation") + if op.FlowId != snapshotOp.FlowId { + t.Errorf("Flow ID mismatch on restore operation") + } + if op.SnapshotId != snapshotOp.SnapshotId { + t.Errorf("Snapshot ID mismatch on restore operation") } return nil } diff --git a/internal/hook/hook.go b/internal/hook/hook.go index fbd37381..663cf72b 100644 --- a/internal/hook/hook.go +++ b/internal/hook/hook.go @@ -35,11 +35,12 @@ func NewHookExecutor(oplog *oplog.OpLog, bigOutputStore *rotatinglog.RotatingLog // ExecuteHooks schedules tasks for the hooks subscribed to the given event. The vars map is used to substitute variables // Hooks are pulled both from the provided plan and from the repo config. -func (e *HookExecutor) ExecuteHooks(repo *v1.Repo, plan *v1.Plan, events []v1.Hook_Condition, vars HookVars) error { +func (e *HookExecutor) ExecuteHooks(flowID int64, repo *v1.Repo, plan *v1.Plan, events []v1.Hook_Condition, vars HookVars) error { operationBase := v1.Operation{ Status: v1.OperationStatus_STATUS_INPROGRESS, PlanId: plan.GetId(), RepoId: repo.GetId(), + FlowId: flowID, } vars.Repo = repo @@ -59,10 +60,11 @@ func (e *HookExecutor) ExecuteHooks(repo *v1.Repo, plan *v1.Plan, events []v1.Ho operation.UnixTimeStartMs = curTimeMs() operation.Op = &v1.Operation_OperationRunHook{ OperationRunHook: &v1.OperationRunHook{ - Name: name, + Name: name, + Condition: event, }, } - zap.L().Info("running hook", zap.String("plan", plan.Id), zap.Int64("opId", operation.Id), zap.String("hook", name)) + zap.L().Info("running hook", zap.String("plan", repo.Id), zap.Int64("opId", operation.Id), zap.String("hook", name)) if err := e.executeHook(operation, h, event, vars); err != nil { zap.S().Errorf("error on repo hook %v on condition %v: %v", idx, event.String(), err) if isHaltingError(err) { @@ -84,7 +86,8 @@ func (e *HookExecutor) ExecuteHooks(repo *v1.Repo, plan *v1.Plan, events []v1.Ho operation.UnixTimeStartMs = curTimeMs() operation.Op = &v1.Operation_OperationRunHook{ OperationRunHook: &v1.OperationRunHook{ - Name: name, + Name: name, + Condition: event, }, } diff --git a/internal/ioutil/ioutil.go b/internal/ioutil/ioutil.go index 939441e2..ca9ace2f 100644 --- a/internal/ioutil/ioutil.go +++ b/internal/ioutil/ioutil.go @@ -110,3 +110,16 @@ func (w *OutputCapturer) Bytes() []byte { return buf.Bytes() } + +type SynchronizedWriter struct { + mu sync.Mutex + W io.Writer +} + +var _ io.Writer = &SynchronizedWriter{} + +func (w *SynchronizedWriter) Write(p []byte) (n int, err error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.W.Write(p) +} diff --git a/internal/oplog/migrations.go b/internal/oplog/migrations.go new file mode 100644 index 00000000..ef1e298e --- /dev/null +++ b/internal/oplog/migrations.go @@ -0,0 +1,110 @@ +package oplog + +import ( + "errors" + "fmt" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/oplog/serializationutil" + "go.etcd.io/bbolt" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" +) + +var migrations = []func(*OpLog, *bbolt.Tx) error{ + migration001FlowID, +} + +var CurrentVersion = int64(len(migrations)) + +func ApplyMigrations(oplog *OpLog, tx *bbolt.Tx) error { + var version int64 + versionBytes := tx.Bucket(SystemBucket).Get([]byte("version")) + if versionBytes == nil { + version = 0 + } else { + v, err := serializationutil.Btoi(versionBytes) + if err != nil { + return fmt.Errorf("couldn't parse version: %w", err) + } + version = v + } + + startMigration := int(version) + if startMigration < 0 { + startMigration = 0 + } + for idx := startMigration; idx < len(migrations); idx += 1 { + zap.L().Info("applying oplog migration", zap.Int("migration_no", idx)) + if err := migrations[idx](oplog, tx); err != nil { + zap.L().Error("failed to apply migration", zap.Int("migration_no", idx), zap.Error(err)) + return fmt.Errorf("couldn't apply migration %d: %w", idx, err) + } + } + + if err := tx.Bucket(SystemBucket).Put([]byte("version"), serializationutil.Itob(CurrentVersion)); err != nil { + return fmt.Errorf("couldn't update version: %w", err) + } + return nil +} + +func transformOperations(oplog *OpLog, tx *bbolt.Tx, f func(op *v1.Operation) error) error { + opLogBucket := tx.Bucket(OpLogBucket) + + if opLogBucket == nil { + return errors.New("oplog bucket not found") + } + + c := opLogBucket.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + op := &v1.Operation{} + if err := proto.Unmarshal(v, op); err != nil { + return fmt.Errorf("unmarshal operation: %w", err) + } + + copy := proto.Clone(op).(*v1.Operation) + err := f(copy) + if err != nil { + return err + } + + if proto.Equal(copy, op) { + continue + } + + if _, err := oplog.deleteOperationHelper(tx, op.Id); err != nil { + return fmt.Errorf("delete operation: %w", err) + } + if err := oplog.addOperationHelper(tx, copy); err != nil { + return fmt.Errorf("create operation: %w", err) + } + } + + return nil +} + +// migration001FlowID sets the flow ID for operations that are missing it. +// All operations with the same snapshot ID will have the same flow ID. +func migration001FlowID(oplog *OpLog, tx *bbolt.Tx) error { + snapshotIdToFlow := make(map[string]int64) + + return transformOperations(oplog, tx, func(op *v1.Operation) error { + if op.FlowId != 0 { + return nil + } + + if op.SnapshotId == "" { + op.FlowId = op.Id + return nil + } + + if flowId, ok := snapshotIdToFlow[op.SnapshotId]; ok { + op.FlowId = flowId + } else { + snapshotIdToFlow[op.SnapshotId] = op.Id + op.FlowId = op.Id + } + + return nil + }) +} diff --git a/internal/oplog/oplog.go b/internal/oplog/oplog.go index 0a74ceee..c1a92038 100644 --- a/internal/oplog/oplog.go +++ b/internal/oplog/oplog.go @@ -34,6 +34,7 @@ var ( OpLogBucket = []byte("oplog.log") // oplog stores existant operations. RepoIndexBucket = []byte("oplog.repo_idx") // repo_index tracks IDs of operations affecting a given repo PlanIndexBucket = []byte("oplog.plan_idx") // plan_index tracks IDs of operations affecting a given plan + FlowIdIndexBucket = []byte("oplog.flow_id_idx") // flow_id_index tracks IDs of operations affecting a given flow SnapshotIndexBucket = []byte("oplog.snapshot_idx") // snapshot_index tracks IDs of operations affecting a given snapshot ) @@ -63,13 +64,17 @@ func NewOpLog(databasePath string) (*OpLog, error) { if err := db.Update(func(tx *bolt.Tx) error { // Create the buckets if they don't exist for _, bucket := range [][]byte{ - SystemBucket, OpLogBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket, + SystemBucket, OpLogBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket, FlowIdIndexBucket, } { if _, err := tx.CreateBucketIfNotExists(bucket); err != nil { return fmt.Errorf("creating bucket %s: %s", string(bucket), err) } } + if err := ApplyMigrations(o, tx); err != nil { + return fmt.Errorf("applying migrations: %w", err) + } + return nil }); err != nil { return nil, err @@ -236,7 +241,7 @@ func (o *OpLog) getOperationHelper(b *bolt.Bucket, id int64) (*v1.Operation, err return &op, nil } -func (o *OpLog) nextOperationId(b *bolt.Bucket, unixTimeMs int64) (int64, error) { +func (o *OpLog) nextID(b *bolt.Bucket, unixTimeMs int64) (int64, error) { seq, err := b.NextSequence() if err != nil { return 0, fmt.Errorf("next sequence: %w", err) @@ -248,12 +253,16 @@ func (o *OpLog) addOperationHelper(tx *bolt.Tx, op *v1.Operation) error { b := tx.Bucket(OpLogBucket) if op.Id == 0 { var err error - op.Id, err = o.nextOperationId(b, time.Now().UnixMilli()) + op.Id, err = o.nextID(b, time.Now().UnixMilli()) if err != nil { return fmt.Errorf("create next operation ID: %w", err) } } + if op.FlowId == 0 { + op.FlowId = op.Id + } + if err := protoutil.ValidateOperation(op); err != nil { return fmt.Errorf("validating operation: %w", err) } @@ -283,6 +292,11 @@ func (o *OpLog) addOperationHelper(tx *bolt.Tx, op *v1.Operation) error { return fmt.Errorf("error adding operation to snapshot index: %w", err) } } + if op.FlowId != 0 { + if err := indexutil.IndexByteValue(tx.Bucket(FlowIdIndexBucket), serializationutil.Itob(op.FlowId), op.Id); err != nil { + return fmt.Errorf("error adding operation to flow index: %w", err) + } + } return nil } @@ -313,6 +327,12 @@ func (o *OpLog) deleteOperationHelper(tx *bolt.Tx, id int64) (*v1.Operation, err } } + if prevValue.FlowId != 0 { + if err := indexutil.IndexRemoveByteValue(tx.Bucket(FlowIdIndexBucket), serializationutil.Itob(prevValue.FlowId), id); err != nil { + return nil, fmt.Errorf("removing operation %v from flow index: %w", id, err) + } + } + if err := b.Delete(serializationutil.Itob(id)); err != nil { return nil, fmt.Errorf("deleting operation %v from bucket: %w", id, err) } @@ -356,6 +376,13 @@ func (o *OpLog) ForEachBySnapshotId(snapshotId string, collector indexutil.Colle }) } +func (o *OpLog) ForEachByFlowId(flowId int64, collector indexutil.Collector, do func(op *v1.Operation) error) error { + return o.db.View(func(tx *bolt.Tx) error { + ids := collector(indexutil.IndexSearchByteValue(tx.Bucket(FlowIdIndexBucket), serializationutil.Itob(flowId))) + return o.forOpsByIds(tx, ids, do) + }) +} + func (o *OpLog) forOpsByIds(tx *bolt.Tx, ids []int64, do func(*v1.Operation) error) error { b := tx.Bucket(OpLogBucket) for _, id := range ids { diff --git a/internal/oplog/oplog_test.go b/internal/oplog/oplog_test.go index 02e1cddc..08c1d089 100644 --- a/internal/oplog/oplog_test.go +++ b/internal/oplog/oplog_test.go @@ -215,6 +215,42 @@ func TestListOperation(t *testing.T) { } } +func TestListByFlowId(t *testing.T) { + t.Parallel() + + log, err := NewOpLog(t.TempDir() + "/test.boltdb") + if err != nil { + t.Fatalf("error creating oplog: %s", err) + } + t.Cleanup(func() { log.Close() }) + + op := &v1.Operation{ + UnixTimeStartMs: 1234, + PlanId: "plan1", + RepoId: "repo1", + FlowId: 1, + Op: &v1.Operation_OperationBackup{}, + } + + if err := log.Add(op); err != nil { + t.Fatalf("error adding operation: %s", err) + } + + var ops []*v1.Operation + if err := log.ForEachByFlowId(1, indexutil.CollectAll(), func(op *v1.Operation) error { + ops = append(ops, op) + return nil + }); err != nil { + t.Fatalf("error listing operations: %s", err) + } + if len(ops) != 1 { + t.Fatalf("want 1 operation, got %d", len(ops)) + } + if ops[0].Id != op.Id { + t.Errorf("want operation ID %d, got %d", op.Id, ops[0].Id) + } +} + func TestBigIO(t *testing.T) { t.Parallel() diff --git a/internal/orchestrator/logging/logging.go b/internal/orchestrator/logging/logging.go new file mode 100644 index 00000000..12ede808 --- /dev/null +++ b/internal/orchestrator/logging/logging.go @@ -0,0 +1,24 @@ +package logging + +import ( + "context" + "io" +) + +type contextKey int + +const ( + contextKeyLogWriter contextKey = iota +) + +func WriterFromContext(ctx context.Context) io.Writer { + writer, ok := ctx.Value(contextKeyLogWriter).(io.Writer) + if !ok { + return nil + } + return writer +} + +func ContextWithWriter(ctx context.Context, logger io.Writer) context.Context { + return context.WithValue(ctx, contextKeyLogWriter, logger) +} diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go index bff51b30..6d4ce35d 100644 --- a/internal/orchestrator/orchestrator.go +++ b/internal/orchestrator/orchestrator.go @@ -1,18 +1,22 @@ package orchestrator import ( + "bytes" "context" "errors" "fmt" "slices" "sync" - "sync/atomic" "time" v1 "github.com/garethgeorge/backrest/gen/go/v1" "github.com/garethgeorge/backrest/internal/config" "github.com/garethgeorge/backrest/internal/hook" + "github.com/garethgeorge/backrest/internal/ioutil" "github.com/garethgeorge/backrest/internal/oplog" + "github.com/garethgeorge/backrest/internal/orchestrator/logging" + "github.com/garethgeorge/backrest/internal/orchestrator/repo" + "github.com/garethgeorge/backrest/internal/orchestrator/tasks" "github.com/garethgeorge/backrest/internal/queue" "github.com/garethgeorge/backrest/internal/rotatinglog" "go.uber.org/zap" @@ -23,32 +27,35 @@ var ErrRepoNotFound = errors.New("repo not found") var ErrRepoInitializationFailed = errors.New("repo initialization failed") var ErrPlanNotFound = errors.New("plan not found") -const PlanForUnassociatedOperations = "_unassociated_" - -const ( - TaskPriorityDefault = 0 - TaskPriorityInteractive = 10 - TaskPriorityIndexSnapshots = 101 - TaskPriorityForget = 102 - TaskPriorityPrune = 103 - TaskPriorityHook = 1000 // runs before any other task. - TaskPriorityStats = -1 // very low priority. -) - // Orchestrator is responsible for managing repos and backups. type Orchestrator struct { mu sync.Mutex config *v1.Config OpLog *oplog.OpLog repoPool *resticRepoPool - taskQueue *queue.TimePriorityQueue[scheduledTask] + taskQueue *queue.TimePriorityQueue[stContainer] hookExecutor *hook.HookExecutor logStore *rotatinglog.RotatingLog + // cancelNotify is a list of channels that are notified when a task should be cancelled. + cancelNotify []chan int64 + // now for the purpose of testing; used by Run() to get the current time. now func() time.Time +} - runningTask atomic.Pointer[taskExecutionInfo] +type stContainer struct { + tasks.ScheduledTask + configModno int32 + callbacks []func(error) +} + +func (st stContainer) Eq(other stContainer) bool { + return st.ScheduledTask.Eq(other.ScheduledTask) +} + +func (st stContainer) Less(other stContainer) bool { + return st.ScheduledTask.Less(other.ScheduledTask) } func NewOrchestrator(resticBin string, cfg *v1.Config, oplog *oplog.OpLog, logStore *rotatinglog.RotatingLog) (*Orchestrator, error) { @@ -61,7 +68,7 @@ func NewOrchestrator(resticBin string, cfg *v1.Config, oplog *oplog.OpLog, logSt config: cfg, // repoPool created with a memory store to ensure the config is updated in an atomic operation with the repo pool's config value. repoPool: newResticRepoPool(resticBin, &config.MemoryStore{Config: cfg}), - taskQueue: queue.NewTimePriorityQueue[scheduledTask](), + taskQueue: queue.NewTimePriorityQueue[stContainer](), hookExecutor: hook.NewHookExecutor(oplog, logStore), logStore: logStore, } @@ -81,7 +88,7 @@ func NewOrchestrator(resticBin string, cfg *v1.Config, oplog *oplog.OpLog, logSt } for _, repoId := range incompleteOpRepos { - repo, err := o.GetRepo(repoId) + repo, err := o.GetRepoOrchestrator(repoId) if err != nil { if errors.Is(err, ErrRepoNotFound) { zap.L().Warn("repo not found for incomplete operation. Possibly just deleted.", zap.String("repo", repoId)) @@ -100,6 +107,8 @@ func NewOrchestrator(resticBin string, cfg *v1.Config, oplog *oplog.OpLog, logSt return nil, fmt.Errorf("apply initial config: %w", err) } + zap.L().Info("orchestrator created") + return o, nil } @@ -112,14 +121,14 @@ func (o *Orchestrator) curTime() time.Time { func (o *Orchestrator) ApplyConfig(cfg *v1.Config) error { o.mu.Lock() - defer o.mu.Unlock() o.config = cfg // Update the config provided to the repo pool which is cached and diffed separately. if err := o.repoPool.configProvider.Update(cfg); err != nil { + o.mu.Unlock() return fmt.Errorf("failed to update repo pool config: %w", err) } - + o.mu.Unlock() return o.ScheduleDefaultTasks(cfg) } @@ -128,35 +137,41 @@ func (o *Orchestrator) ScheduleDefaultTasks(config *v1.Config) error { zap.L().Info("scheduling default tasks, waiting for task queue reset.") removedTasks := o.taskQueue.Reset() for _, t := range removedTasks { - if err := t.task.Cancel(v1.OperationStatus_STATUS_SYSTEM_CANCELLED); err != nil { - zap.L().Error("failed to cancel queued task", zap.String("task", t.task.Name()), zap.Error(err)) + if t.Op == nil { + continue + } + + if err := o.cancelHelper(t.Op, v1.OperationStatus_STATUS_SYSTEM_CANCELLED); err != nil { + zap.L().Error("failed to cancel queued task", zap.String("task", t.Task.Name()), zap.Error(err)) } else { - zap.L().Debug("queued task cancelled due to config change", zap.String("task", t.task.Name())) + zap.L().Debug("queued task cancelled due to config change", zap.String("task", t.Task.Name())) } } zap.L().Info("reset task queue, scheduling new task set.") // Requeue tasks that are affected by the config change. - o.ScheduleTask(&CollectGarbageTask{ - orchestrator: o, - }, TaskPriorityDefault) + if err := o.ScheduleTask(tasks.NewCollectGarbageTask(), tasks.TaskPriorityDefault); err != nil { + return fmt.Errorf("schedule collect garbage task: %w", err) + } for _, plan := range config.Plans { if plan.Disabled { continue } - t, err := NewScheduledBackupTask(o, plan) + t, err := tasks.NewScheduledBackupTask(plan) if err != nil { return fmt.Errorf("schedule backup task for plan %q: %w", plan.Id, err) } - o.ScheduleTask(t, TaskPriorityDefault) + if err := o.ScheduleTask(t, tasks.TaskPriorityDefault); err != nil { + return fmt.Errorf("schedule backup task for plan %q: %w", plan.Id, err) + } } return nil } -func (o *Orchestrator) GetRepo(repoId string) (repo *RepoOrchestrator, err error) { +func (o *Orchestrator) GetRepoOrchestrator(repoId string) (repo *repo.RepoOrchestrator, err error) { o.mu.Lock() defer o.mu.Unlock() @@ -167,130 +182,245 @@ func (o *Orchestrator) GetRepo(repoId string) (repo *RepoOrchestrator, err error return r, nil } -func (o *Orchestrator) GetPlan(planId string) (*v1.Plan, error) { +func (o *Orchestrator) GetRepo(repoId string) (*v1.Repo, error) { o.mu.Lock() defer o.mu.Unlock() - if o.config.Plans == nil { - return nil, ErrPlanNotFound + for _, r := range o.config.Repos { + if r.GetId() == repoId { + return r, nil + } } + return nil, fmt.Errorf("get repo %q: %w", repoId, ErrRepoNotFound) +} + +func (o *Orchestrator) GetPlan(planId string) (*v1.Plan, error) { + o.mu.Lock() + defer o.mu.Unlock() + for _, p := range o.config.Plans { if p.Id == planId { return p, nil } } - return nil, ErrPlanNotFound + return nil, fmt.Errorf("get plan %q: %w", planId, ErrPlanNotFound) } func (o *Orchestrator) CancelOperation(operationId int64, status v1.OperationStatus) error { o.mu.Lock() - defer o.mu.Unlock() - - // note: if the task is running the requested status will not be set. - if running := o.runningTask.Load(); running != nil && running.operationId == operationId { - running.cancel() + for _, c := range o.cancelNotify { + select { + case c <- operationId: + default: + } } + o.mu.Unlock() allTasks := o.taskQueue.GetAll() - idx := slices.IndexFunc(allTasks, func(t scheduledTask) bool { - return t.task.OperationId() == operationId + idx := slices.IndexFunc(allTasks, func(t stContainer) bool { + return t.Op != nil && t.Op.GetId() == operationId }) if idx == -1 { return nil } - t := allTasks[idx] - o.taskQueue.Remove(t) - if err := t.task.Cancel(status); err != nil { - return fmt.Errorf("cancel task %q: %w", t.task.Name(), err) + + if err := o.cancelHelper(t.Op, status); err != nil { + return fmt.Errorf("cancel operation: %w", err) } - if nextTime := t.task.Next(t.runAt.Add(1 * time.Second)); nextTime != nil { - t.runAt = *nextTime - o.taskQueue.Enqueue(*nextTime, t.priority, t) + o.taskQueue.Remove(t) + + if err := o.scheduleTaskHelper(t.Task, tasks.TaskPriorityDefault, t.RunAt); err != nil { + return fmt.Errorf("reschedule cancelled task: %w", err) + } + return nil +} + +func (o *Orchestrator) cancelHelper(op *v1.Operation, status v1.OperationStatus) error { + op.Status = status + op.UnixTimeEndMs = time.Now().UnixMilli() + if err := o.OpLog.Update(op); err != nil { + return fmt.Errorf("update cancelled operation: %w", err) } return nil } // Run is the main orchestration loop. Cancel the context to stop the loop. -func (o *Orchestrator) Run(mainCtx context.Context) { +func (o *Orchestrator) Run(ctx context.Context) { zap.L().Info("starting orchestrator loop") + // subscribe to cancel notifications. + o.mu.Lock() + cancelNotifyChan := make(chan int64, 10) // buffered to queue up cancel notifications. + o.cancelNotify = append(o.cancelNotify, cancelNotifyChan) + o.mu.Unlock() + + defer func() { + o.mu.Lock() + if idx := slices.Index(o.cancelNotify, cancelNotifyChan); idx != -1 { + o.cancelNotify = slices.Delete(o.cancelNotify, idx, idx+1) + } + o.mu.Unlock() + }() + for { - if mainCtx.Err() != nil { + if ctx.Err() != nil { zap.L().Info("shutting down orchestrator loop, context cancelled.") break } - t := o.taskQueue.Dequeue(mainCtx) - if t.task == nil { + t := o.taskQueue.Dequeue(ctx) + if t.Task == nil { continue } - zap.L().Info("running task", zap.String("task", t.task.Name())) + zap.L().Info("running task", zap.String("task", t.Task.Name())) - taskCtx, cancel := context.WithCancel(mainCtx) + logs := bytes.NewBuffer(nil) + taskCtx, cancelTaskCtx := context.WithCancel(ctx) + taskCtx = logging.ContextWithWriter(taskCtx, &ioutil.SynchronizedWriter{W: logs}) - if swapped := o.runningTask.CompareAndSwap(nil, &taskExecutionInfo{ - operationId: t.task.OperationId(), - cancel: cancel, - }); !swapped { - zap.L().Fatal("failed to start task, another task is already running. Was Run() called twice?") - } + go func() { + for { + select { + case <-taskCtx.Done(): + return + case opID := <-cancelNotifyChan: + if t.Op != nil && opID == t.Op.GetId() { + cancelTaskCtx() + } + } + } + }() start := time.Now() - err := t.task.Run(taskCtx) + runner := newTaskRunnerImpl(o, t.Task, t.Op) + + op := t.Op + if op != nil { + op.UnixTimeStartMs = time.Now().UnixMilli() + if op.Status == v1.OperationStatus_STATUS_PENDING || op.Status == v1.OperationStatus_STATUS_UNKNOWN { + op.Status = v1.OperationStatus_STATUS_INPROGRESS + } + if op.Id != 0 { + if err := o.OpLog.Update(op); err != nil { + zap.S().Errorf("failed to add operation to oplog: %w", err) + } + } else { + if err := o.OpLog.Add(op); err != nil { + zap.S().Errorf("failed to add operation to oplog: %w", err) + } + } + } + + err := t.Task.Run(taskCtx, t.ScheduledTask, runner) + + if op != nil { + // write logs to log storage for this task. + if logs.Len() > 0 { + ref, err := o.logStore.Write(logs.Bytes()) + if err != nil { + zap.S().Errorf("failed to write logs for task %q to log store: %v", t.Task.Name(), err) + } else { + op.Logref = ref + } + } + + if err != nil { + if taskCtx.Err() != nil { + // task was cancelled + op.Status = v1.OperationStatus_STATUS_USER_CANCELLED + } else { + op.Status = v1.OperationStatus_STATUS_ERROR + } + op.DisplayMessage = err.Error() + } + op.UnixTimeEndMs = time.Now().UnixMilli() + if op.Status == v1.OperationStatus_STATUS_INPROGRESS { + op.Status = v1.OperationStatus_STATUS_SUCCESS + } + if e := o.OpLog.Update(op); e != nil { + zap.S().Errorf("failed to update operation in oplog: %v", e) + } + } + if err != nil { - zap.L().Error("task failed", zap.String("task", t.task.Name()), zap.Error(err), zap.Duration("duration", time.Since(start))) + zap.L().Error("task failed", zap.String("task", t.Task.Name()), zap.Error(err), zap.Duration("duration", time.Since(start))) } else { - zap.L().Info("task finished", zap.String("task", t.task.Name()), zap.Duration("duration", time.Since(start))) - } - o.runningTask.Store(nil) - - for _, cb := range t.callbacks { - cb(err) + zap.L().Info("task finished", zap.String("task", t.Task.Name()), zap.Duration("duration", time.Since(start))) } - if t.config == o.config { + o.mu.Lock() + if t.configModno == o.config.Modno { // Only reschedule tasks if the config hasn't changed since the task was scheduled. - o.ScheduleTask(t.task, t.priority) + if err := o.ScheduleTask(t.Task, tasks.TaskPriorityDefault); err != nil { + zap.L().Error("reschedule task", zap.String("task", t.Task.Name()), zap.Error(err)) + } } + o.mu.Unlock() + cancelTaskCtx() + + go func() { + for _, cb := range t.callbacks { + cb(err) + } + }() } } -func (o *Orchestrator) ScheduleTask(t Task, priority int, callbacks ...func(error)) { - nextRun := t.Next(o.curTime()) - if nextRun == nil { - return +// ScheduleTask schedules a task to run at the next available time. +// note that o.mu must not be held when calling this function. +func (o *Orchestrator) ScheduleTask(t tasks.Task, priority int, callbacks ...func(error)) error { + return o.scheduleTaskHelper(t, priority, o.curTime(), callbacks...) +} + +func (o *Orchestrator) scheduleTaskHelper(t tasks.Task, priority int, curTime time.Time, callbacks ...func(error)) error { + nextRun := t.Next(curTime, newTaskRunnerImpl(o, t, nil)) + if nextRun.Eq(tasks.NeverScheduledTask) { + return nil } - zap.L().Info("scheduling task", zap.String("task", t.Name()), zap.String("runAt", nextRun.Format(time.RFC3339))) - o.taskQueue.Enqueue(*nextRun, priority, scheduledTask{ - task: t, - runAt: *nextRun, - priority: priority, - callbacks: callbacks, - config: o.config, - }) + nextRun.Task = t + stc := stContainer{ + ScheduledTask: nextRun, + configModno: o.config.Modno, + callbacks: callbacks, + } + + if stc.Op != nil { + stc.Op.PlanId = t.PlanID() + stc.Op.RepoId = t.RepoID() + stc.Op.Status = v1.OperationStatus_STATUS_PENDING + stc.Op.UnixTimeStartMs = nextRun.RunAt.UnixMilli() + + if err := o.OpLog.Add(nextRun.Op); err != nil { + return fmt.Errorf("add operation to oplog: %w", err) + } + } + + zap.L().Info("scheduling task", zap.String("task", t.Name()), zap.String("runAt", nextRun.RunAt.Format(time.RFC3339))) + o.taskQueue.Enqueue(nextRun.RunAt, priority, stc) + return nil } // resticRepoPool caches restic repos. type resticRepoPool struct { mu sync.Mutex resticPath string - repos map[string]*RepoOrchestrator + repos map[string]*repo.RepoOrchestrator configProvider config.ConfigStore } func newResticRepoPool(resticPath string, configProvider config.ConfigStore) *resticRepoPool { return &resticRepoPool{ resticPath: resticPath, - repos: make(map[string]*RepoOrchestrator), + repos: make(map[string]*repo.RepoOrchestrator), configProvider: configProvider, } } -func (rp *resticRepoPool) GetRepo(repoId string) (repo *RepoOrchestrator, err error) { +func (rp *resticRepoPool) GetRepo(repoId string) (*repo.RepoOrchestrator, error) { cfg, err := rp.configProvider.Get() if err != nil { return nil, fmt.Errorf("failed to get config: %w", err) @@ -315,19 +445,19 @@ func (rp *resticRepoPool) GetRepo(repoId string) (repo *RepoOrchestrator, err er } // Check if we already have a repo for this id, if we do return it. - repo, ok := rp.repos[repoId] - if ok && proto.Equal(repo.repoConfig, repoProto) { - return repo, nil + r, ok := rp.repos[repoId] + if ok && proto.Equal(r.Config(), repoProto) { + return r, nil } delete(rp.repos, repoId) // Otherwise create a new repo. - repo, err = NewRepoOrchestrator(repoProto, rp.resticPath) + r, err = repo.NewRepoOrchestrator(repoProto, rp.resticPath) if err != nil { return nil, err } - rp.repos[repoId] = repo - return repo, nil + rp.repos[repoId] = r + return r, nil } type taskExecutionInfo struct { diff --git a/internal/orchestrator/orchestrator_test.go b/internal/orchestrator/orchestrator_test.go index 4b832c2f..b4756bdb 100644 --- a/internal/orchestrator/orchestrator_test.go +++ b/internal/orchestrator/orchestrator_test.go @@ -6,35 +6,44 @@ import ( "testing" "time" - v1 "github.com/garethgeorge/backrest/gen/go/v1" "github.com/garethgeorge/backrest/internal/config" + "github.com/garethgeorge/backrest/internal/orchestrator/tasks" ) type testTask struct { + tasks.BaseTask onRun func() error onNext func(curTime time.Time) *time.Time } -func (t *testTask) Name() string { - return "test" +var _ tasks.Task = &testTask{} + +func newTestTask(onRun func() error, onNext func(curTime time.Time) *time.Time) tasks.Task { + return &testTask{ + BaseTask: tasks.BaseTask{ + TaskName: "test task", + TaskRepoID: "repo", + TaskPlanID: "plan", + }, + onRun: onRun, + onNext: onNext, + } +} +func (t *testTask) Next(curTime time.Time, runner tasks.TaskRunner) tasks.ScheduledTask { + at := t.onNext(curTime) + if at == nil { + return tasks.NeverScheduledTask + } + return tasks.ScheduledTask{ + Task: t, + RunAt: *at, + } } -func (t *testTask) Next(now time.Time) *time.Time { - return t.onNext(now) -} - -func (t *testTask) Run(ctx context.Context) error { +func (t *testTask) Run(ctx context.Context, st tasks.ScheduledTask, runner tasks.TaskRunner) error { return t.onRun() } -func (t *testTask) Cancel(withStatus v1.OperationStatus) error { - return nil -} - -func (t *testTask) OperationId() int64 { - return 0 -} - func TestTaskScheduling(t *testing.T) { t.Parallel() @@ -49,17 +58,17 @@ func TestTaskScheduling(t *testing.T) { var wg sync.WaitGroup wg.Add(1) - task := &testTask{ - onRun: func() error { + task := newTestTask( + func() error { wg.Done() cancel() return nil }, - onNext: func(t time.Time) *time.Time { + func(t time.Time) *time.Time { t = t.Add(10 * time.Millisecond) return &t }, - } + ) wg.Add(1) go func() { @@ -68,7 +77,7 @@ func TestTaskScheduling(t *testing.T) { }() // Act - orch.ScheduleTask(task, TaskPriorityDefault) + orch.ScheduleTask(task, tasks.TaskPriorityDefault) // Assert passes if all tasks run and the orchestrator exists when cancelled. wg.Wait() @@ -98,22 +107,22 @@ func TestTaskRescheduling(t *testing.T) { count := 0 ranTimes := 0 - orch.ScheduleTask(&testTask{ - onNext: func(t time.Time) *time.Time { - if count < 10 { - count += 1 - return &t - } - return nil - }, - onRun: func() error { + orch.ScheduleTask(newTestTask( + func() error { ranTimes += 1 if ranTimes == 10 { cancel() } return nil }, - }, TaskPriorityDefault) + func(t time.Time) *time.Time { + if count < 10 { + count += 1 + return &t + } + return nil + }, + ), tasks.TaskPriorityDefault) wg.Wait() @@ -157,8 +166,12 @@ func TestSchedulerWait(t *testing.T) { ran := make(chan struct{}) didRun := false - orch.ScheduleTask(&testTask{ - onNext: func(t time.Time) *time.Time { + orch.ScheduleTask(newTestTask( + func() error { + close(ran) + return nil + }, + func(t time.Time) *time.Time { if didRun { return nil } @@ -166,11 +179,7 @@ func TestSchedulerWait(t *testing.T) { didRun = true return &t }, - onRun: func() error { - close(ran) - return nil - }, - }, TaskPriorityDefault) + ), tasks.TaskPriorityDefault) // Act go orch.Run(context.Background()) @@ -192,7 +201,7 @@ func TestSchedulerWait(t *testing.T) { t.Fatalf("should never run") return nil }, - }, TaskPriorityDefault) + }, tasks.TaskPriorityDefault) select { case <-time.NewTimer(200 * time.Millisecond).C: diff --git a/internal/orchestrator/repo/logging.go b/internal/orchestrator/repo/logging.go new file mode 100644 index 00000000..ebb8a18f --- /dev/null +++ b/internal/orchestrator/repo/logging.go @@ -0,0 +1,22 @@ +package repo + +import ( + "context" + + "github.com/garethgeorge/backrest/internal/ioutil" + "github.com/garethgeorge/backrest/internal/orchestrator/logging" + "github.com/garethgeorge/backrest/pkg/restic" +) + +// pipeResticLogsToWriter sets the restic logger to write to the provided writer. +// returns a new context with the logger set and a function to flush the logs. +func forwardResticLogs(ctx context.Context) (context.Context, func()) { + capture := ioutil.NewOutputCapturer(8 * 1024) + return restic.ContextWithLogger(ctx, capture), func() { + writer := logging.WriterFromContext(ctx) + if writer == nil { + return + } + _, _ = writer.Write(capture.Bytes()) + } +} diff --git a/internal/orchestrator/repo.go b/internal/orchestrator/repo/repo.go similarity index 93% rename from internal/orchestrator/repo.go rename to internal/orchestrator/repo/repo.go index 310793d8..a0cf2083 100644 --- a/internal/orchestrator/repo.go +++ b/internal/orchestrator/repo/repo.go @@ -1,4 +1,4 @@ -package orchestrator +package repo import ( "context" @@ -15,7 +15,6 @@ import ( "github.com/garethgeorge/backrest/pkg/restic" "github.com/google/shlex" "go.uber.org/zap" - "google.golang.org/protobuf/proto" ) // RepoOrchestrator is responsible for managing a single repo. @@ -70,10 +69,16 @@ func NewRepoOrchestrator(repoConfig *v1.Repo, resticPath string) (*RepoOrchestra } func (r *RepoOrchestrator) Init(ctx context.Context) error { + ctx, flush := forwardResticLogs(ctx) + defer flush() + return r.repo.Init(ctx) } func (r *RepoOrchestrator) Snapshots(ctx context.Context) ([]*restic.Snapshot, error) { + ctx, flush := forwardResticLogs(ctx) + defer flush() + snapshots, err := r.repo.Snapshots(ctx) if err != nil { return nil, fmt.Errorf("get snapshots for repo %v: %w", r.repoConfig.Id, err) @@ -83,6 +88,9 @@ func (r *RepoOrchestrator) Snapshots(ctx context.Context) ([]*restic.Snapshot, e } func (r *RepoOrchestrator) SnapshotsForPlan(ctx context.Context, plan *v1.Plan) ([]*restic.Snapshot, error) { + ctx, flush := forwardResticLogs(ctx) + defer flush() + snapshots, err := r.repo.Snapshots(ctx, restic.WithFlags("--tag", tagForPlan(plan))) if err != nil { return nil, fmt.Errorf("get snapshots for plan %q: %w", plan.Id, err) @@ -96,6 +104,7 @@ func (r *RepoOrchestrator) Backup(ctx context.Context, plan *v1.Plan, progressCa r.mu.Lock() defer r.mu.Unlock() + if !r.initialized { if err := r.repo.Init(ctx, restic.WithEnviron()); err != nil { return nil, fmt.Errorf("failed to initialize repo: %w", err) @@ -134,6 +143,8 @@ func (r *RepoOrchestrator) Backup(ctx context.Context, plan *v1.Plan, progressCa opts = append(opts, restic.WithFlags(args...)) } + ctx, flush := forwardResticLogs(ctx) + defer flush() summary, err := r.repo.Backup(ctx, plan.Paths, progressCallback, opts...) if err != nil { return summary, fmt.Errorf("failed to backup: %w", err) @@ -144,6 +155,9 @@ func (r *RepoOrchestrator) Backup(ctx context.Context, plan *v1.Plan, progressCa } func (r *RepoOrchestrator) ListSnapshotFiles(ctx context.Context, snapshotId string, path string) ([]*v1.LsEntry, error) { + ctx, flush := forwardResticLogs(ctx) + defer flush() + _, entries, err := r.repo.ListDirectory(ctx, snapshotId, path) if err != nil { return nil, fmt.Errorf("failed to list snapshot files: %w", err) @@ -160,6 +174,8 @@ func (r *RepoOrchestrator) ListSnapshotFiles(ctx context.Context, snapshotId str func (r *RepoOrchestrator) Forget(ctx context.Context, plan *v1.Plan) ([]*v1.ResticSnapshot, error) { r.mu.Lock() defer r.mu.Unlock() + ctx, flush := forwardResticLogs(ctx) + defer flush() policy := plan.Retention if policy == nil { @@ -190,6 +206,8 @@ func (r *RepoOrchestrator) Forget(ctx context.Context, plan *v1.Plan) ([]*v1.Res func (r *RepoOrchestrator) ForgetSnapshot(ctx context.Context, snapshotId string) error { r.mu.Lock() defer r.mu.Unlock() + ctx, flush := forwardResticLogs(ctx) + defer flush() r.l.Debug("forget snapshot with ID", zap.String("snapshot", snapshotId), zap.String("repo", r.repoConfig.Id)) return r.repo.ForgetSnapshot(ctx, snapshotId) @@ -198,6 +216,8 @@ func (r *RepoOrchestrator) ForgetSnapshot(ctx context.Context, snapshotId string func (r *RepoOrchestrator) Prune(ctx context.Context, output io.Writer) error { r.mu.Lock() defer r.mu.Unlock() + ctx, flush := forwardResticLogs(ctx) + defer flush() policy := r.repoConfig.PrunePolicy if policy == nil { @@ -224,6 +244,8 @@ func (r *RepoOrchestrator) Prune(ctx context.Context, output io.Writer) error { func (r *RepoOrchestrator) Restore(ctx context.Context, snapshotId string, path string, target string, progressCallback func(event *v1.RestoreProgressEntry)) (*v1.RestoreProgressEntry, error) { r.mu.Lock() defer r.mu.Unlock() + ctx, flush := forwardResticLogs(ctx) + defer flush() r.l.Debug("restore snapshot", zap.String("snapshot", snapshotId), zap.String("target", target)) @@ -250,9 +272,10 @@ func (r *RepoOrchestrator) UnlockIfAutoEnabled(ctx context.Context) error { if !r.repoConfig.AutoUnlock { return nil } - r.mu.Lock() defer r.mu.Unlock() + ctx, flush := forwardResticLogs(ctx) + defer flush() zap.L().Debug("auto-unlocking repo", zap.String("repo", r.repoConfig.Id)) @@ -272,6 +295,8 @@ func (r *RepoOrchestrator) Unlock(ctx context.Context) error { func (r *RepoOrchestrator) Stats(ctx context.Context) (*v1.RepoStats, error) { r.mu.Lock() defer r.mu.Unlock() + ctx, flush := forwardResticLogs(ctx) + defer flush() r.l.Debug("getting repo stats", zap.String("repo", r.repoConfig.Id)) stats, err := r.repo.Stats(ctx) @@ -286,7 +311,7 @@ func (r *RepoOrchestrator) Config() *v1.Repo { if r == nil { return nil } - return proto.Clone(r.repoConfig).(*v1.Repo) + return r.repoConfig } func tagForPlan(plan *v1.Plan) string { diff --git a/internal/orchestrator/repo_test.go b/internal/orchestrator/repo/repo_test.go similarity index 99% rename from internal/orchestrator/repo_test.go rename to internal/orchestrator/repo/repo_test.go index 0dace95f..2f06340f 100644 --- a/internal/orchestrator/repo_test.go +++ b/internal/orchestrator/repo/repo_test.go @@ -1,4 +1,4 @@ -package orchestrator +package repo import ( "context" diff --git a/internal/orchestrator/scheduledtaskheap.go b/internal/orchestrator/scheduledtaskheap.go deleted file mode 100644 index 1944b311..00000000 --- a/internal/orchestrator/scheduledtaskheap.go +++ /dev/null @@ -1,231 +0,0 @@ -package orchestrator - -import ( - "container/heap" - "context" - "sync" - "time" - - v1 "github.com/garethgeorge/backrest/gen/go/v1" -) - -var taskQueueDefaultPollInterval = 3 * time.Minute - -type taskQueue struct { - dequeueMu sync.Mutex - mu sync.Mutex - heap scheduledTaskHeapByTime - notify chan struct{} - ready scheduledTaskHeapByPriorityThenTime - pollInterval time.Duration - - Now func() time.Time -} - -func newTaskQueue(now func() time.Time) taskQueue { - return taskQueue{ - heap: scheduledTaskHeapByTime{}, - ready: scheduledTaskHeapByPriorityThenTime{}, - pollInterval: taskQueueDefaultPollInterval, - Now: now, - } -} - -func (t *taskQueue) curTime() time.Time { - if t.Now != nil { - return t.Now() - } - return time.Now() -} - -func (t *taskQueue) Push(tasks ...scheduledTask) { - t.mu.Lock() - defer t.mu.Unlock() - - for _, task := range tasks { - task := task - if task.task == nil { - panic("task cannot be nil") - } - heap.Push(&t.heap, &task) - } - - if t.notify != nil { - t.notify <- struct{}{} - } -} - -func (t *taskQueue) Reset() []*scheduledTask { - t.mu.Lock() - defer t.mu.Unlock() - - oldTasks := t.heap.tasks - oldTasks = append(oldTasks, t.ready.tasks...) - t.heap.tasks = nil - t.ready.tasks = nil - - if t.notify != nil { - t.notify <- struct{}{} - } - return oldTasks -} - -func (t *taskQueue) Dequeue(ctx context.Context) *scheduledTask { - t.dequeueMu.Lock() - defer t.dequeueMu.Unlock() - - t.mu.Lock() - defer t.mu.Unlock() - t.notify = make(chan struct{}, 10) - defer func() { - close(t.notify) - t.notify = nil - }() - - for { - first, ok := t.heap.Peek().(*scheduledTask) - if !ok { // no tasks in heap. - if t.ready.Len() > 0 { - return heap.Pop(&t.ready).(*scheduledTask) - } - t.mu.Unlock() - select { - case <-ctx.Done(): - t.mu.Lock() - return nil - case <-t.notify: - } - t.mu.Lock() - continue - } - - now := t.curTime() - - // if there's a task in the ready queue AND the first task isn't ready yet then immediately return the ready task. - ready, ok := t.ready.Peek().(*scheduledTask) - if ok && now.Before(first.runAt) { - heap.Pop(&t.ready) - return ready - } - - t.mu.Unlock() - d := first.runAt.Sub(now) - if t.pollInterval > 0 && d > t.pollInterval { - // A poll interval may be set to work around clock changes - // e.g. when a laptop wakes from sleep or the system clock is adjusted. - d = t.pollInterval - } - timer := time.NewTimer(d) - - select { - case <-timer.C: - t.mu.Lock() - if t.heap.Len() == 0 { - break - } - - for { - first, ok := t.heap.Peek().(*scheduledTask) - if !ok { - break - } - if first.runAt.After(t.curTime()) { - // task is not yet ready to run - break - } - heap.Pop(&t.heap) // remove the task from the heap - heap.Push(&t.ready, first) - } - - if t.ready.Len() == 0 { - break - } - return heap.Pop(&t.ready).(*scheduledTask) - case <-t.notify: // new task was added, loop again to ensure we have the earliest task. - t.mu.Lock() - if !timer.Stop() { - <-timer.C - } - case <-ctx.Done(): - t.mu.Lock() - if !timer.Stop() { - <-timer.C - } - return nil - } - } -} - -type scheduledTask struct { - task Task - runAt time.Time - priority int - callbacks []func(error) - config *v1.Config -} - -func (s *scheduledTask) Less(other *scheduledTask) bool { - if s.priority != other.priority { - return s.priority > other.priority - } - return s.runAt.Before(other.runAt) -} - -func (s scheduledTask) Eq(other scheduledTask) bool { - return s.task == other.task && s.runAt.Equal(other.runAt) && s.priority == other.priority && s.config == other.config -} - -type scheduledTaskHeap struct { - tasks []*scheduledTask - comparator func(i, j *scheduledTask) bool -} - -func (h *scheduledTaskHeap) Len() int { - return len(h.tasks) -} - -func (h *scheduledTaskHeap) Swap(i, j int) { - h.tasks[i], h.tasks[j] = h.tasks[j], h.tasks[i] -} - -func (h *scheduledTaskHeap) Push(x interface{}) { - h.tasks = append(h.tasks, x.(*scheduledTask)) -} - -func (h *scheduledTaskHeap) Pop() interface{} { - old := h.tasks - n := len(old) - x := old[n-1] - h.tasks = old[0 : n-1] - return x -} - -func (h *scheduledTaskHeap) Peek() interface{} { - if len(h.tasks) == 0 { - return nil - } - return h.tasks[0] -} - -type scheduledTaskHeapByTime struct { - scheduledTaskHeap -} - -var _ heap.Interface = &scheduledTaskHeapByTime{} - -func (h *scheduledTaskHeapByTime) Less(i, j int) bool { - return h.tasks[i].runAt.Before(h.tasks[j].runAt) -} - -type scheduledTaskHeapByPriorityThenTime struct { - scheduledTaskHeap -} - -var _ heap.Interface = &scheduledTaskHeapByPriorityThenTime{} - -func (h *scheduledTaskHeapByPriorityThenTime) Less(i, j int) bool { - if h.tasks[i].priority != h.tasks[j].priority { - return h.tasks[i].priority > h.tasks[j].priority - } - return h.tasks[i].runAt.Before(h.tasks[j].runAt) -} diff --git a/internal/orchestrator/task.go b/internal/orchestrator/task.go deleted file mode 100644 index a9f76939..00000000 --- a/internal/orchestrator/task.go +++ /dev/null @@ -1,137 +0,0 @@ -package orchestrator - -import ( - "context" - "errors" - "fmt" - "sync/atomic" - "time" - - v1 "github.com/garethgeorge/backrest/gen/go/v1" - "github.com/garethgeorge/backrest/internal/ioutil" - "github.com/garethgeorge/backrest/internal/oplog" - "github.com/garethgeorge/backrest/pkg/restic" - "github.com/hashicorp/go-multierror" - "go.uber.org/zap" -) - -type Task interface { - Name() string // huamn readable name for this task. - Next(now time.Time) *time.Time // when this task would like to be run. - Run(ctx context.Context) error // run the task. - Cancel(withStatus v1.OperationStatus) error // informat the task that it's scheduled execution will be skipped (either STATUS_USER_CANCELLED or STATUS_SYSTEM_CANCELLED). - OperationId() int64 // the id of the operation associated with this task (if any). -} - -type TaskWithOperation struct { - orch *Orchestrator - op *v1.Operation - running atomic.Bool -} - -func (t *TaskWithOperation) OperationId() int64 { - if t.op == nil { - return 0 - } - return t.op.Id -} - -func (t *TaskWithOperation) setOperation(op *v1.Operation) error { - if t.op != nil { - return errors.New("task already has an operation") - } - if err := t.orch.OpLog.Add(op); err != nil { - return fmt.Errorf("task failed to add operation to oplog: %v", err) - } - t.op = op - return nil -} - -func (t *TaskWithOperation) runWithOpAndContext(ctx context.Context, do func(ctx context.Context, op *v1.Operation) error) error { - if t.op == nil { - return errors.New("task has no operation, a call to setOperation first is required") - } - if t.running.Load() { - return errors.New("task is already running") - } - - t.running.Store(true) - defer t.running.Store(false) - defer func() { - t.op = nil - }() - - return WithOperation(t.orch.OpLog, t.op, func() error { - capture := ioutil.NewOutputCapturer(32_000) // 32k of logs - ctx = restic.ContextWithLogger(ctx, capture) - - err := do(ctx, t.op) - - if bytes := capture.Bytes(); len(bytes) > 0 { - ref, e := t.orch.logStore.Write(bytes) - if e != nil { - errors.Join(err, fmt.Errorf("failed to write log to logstore: %w", e)) - } - t.op.Logref = ref - zap.S().Debugf("wrote operation log to %v", ref) - } - - return err - }) -} - -// Cancel marks a task as cancelled. Note that, unintuitively, it is actually an error to call cancel on a running task. -func (t *TaskWithOperation) Cancel(withStatus v1.OperationStatus) error { - if t.running.Load() { - return errors.New("cannot cancel a running task") // should never happen. - } - if t.op == nil { - return nil - } - t.op.Status = withStatus - t.op.UnixTimeEndMs = curTimeMillis() - if err := t.orch.OpLog.Update(t.op); err != nil { - return fmt.Errorf("failed to update operation %v in oplog: %w", t.op.Id, err) - } - t.op = nil - return nil -} - -// WithOperation is a utility that creates an operation to track the function's execution. -// timestamps are automatically added and the status is automatically updated if an error occurs. -func WithOperation(oplog *oplog.OpLog, op *v1.Operation, do func() error) error { - op.UnixTimeStartMs = curTimeMillis() // update the start time from the planned time to the actual time. - if op.Status == v1.OperationStatus_STATUS_PENDING || op.Status == v1.OperationStatus_STATUS_UNKNOWN { - op.Status = v1.OperationStatus_STATUS_INPROGRESS - } - if op.Id != 0 { - if err := oplog.Update(op); err != nil { - return fmt.Errorf("failed to add operation to oplog: %w", err) - } - } else { - if err := oplog.Add(op); err != nil { - return fmt.Errorf("failed to add operation to oplog: %w", err) - } - } - err := do() - if err != nil { - op.Status = v1.OperationStatus_STATUS_ERROR - op.DisplayMessage = err.Error() - } - op.UnixTimeEndMs = curTimeMillis() - if op.Status == v1.OperationStatus_STATUS_INPROGRESS { - op.Status = v1.OperationStatus_STATUS_SUCCESS - } - if e := oplog.Update(op); e != nil { - return multierror.Append(err, fmt.Errorf("failed to update operation in oplog: %w", e)) - } - return err -} - -func timeToUnixMillis(t time.Time) int64 { - return t.Unix()*1000 + int64(t.Nanosecond()/1000000) -} - -func curTimeMillis() int64 { - return timeToUnixMillis(time.Now()) -} diff --git a/internal/orchestrator/taskcollectgarbage.go b/internal/orchestrator/taskcollectgarbage.go deleted file mode 100644 index 5e75c10f..00000000 --- a/internal/orchestrator/taskcollectgarbage.go +++ /dev/null @@ -1,134 +0,0 @@ -package orchestrator - -import ( - "context" - "fmt" - "time" - - v1 "github.com/garethgeorge/backrest/gen/go/v1" - "go.uber.org/zap" -) - -const ( - gcStartupDelay = 5 * time.Second - gcInterval = 24 * time.Hour - // keep operations that are eligible for gc for 30 days OR up to a limit of 100 for any one plan. - // an operation is eligible for gc if: - // - it has no snapshot associated with it - // - it has a forgotten snapshot associated with it - gcHistoryAge = 30 * 24 * time.Hour - gcHistoryMaxCount = 1000 - // keep stats operations for 1 year (they're small and useful for long term trends) - gcHistoryStatsAge = 365 * 24 * time.Hour -) - -type CollectGarbageTask struct { - orchestrator *Orchestrator // owning orchestrator - firstRun bool -} - -var _ Task = &CollectGarbageTask{} - -func (t *CollectGarbageTask) Name() string { - return "collect garbage" -} - -func (t *CollectGarbageTask) Next(now time.Time) *time.Time { - if !t.firstRun { - t.firstRun = true - runAt := now.Add(gcStartupDelay) - return &runAt - } - - runAt := now.Add(gcInterval) - return &runAt -} - -func (t *CollectGarbageTask) Run(ctx context.Context) error { - if err := t.gcOperations(); err != nil { - return fmt.Errorf("collecting garbage: %w", err) - } - - return nil -} - -func (t *CollectGarbageTask) gcOperations() error { - oplog := t.orchestrator.OpLog - - // pass 1: identify forgotten snapshots. - snapshotIsForgotten := make(map[string]bool) - if err := oplog.ForAll(func(op *v1.Operation) error { - if snapshotOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok { - if snapshotOp.OperationIndexSnapshot.Forgot { - snapshotIsForgotten[snapshotOp.OperationIndexSnapshot.Snapshot.Id] = true - } - } - return nil - }); err != nil { - return fmt.Errorf("identifying forgotten snapshots: %w", err) - } - - // pass 2: identify operations that are gc eligible - // - any operation that has no snapshot associated with it - // - any operation that has a forgotten snapshot associated with it - operationsByPlan := make(map[string][]gcOpInfo) - if err := oplog.ForAll(func(op *v1.Operation) error { - if op.SnapshotId == "" || snapshotIsForgotten[op.SnapshotId] { - _, isStats := op.Op.(*v1.Operation_OperationStats) - operationsByPlan[op.PlanId] = append(operationsByPlan[op.PlanId], gcOpInfo{ - id: op.Id, - timestamp: op.UnixTimeStartMs, - isStats: isStats, - }) - } - return nil - }); err != nil { - return fmt.Errorf("identifying gc eligible operations: %w", err) - } - - var gcOps []int64 - curTime := curTimeMillis() - for _, opInfos := range operationsByPlan { - if len(opInfos) >= gcHistoryMaxCount { - for _, opInfo := range opInfos[:len(opInfos)-gcHistoryMaxCount] { - gcOps = append(gcOps, opInfo.id) - } - opInfos = opInfos[len(opInfos)-gcHistoryMaxCount:] - } - - // check if each operation timestamp is old. - for _, opInfo := range opInfos { - maxAgeForType := gcHistoryAge.Milliseconds() - if opInfo.isStats { - maxAgeForType = gcHistoryStatsAge.Milliseconds() - } - if curTime-opInfo.timestamp > maxAgeForType { - gcOps = append(gcOps, opInfo.id) - } - } - } - - // pass 3: remove gc eligible operations - if err := oplog.Delete(gcOps...); err != nil { - return fmt.Errorf("removing gc eligible operations: %w", err) - } - - zap.L().Info("collecting garbage", - zap.Int("forgotten_snapshots", len(snapshotIsForgotten)), - zap.Any("operations_removed", len(gcOps))) - return nil -} - -func (t *CollectGarbageTask) Cancel(withStatus v1.OperationStatus) error { - return nil -} - -func (t *CollectGarbageTask) OperationId() int64 { - return 0 -} - -type gcOpInfo struct { - id int64 // operation ID - timestamp int64 // unix time milliseconds - isStats bool // true if this is a stats operation -} diff --git a/internal/orchestrator/taskforget.go b/internal/orchestrator/taskforget.go deleted file mode 100644 index 46020b30..00000000 --- a/internal/orchestrator/taskforget.go +++ /dev/null @@ -1,121 +0,0 @@ -package orchestrator - -import ( - "context" - "fmt" - "time" - - v1 "github.com/garethgeorge/backrest/gen/go/v1" - "github.com/garethgeorge/backrest/internal/hook" - "github.com/garethgeorge/backrest/internal/oplog/indexutil" - "github.com/hashicorp/go-multierror" - "go.uber.org/zap" -) - -// ForgetTask tracks a forget operation. -type ForgetTask struct { - name string - TaskWithOperation - plan *v1.Plan - linkSnapshot string // snapshot to link the task to. - at *time.Time -} - -var _ Task = &ForgetTask{} - -func NewOneoffForgetTask(orchestrator *Orchestrator, plan *v1.Plan, linkSnapshot string, at time.Time) *ForgetTask { - return &ForgetTask{ - TaskWithOperation: TaskWithOperation{ - orch: orchestrator, - }, - plan: plan, - at: &at, - linkSnapshot: linkSnapshot, - } -} - -func (t *ForgetTask) Name() string { - return fmt.Sprintf("forget for plan %q", t.plan.Id) -} - -func (t *ForgetTask) Next(now time.Time) *time.Time { - ret := t.at - if ret != nil { - t.at = nil - if err := t.setOperation(&v1.Operation{ - PlanId: t.plan.Id, - RepoId: t.plan.Repo, - SnapshotId: t.linkSnapshot, - UnixTimeStartMs: timeToUnixMillis(*ret), - Status: v1.OperationStatus_STATUS_PENDING, - Op: &v1.Operation_OperationForget{}, - }); err != nil { - zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err) - return nil - } - } - return ret -} - -func (t *ForgetTask) Run(ctx context.Context) error { - if err := t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error { - forgetOp := &v1.Operation_OperationForget{ - OperationForget: &v1.OperationForget{}, - } - op.Op = forgetOp - - var err error - repo, err := t.orch.GetRepo(t.plan.Repo) - if err != nil { - return fmt.Errorf("get repo %q: %w", t.plan.Repo, err) - } - - err = repo.UnlockIfAutoEnabled(ctx) - if err != nil { - return fmt.Errorf("auto unlock repo %q: %w", t.plan.Repo, err) - } - - forgot, err := repo.Forget(ctx, t.plan) - if err != nil { - return fmt.Errorf("forget: %w", err) - } - - forgetOp.OperationForget.Forget = append(forgetOp.OperationForget.Forget, forgot...) - forgetOp.OperationForget.Policy = t.plan.Retention - - var ops []*v1.Operation - for _, forgot := range forgot { - if e := t.orch.OpLog.ForEachBySnapshotId(forgot.Id, indexutil.CollectAll(), func(op *v1.Operation) error { - ops = append(ops, op) - return nil - }); e != nil { - err = multierror.Append(err, fmt.Errorf("cleanup snapshot %v: %w", forgot.Id, e)) - } - } - for _, op := range ops { - if indexOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok { - indexOp.OperationIndexSnapshot.Forgot = true - if e := t.orch.OpLog.Update(op); err != nil { - err = multierror.Append(err, fmt.Errorf("mark index snapshot %v as forgotten: %w", op.Id, e)) - continue - } - } - } - - if len(forgot) > 0 { - t.orch.ScheduleTask(NewOneoffPruneTask(t.orch, t.plan, time.Now(), false), TaskPriorityPrune) - } - - return err - }); err != nil { - repo, _ := t.orch.GetRepo(t.plan.Repo) - _ = t.orch.hookExecutor.ExecuteHooks(repo.Config(), t.plan, []v1.Hook_Condition{ - v1.Hook_CONDITION_ANY_ERROR, - }, hook.HookVars{ - Task: t.Name(), - Error: err.Error(), - SnapshotId: t.linkSnapshot, - }) - } - return nil -} diff --git a/internal/orchestrator/taskforgetsnapshot.go b/internal/orchestrator/taskforgetsnapshot.go deleted file mode 100644 index 0cc435ee..00000000 --- a/internal/orchestrator/taskforgetsnapshot.go +++ /dev/null @@ -1,94 +0,0 @@ -package orchestrator - -import ( - "context" - "fmt" - "time" - - v1 "github.com/garethgeorge/backrest/gen/go/v1" - "github.com/garethgeorge/backrest/internal/oplog/indexutil" - "github.com/hashicorp/go-multierror" - "go.uber.org/zap" -) - -// ForgetTask tracks a forget operation. -type ForgetSnapshotTask struct { - TaskWithOperation - repoId string - planId string - forgetSnapshot string - at *time.Time -} - -var _ Task = &ForgetSnapshotTask{} - -func NewOneoffForgetSnapshotTask(orchestrator *Orchestrator, repoId, planId, forgetSnapshot string, at time.Time) *ForgetSnapshotTask { - return &ForgetSnapshotTask{ - TaskWithOperation: TaskWithOperation{ - orch: orchestrator, - }, - repoId: repoId, - planId: planId, - at: &at, - forgetSnapshot: forgetSnapshot, - } -} - -func (t *ForgetSnapshotTask) Name() string { - return fmt.Sprintf("forget snapshot %q", t.forgetSnapshot) -} - -func (t *ForgetSnapshotTask) Next(now time.Time) *time.Time { - ret := t.at - if ret != nil { - t.at = nil - if err := t.setOperation(&v1.Operation{ - PlanId: t.planId, - RepoId: t.repoId, - UnixTimeStartMs: timeToUnixMillis(*ret), - Status: v1.OperationStatus_STATUS_PENDING, - Op: &v1.Operation_OperationForget{}, - }); err != nil { - zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err) - return nil - } - } - return ret -} - -func (t *ForgetSnapshotTask) Run(ctx context.Context) error { - id := t.op.Id - if err := t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error { - repo, err := t.orch.GetRepo(t.repoId) - if err != nil { - return fmt.Errorf("get repo %q: %w", t.repoId, err) - } - - // Find snapshot to forget - var ops []*v1.Operation - t.orch.OpLog.ForEachBySnapshotId(t.forgetSnapshot, indexutil.CollectAll(), func(op *v1.Operation) error { - ops = append(ops, op) - return nil - }) - - for _, op := range ops { - if indexOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok { - err := repo.ForgetSnapshot(ctx, op.SnapshotId) - if err != nil { - return fmt.Errorf("forget %q: %w", op.SnapshotId, err) - } - indexOp.OperationIndexSnapshot.Forgot = true - if e := t.orch.OpLog.Update(op); err != nil { - err = multierror.Append(err, fmt.Errorf("mark index snapshot %v as forgotten: %w", op.Id, e)) - continue - } - } - } - - return err - }); err != nil { - return err - } - - return t.orch.OpLog.Delete(id) -} diff --git a/internal/orchestrator/taskprune.go b/internal/orchestrator/taskprune.go deleted file mode 100644 index 592701e2..00000000 --- a/internal/orchestrator/taskprune.go +++ /dev/null @@ -1,203 +0,0 @@ -package orchestrator - -import ( - "bytes" - "context" - "fmt" - "sync" - "time" - - v1 "github.com/garethgeorge/backrest/gen/go/v1" - "github.com/garethgeorge/backrest/internal/hook" - "github.com/garethgeorge/backrest/internal/oplog" - "github.com/garethgeorge/backrest/internal/oplog/indexutil" - "go.uber.org/zap" -) - -// PruneTask tracks a forget operation. -type PruneTask struct { - TaskWithOperation - plan *v1.Plan - at *time.Time - force bool -} - -var _ Task = &PruneTask{} - -func NewOneoffPruneTask(orchestrator *Orchestrator, plan *v1.Plan, at time.Time, force bool) *PruneTask { - return &PruneTask{ - TaskWithOperation: TaskWithOperation{ - orch: orchestrator, - }, - plan: plan, - at: &at, - force: force, // overrides the PrunePolicy's MaxFrequencyDays - } -} - -func (t *PruneTask) Name() string { - return fmt.Sprintf("prune for plan %q", t.plan.Id) -} - -func (t *PruneTask) Next(now time.Time) *time.Time { - shouldRun, err := t.shouldRun(now) - if err != nil { - zap.S().Errorf("task %v failed to check if it should run: %v", t.Name(), err) - } - if !shouldRun { - return nil - } - - ret := t.at - if ret != nil { - t.at = nil - if err := t.setOperation(&v1.Operation{ - PlanId: t.plan.Id, - RepoId: t.plan.Repo, - UnixTimeStartMs: timeToUnixMillis(*ret), - Status: v1.OperationStatus_STATUS_PENDING, - Op: &v1.Operation_OperationPrune{}, - }); err != nil { - zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err) - return nil - } - } - return ret -} - -func (t *PruneTask) shouldRun(now time.Time) (bool, error) { - if t.force { - return true, nil - } - - repo, err := t.orch.GetRepo(t.plan.Repo) - if err != nil { - return false, fmt.Errorf("get repo %v: %w", t.plan.Repo, err) - } - - nextPruneTime, err := t.getNextPruneTime(repo, repo.repoConfig.PrunePolicy) - if err != nil { - return false, fmt.Errorf("get next prune time: %w", err) - } - - return nextPruneTime.Before(now), nil -} - -func (t *PruneTask) getNextPruneTime(repo *RepoOrchestrator, policy *v1.PrunePolicy) (time.Time, error) { - var lastPruneTime time.Time - t.orch.OpLog.ForEachByRepo(t.plan.Repo, indexutil.Reversed(indexutil.CollectAll()), func(op *v1.Operation) error { - if _, ok := op.Op.(*v1.Operation_OperationPrune); ok { - lastPruneTime = time.Unix(0, op.UnixTimeStartMs*int64(time.Millisecond)) - return oplog.ErrStopIteration - } - return nil - }) - - if repo.repoConfig.PrunePolicy != nil { - return lastPruneTime.Add(time.Duration(repo.repoConfig.PrunePolicy.MaxFrequencyDays) * 24 * time.Hour), nil - } else { - return lastPruneTime.Add(7 * 24 * time.Hour), nil // default to 7 days. - } -} - -func (t *PruneTask) Run(ctx context.Context) error { - if err := t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error { - repo, err := t.orch.GetRepo(t.plan.Repo) - if err != nil { - return fmt.Errorf("get repo %v: %w", t.plan.Repo, err) - } - - err = repo.UnlockIfAutoEnabled(ctx) - if err != nil { - return fmt.Errorf("auto unlock repo %q: %w", t.plan.Repo, err) - } - - opPrune := &v1.Operation_OperationPrune{ - OperationPrune: &v1.OperationPrune{}, - } - op.Op = opPrune - - ctx, cancel := context.WithCancel(ctx) - interval := time.NewTicker(1 * time.Second) - defer interval.Stop() - var buf synchronizedBuffer - var wg sync.WaitGroup - wg.Add(1) - go func() { - - defer wg.Done() - for { - select { - case <-interval.C: - output := buf.String() - if len(output) > 8*1024 { // only provide live status upto the first 8K of output. - output = output[:len(output)-8*1024] - } - - if opPrune.OperationPrune.Output != output { - opPrune.OperationPrune.Output = buf.String() - - if err := t.orch.OpLog.Update(op); err != nil { - zap.L().Error("update prune operation with status output", zap.Error(err)) - } - } - case <-ctx.Done(): - return - } - } - }() - - if err := repo.Prune(ctx, &buf); err != nil { - cancel() - return fmt.Errorf("prune: %w", err) - } - cancel() - wg.Wait() - - // TODO: it would be best to store the output in separate storage for large status data. - output := buf.String() - if len(output) > 8*1024 { // only save the first 4K of output. - output = output[:len(output)-8*1024] - } - op.Op = &v1.Operation_OperationPrune{ - OperationPrune: &v1.OperationPrune{ - Output: output, - }, - } - - return nil - }); err != nil { - repo, _ := t.orch.GetRepo(t.plan.Repo) - _ = t.orch.hookExecutor.ExecuteHooks(repo.Config(), t.plan, []v1.Hook_Condition{ - v1.Hook_CONDITION_ANY_ERROR, - }, hook.HookVars{ - Task: t.Name(), - Error: err.Error(), - }) - return err - } - - t.orch.ScheduleTask(NewOneoffStatsTask(t.orch, t.plan.Repo, t.plan.Id, time.Now()), TaskPriorityStats) - - return nil -} - -// synchronizedBuffer is used for collecting prune command's output -type synchronizedBuffer struct { - mu sync.Mutex - buf bytes.Buffer -} - -func (w *synchronizedBuffer) Write(p []byte) (n int, err error) { - w.mu.Lock() - defer w.mu.Unlock() - - return w.buf.Write(p) -} - -func (w *synchronizedBuffer) String() string { - w.mu.Lock() - defer w.mu.Unlock() - - return w.buf.String() -} diff --git a/internal/orchestrator/taskrestore.go b/internal/orchestrator/taskrestore.go deleted file mode 100644 index 8fea3a9f..00000000 --- a/internal/orchestrator/taskrestore.go +++ /dev/null @@ -1,126 +0,0 @@ -package orchestrator - -import ( - "context" - "errors" - "fmt" - "sync" - "time" - - v1 "github.com/garethgeorge/backrest/gen/go/v1" - "github.com/garethgeorge/backrest/internal/hook" - "go.uber.org/zap" -) - -type RestoreTaskOpts struct { - RepoId string // optional - PlanId string // optional - SnapshotId string // required - Path string // required - Target string // required -} - -// RestoreTask tracks a forget operation. -type RestoreTask struct { - TaskWithOperation - restoreOpts RestoreTaskOpts - at *time.Time -} - -var _ Task = &RestoreTask{} - -func NewOneoffRestoreTask(orchestrator *Orchestrator, opts RestoreTaskOpts, at time.Time) *RestoreTask { - return &RestoreTask{ - TaskWithOperation: TaskWithOperation{ - orch: orchestrator, - }, - restoreOpts: opts, - at: &at, - } -} - -func (t *RestoreTask) Name() string { - return fmt.Sprintf("restore snapshot %v in repo %v", t.restoreOpts.SnapshotId, t.restoreOpts.RepoId) -} - -func (t *RestoreTask) Next(now time.Time) *time.Time { - ret := t.at - if ret != nil { - t.at = nil - if err := t.setOperation(&v1.Operation{ - PlanId: t.restoreOpts.PlanId, - RepoId: t.restoreOpts.RepoId, - SnapshotId: t.restoreOpts.SnapshotId, - UnixTimeStartMs: timeToUnixMillis(*ret), - Status: v1.OperationStatus_STATUS_PENDING, - Op: &v1.Operation_OperationRestore{}, - }); err != nil { - zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err) - return nil - } - } - return ret -} - -func (t *RestoreTask) Run(ctx context.Context) error { - if t.restoreOpts.SnapshotId == "" || t.restoreOpts.Path == "" || t.restoreOpts.Target == "" { - return errors.New("snapshotId, path, and target are required") - } - - if err := t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error { - forgetOp := &v1.Operation_OperationRestore{ - OperationRestore: &v1.OperationRestore{ - Path: t.restoreOpts.Path, - Target: t.restoreOpts.Target, - }, - } - op.Op = forgetOp - op.UnixTimeStartMs = curTimeMillis() - - repo, err := t.orch.GetRepo(t.restoreOpts.RepoId) - if err != nil { - return fmt.Errorf("couldn't get repo %q: %w", t.restoreOpts.RepoId, err) - } - - var sendWg sync.WaitGroup - lastSent := time.Now() // debounce progress updates, these can endup being very frequent. - summary, err := repo.Restore(ctx, t.restoreOpts.SnapshotId, t.restoreOpts.Path, t.restoreOpts.Target, func(entry *v1.RestoreProgressEntry) { - sendWg.Wait() - if time.Since(lastSent) < 1*time.Second { - return - } - lastSent = time.Now() - - zap.S().Infof("restore progress: %v", entry) - - forgetOp.OperationRestore.Status = entry - - sendWg.Add(1) - go func() { - if err := t.orch.OpLog.Update(op); err != nil { - zap.S().Errorf("failed to update oplog with progress for restore: %v", err) - } - sendWg.Done() - }() - }) - if err != nil { - return fmt.Errorf("restore failed: %w", err) - } - forgetOp.OperationRestore.Status = summary - - return nil - }); err != nil { - if t.restoreOpts.RepoId != "" { - repo, _ := t.orch.GetRepo(t.restoreOpts.RepoId) - _ = t.orch.hookExecutor.ExecuteHooks(repo.Config(), nil, []v1.Hook_Condition{ - v1.Hook_CONDITION_ANY_ERROR, - }, hook.HookVars{ - Task: t.Name(), - Error: err.Error(), - SnapshotId: t.restoreOpts.SnapshotId, - }) - } - return err - } - return nil -} diff --git a/internal/orchestrator/taskrunnerimpl.go b/internal/orchestrator/taskrunnerimpl.go new file mode 100644 index 00000000..7adfe563 --- /dev/null +++ b/internal/orchestrator/taskrunnerimpl.go @@ -0,0 +1,104 @@ +package orchestrator + +import ( + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/hook" + "github.com/garethgeorge/backrest/internal/oplog" + "github.com/garethgeorge/backrest/internal/orchestrator/repo" + "github.com/garethgeorge/backrest/internal/orchestrator/tasks" +) + +// taskRunnerImpl is an implementation of TaskRunner for the default orchestrator. +type taskRunnerImpl struct { + orchestrator *Orchestrator + t tasks.Task + op *v1.Operation + repo *v1.Repo // cache, populated on first call to Repo() + plan *v1.Plan // cache, populated on first call to Plan() +} + +var _ tasks.TaskRunner = &taskRunnerImpl{} + +func (t *taskRunnerImpl) FindRepo() (*v1.Repo, error) { + if t.repo != nil { + return t.repo, nil + } + var err error + t.repo, err = t.orchestrator.GetRepo(t.t.RepoID()) + return t.repo, err +} + +func (t *taskRunnerImpl) FindPlan() (*v1.Plan, error) { + if t.plan != nil { + return t.plan, nil + } + var err error + t.plan, err = t.orchestrator.GetPlan(t.t.PlanID()) + return t.plan, err +} + +func newTaskRunnerImpl(orchestrator *Orchestrator, task tasks.Task, op *v1.Operation) *taskRunnerImpl { + return &taskRunnerImpl{ + orchestrator: orchestrator, + t: task, + op: op, + } +} + +func (t *taskRunnerImpl) CreateOperation(op *v1.Operation) error { + return t.orchestrator.OpLog.Add(op) +} + +func (t *taskRunnerImpl) UpdateOperation(op *v1.Operation) error { + return t.orchestrator.OpLog.Update(op) +} + +func (t *taskRunnerImpl) Orchestrator() *Orchestrator { + return t.orchestrator +} + +func (t *taskRunnerImpl) OpLog() *oplog.OpLog { + return t.orchestrator.OpLog +} + +func (t *taskRunnerImpl) ExecuteHooks(events []v1.Hook_Condition, vars hook.HookVars) error { + repoID := t.t.RepoID() + planID := t.t.PlanID() + var repo *v1.Repo + var plan *v1.Plan + if repoID != "" { + var err error + repo, err = t.FindRepo() + if err != nil { + return err + } + } + if planID != "" { + var err error + plan, err = t.FindPlan() + if err != nil { + return err + } + } + var flowID int64 + if t.op != nil { + flowID = t.op.FlowId + } + return t.orchestrator.hookExecutor.ExecuteHooks(flowID, repo, plan, events, vars) +} + +func (t *taskRunnerImpl) GetRepo(repoID string) (*v1.Repo, error) { + return t.orchestrator.GetRepo(repoID) +} + +func (t *taskRunnerImpl) GetPlan(planID string) (*v1.Plan, error) { + return t.orchestrator.GetPlan(planID) +} + +func (t *taskRunnerImpl) GetRepoOrchestrator(repoID string) (*repo.RepoOrchestrator, error) { + return t.orchestrator.GetRepoOrchestrator(repoID) +} + +func (t *taskRunnerImpl) ScheduleTask(task tasks.Task, priority int) error { + return t.orchestrator.ScheduleTask(task, priority) +} diff --git a/internal/orchestrator/tasks/flowidutil.go b/internal/orchestrator/tasks/flowidutil.go new file mode 100644 index 00000000..0dbcc716 --- /dev/null +++ b/internal/orchestrator/tasks/flowidutil.go @@ -0,0 +1,24 @@ +package tasks + +import ( + "fmt" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/oplog" + "github.com/garethgeorge/backrest/internal/oplog/indexutil" +) + +// FlowIDForSnapshotID returns the flow ID associated with the backup task that created snapshot ID or 0 if not found. +func FlowIDForSnapshotID(oplog *oplog.OpLog, snapshotID string) (int64, error) { + var flowID int64 + if err := oplog.ForEachBySnapshotId(snapshotID, indexutil.CollectAll(), func(op *v1.Operation) error { + if _, ok := op.Op.(*v1.Operation_OperationBackup); !ok { + return nil + } + flowID = op.FlowId + return nil + }); err != nil { + return 0, fmt.Errorf("get flow id for snapshot %q: %w", snapshotID, err) + } + return flowID, nil +} diff --git a/internal/orchestrator/tasks/loggingutil.go b/internal/orchestrator/tasks/loggingutil.go new file mode 100644 index 00000000..cba2e91e --- /dev/null +++ b/internal/orchestrator/tasks/loggingutil.go @@ -0,0 +1,26 @@ +package tasks + +import ( + "context" + + "github.com/garethgeorge/backrest/internal/orchestrator/logging" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// LoggerFromContext returns a logger from the context, or the global logger if none is found. +// this is somewhat expensive, it should be called once per task. +func Logger(ctx context.Context, task Task) *zap.Logger { + writer := logging.WriterFromContext(ctx) + if writer == nil { + return zap.L() + } + p := zap.NewProductionEncoderConfig() + p.EncodeTime = zapcore.ISO8601TimeEncoder + fe := zapcore.NewJSONEncoder(p) + l := zap.New(zapcore.NewTee( + zap.L().Core(), + zapcore.NewCore(fe, zapcore.AddSync(writer), zapcore.DebugLevel), + )) + return l +} diff --git a/internal/orchestrator/tasks/task.go b/internal/orchestrator/tasks/task.go new file mode 100644 index 00000000..8a37842f --- /dev/null +++ b/internal/orchestrator/tasks/task.go @@ -0,0 +1,156 @@ +package tasks + +import ( + "context" + "time" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/hook" + "github.com/garethgeorge/backrest/internal/oplog" + "github.com/garethgeorge/backrest/internal/orchestrator/repo" + "google.golang.org/protobuf/proto" +) + +var NeverScheduledTask = ScheduledTask{} + +const ( + PlanForUnassociatedOperations = "_unassociated_" + + TaskPriorityStats = -1 + TaskPriorityDefault = 0 + TaskPriorityInteractive = 1 << 1 + TaskPriorityIndexSnapshots = 1 << 2 + TaskPriorityForget = 1 << 3 + TaskPriorityPrune = 1 << 4 +) + +// TaskRunner is an interface for running tasks. It is used by tasks to create operations and write logs. +type TaskRunner interface { + // CreateOperation creates the operation in storage and sets the operation ID in the task. + CreateOperation(*v1.Operation) error + // UpdateOperation updates the operation in storage. It must be called after CreateOperation. + UpdateOperation(*v1.Operation) error + // ExecuteHooks + ExecuteHooks(events []v1.Hook_Condition, vars hook.HookVars) error + // OpLog returns the oplog for the operations. + OpLog() *oplog.OpLog + // GetRepo returns the repo with the given ID. + GetRepo(repoID string) (*v1.Repo, error) + // GetPlan returns the plan with the given ID. + GetPlan(planID string) (*v1.Plan, error) + // GetRepoOrchestrator returns the orchestrator for the repo with the given ID. + GetRepoOrchestrator(repoID string) (*repo.RepoOrchestrator, error) + // ScheduleTask schedules a task to run at a specific time. + ScheduleTask(task Task, priority int) error +} + +// ScheduledTask is a task that is scheduled to run at a specific time. +type ScheduledTask struct { + Task Task // the task to run + RunAt time.Time // the time at which the task should be run. + Op *v1.Operation // operation associated with this execution of the task. + configModno int32 // used by to detect if the config has changed since this task was scheduled. + callbacks []func(error) // callbacks to run after the task is run. +} + +func (s ScheduledTask) Eq(other ScheduledTask) bool { + return s.Task == other.Task && s.RunAt.Equal(other.RunAt) +} + +func (s ScheduledTask) Less(other ScheduledTask) bool { + if s.RunAt.Equal(other.RunAt) { + return s.Task.Name() < other.Task.Name() + } + return s.RunAt.Before(other.RunAt) +} + +func (s ScheduledTask) cancel(oplog *oplog.OpLog) error { + if s.Op == nil { + return nil + } + + opCopy := proto.Clone(s.Op).(*v1.Operation) + opCopy.Status = v1.OperationStatus_STATUS_SYSTEM_CANCELLED + opCopy.DisplayMessage = "operation cancelled" + opCopy.UnixTimeEndMs = curTimeMillis() + if err := oplog.Update(opCopy); err != nil { + return err + } + + return nil +} + +// Task is a task that can be scheduled to run at a specific time. +type Task interface { + Name() string // human readable name for this task. + Next(now time.Time, runner TaskRunner) ScheduledTask // returns the next scheduled task. + Run(ctx context.Context, st ScheduledTask, runner TaskRunner) error // run the task. + PlanID() string // the ID of the plan this task is associated with. + RepoID() string // the ID of the repo this task is associated with. +} + +type BaseTask struct { + TaskName string + TaskPlanID string + TaskRepoID string +} + +func (b BaseTask) Name() string { + return b.TaskName +} + +func (b BaseTask) PlanID() string { + return b.TaskPlanID +} + +func (b BaseTask) RepoID() string { + return b.TaskRepoID +} + +type OneoffTask struct { + BaseTask + RunAt time.Time + FlowID int64 // the ID of the flow this task is associated with. + DidSchedule bool + ProtoOp *v1.Operation // the prototype operation for this class of task. +} + +func (o *OneoffTask) Next(now time.Time, runner TaskRunner) ScheduledTask { + if o.DidSchedule { + return NeverScheduledTask + } + o.DidSchedule = true + + var op *v1.Operation + if o.ProtoOp != nil { + op = proto.Clone(o.ProtoOp).(*v1.Operation) + op.PlanId = o.PlanID() + op.RepoId = o.RepoID() + op.FlowId = o.FlowID + op.UnixTimeStartMs = timeToUnixMillis(o.RunAt) // TODO: this should be updated before Run is called. + op.Status = v1.OperationStatus_STATUS_PENDING + } + + return ScheduledTask{ + RunAt: o.RunAt, + Op: op, + } +} + +type GenericOneoffTask struct { + BaseTask + OneoffTask + Do func(ctx context.Context, st ScheduledTask, runner TaskRunner) error +} + +func (g *GenericOneoffTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner) error { + return g.Do(ctx, st, runner) +} + +func timeToUnixMillis(t time.Time) int64 { + return t.Unix()*1000 + int64(t.Nanosecond()/1000000) +} + +func curTimeMillis() int64 { + return timeToUnixMillis(time.Now()) +} diff --git a/internal/orchestrator/taskbackup.go b/internal/orchestrator/tasks/taskbackup.go similarity index 58% rename from internal/orchestrator/taskbackup.go rename to internal/orchestrator/tasks/taskbackup.go index 478b9298..d913aaa9 100644 --- a/internal/orchestrator/taskbackup.go +++ b/internal/orchestrator/tasks/taskbackup.go @@ -1,4 +1,4 @@ -package orchestrator +package tasks import ( "context" @@ -20,26 +20,24 @@ var maxBackupErrorHistoryLength = 20 // arbitrary limit on the number of file re // BackupTask is a scheduled backup operation. type BackupTask struct { - name string - TaskWithOperation - plan *v1.Plan + BaseTask scheduler func(curTime time.Time) *time.Time } var _ Task = &BackupTask{} -func NewScheduledBackupTask(orchestrator *Orchestrator, plan *v1.Plan) (*BackupTask, error) { +func NewScheduledBackupTask(plan *v1.Plan) (*BackupTask, error) { sched, err := cronexpr.ParseInLocation(plan.Cron, time.Now().Location().String()) if err != nil { return nil, fmt.Errorf("failed to parse schedule %q: %w", plan.Cron, err) } return &BackupTask{ - name: fmt.Sprintf("backup for plan %q", plan.Id), - TaskWithOperation: TaskWithOperation{ - orch: orchestrator, + BaseTask: BaseTask{ + TaskName: fmt.Sprintf("backup for plan %q", plan.Id), + TaskRepoID: plan.Repo, + TaskPlanID: plan.Id, }, - plan: plan, scheduler: func(curTime time.Time) *time.Time { next := sched.Next(curTime) return &next @@ -47,14 +45,14 @@ func NewScheduledBackupTask(orchestrator *Orchestrator, plan *v1.Plan) (*BackupT }, nil } -func NewOneoffBackupTask(orchestrator *Orchestrator, plan *v1.Plan, at time.Time) *BackupTask { +func NewOneoffBackupTask(plan *v1.Plan, at time.Time) *BackupTask { didOnce := false return &BackupTask{ - name: fmt.Sprintf("onetime backup for plan %q", plan.Id), - TaskWithOperation: TaskWithOperation{ - orch: orchestrator, + BaseTask: BaseTask{ + TaskName: fmt.Sprintf("backup for plan %q", plan.Id), + TaskRepoID: plan.Repo, + TaskPlanID: plan.Id, }, - plan: plan, scheduler: func(curTime time.Time) *time.Time { if didOnce { return nil @@ -65,72 +63,55 @@ func NewOneoffBackupTask(orchestrator *Orchestrator, plan *v1.Plan, at time.Time } } -func (t *BackupTask) Name() string { - return t.name -} - -func (t *BackupTask) Next(now time.Time) *time.Time { +func (t *BackupTask) Next(now time.Time, runner TaskRunner) ScheduledTask { next := t.scheduler(now) if next == nil { - return nil + return NeverScheduledTask } - if err := t.setOperation(&v1.Operation{ - PlanId: t.plan.Id, - RepoId: t.plan.Repo, - UnixTimeStartMs: timeToUnixMillis(*next), - Status: v1.OperationStatus_STATUS_PENDING, - Op: &v1.Operation_OperationBackup{}, - }); err != nil { - zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err) + return ScheduledTask{ + Task: t, + RunAt: *next, + Op: &v1.Operation{ + PlanId: t.PlanID(), + RepoId: t.RepoID(), + UnixTimeStartMs: (*next).UnixMilli(), + Status: v1.OperationStatus_STATUS_PENDING, + Op: &v1.Operation_OperationBackup{}, + }, } - - return next } -func (t *BackupTask) Run(ctx context.Context) error { - return t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error { - return backupHelper(ctx, t, t.orch, t.plan, op) - }) -} +func (t *BackupTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner) error { + l := Logger(ctx, st.Task) -// backupHelper does a backup. -func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan *v1.Plan, op *v1.Operation) error { startTime := time.Now() + op := st.Op backupOp := &v1.Operation_OperationBackup{ OperationBackup: &v1.OperationBackup{}, } op.Op = backupOp - zap.L().Info("Starting backup", zap.String("plan", plan.Id), zap.Int64("opId", op.Id)) - repo, err := orchestrator.GetRepo(plan.Repo) + repo, err := runner.GetRepoOrchestrator(t.RepoID()) if err != nil { - return fmt.Errorf("couldn't get repo %q: %w", plan.Repo, err) + return err } - // Run start hooks e.g. preflight checks and backup start notifications. - if err := orchestrator.hookExecutor.ExecuteHooks(repo.Config(), plan, []v1.Hook_Condition{ + plan, err := runner.GetPlan(t.PlanID()) + if err != nil { + return err + } + + if err := runner.ExecuteHooks([]v1.Hook_Condition{ v1.Hook_CONDITION_SNAPSHOT_START, - }, hook.HookVars{ - Task: t.Name(), - }); err != nil { + }, hook.HookVars{}); err != nil { var cancelErr *hook.HookErrorRequestCancel if errors.As(err, &cancelErr) { op.Status = v1.OperationStatus_STATUS_USER_CANCELLED // user visible cancelled status op.DisplayMessage = err.Error() return nil } - - // If the snapshot start hook fails we trigger error notification hooks. - retErr := fmt.Errorf("hook failed: %w", err) - _ = orchestrator.hookExecutor.ExecuteHooks(repo.Config(), plan, []v1.Hook_Condition{ - v1.Hook_CONDITION_ANY_ERROR, - }, hook.HookVars{ - Task: t.Name(), - Error: retErr.Error(), - }) - - return retErr + return fmt.Errorf("hook failed: %w", err) } var sendWg sync.WaitGroup @@ -172,13 +153,12 @@ func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan sendWg.Add(1) go func() { - if err := orchestrator.OpLog.Update(op); err != nil { + if err := runner.UpdateOperation(op); err != nil { zap.S().Errorf("failed to update oplog with progress for backup: %v", err) } sendWg.Done() }() }) - sendWg.Wait() if summary == nil { @@ -193,16 +173,17 @@ func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan if err != nil { vars.Error = err.Error() if !errors.Is(err, restic.ErrPartialBackup) { - _ = orchestrator.hookExecutor.ExecuteHooks(repo.Config(), plan, []v1.Hook_Condition{ - v1.Hook_CONDITION_SNAPSHOT_ERROR, v1.Hook_CONDITION_ANY_ERROR, + runner.ExecuteHooks([]v1.Hook_Condition{ + v1.Hook_CONDITION_SNAPSHOT_ERROR, + v1.Hook_CONDITION_ANY_ERROR, }, vars) - return fmt.Errorf("repo.Backup for repo %q: %w", plan.Repo, err) + return err } op.Status = v1.OperationStatus_STATUS_WARNING op.DisplayMessage = "Partial backup, some files may not have been read completely." } - orchestrator.hookExecutor.ExecuteHooks(repo.Config(), plan, []v1.Hook_Condition{ + runner.ExecuteHooks([]v1.Hook_Condition{ v1.Hook_CONDITION_SNAPSHOT_END, }, vars) @@ -212,14 +193,18 @@ func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan return fmt.Errorf("expected a final backup progress entry, got nil") } - zap.L().Info("Backup complete", zap.String("plan", plan.Id), zap.Duration("duration", time.Since(startTime)), zap.Any("summary", backupOp.OperationBackup.LastStatus)) + l.Info("backup complete", zap.String("plan", plan.Id), zap.Duration("duration", time.Since(startTime)), zap.Any("summary", backupOp.OperationBackup.LastStatus)) // schedule followup tasks at := time.Now() if _, ok := plan.Retention.GetPolicy().(*v1.RetentionPolicy_PolicyKeepAll); plan.Retention != nil && !ok { - orchestrator.ScheduleTask(NewOneoffForgetTask(orchestrator, plan, op.SnapshotId, at), TaskPriorityForget) + if err := runner.ScheduleTask(NewOneoffForgetTask(t.RepoID(), t.PlanID(), op.FlowId, at), TaskPriorityForget); err != nil { + return fmt.Errorf("failed to schedule forget task: %w", err) + } + } + if err := runner.ScheduleTask(NewOneoffIndexSnapshotsTask(t.RepoID(), at), TaskPriorityIndexSnapshots); err != nil { + return fmt.Errorf("failed to schedule index snapshots task: %w", err) } - orchestrator.ScheduleTask(NewOneoffIndexSnapshotsTask(orchestrator, plan.Repo, at), TaskPriorityIndexSnapshots) return nil } diff --git a/internal/orchestrator/tasks/taskcollectgarbage.go b/internal/orchestrator/tasks/taskcollectgarbage.go new file mode 100644 index 00000000..b36533bf --- /dev/null +++ b/internal/orchestrator/tasks/taskcollectgarbage.go @@ -0,0 +1,123 @@ +package tasks + +import ( + "context" + "fmt" + "time" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/oplog" + "go.uber.org/zap" +) + +const ( + gcStartupDelay = 5 * time.Second + gcInterval = 24 * time.Hour + // keep operations that are eligible for gc for 30 days OR up to a limit of 100 for any one plan. + // an operation is eligible for gc if: + // - it has no snapshot associated with it + // - it has a forgotten snapshot associated with it + gcHistoryAge = 30 * 24 * time.Hour + gcHistoryMaxCount = 1000 + // keep stats operations for 1 year (they're small and useful for long term trends) + gcHistoryStatsAge = 365 * 24 * time.Hour +) + +type CollectGarbageTask struct { + BaseTask + firstRun bool +} + +func NewCollectGarbageTask() *CollectGarbageTask { + return &CollectGarbageTask{ + BaseTask: BaseTask{ + TaskName: "collect garbage", + }, + } +} + +var _ Task = &CollectGarbageTask{} + +func (t *CollectGarbageTask) Next(now time.Time, runner TaskRunner) ScheduledTask { + if !t.firstRun { + t.firstRun = true + runAt := now.Add(gcStartupDelay) + return ScheduledTask{ + Task: t, + RunAt: runAt, + } + } + + runAt := now.Add(gcInterval) + return ScheduledTask{ + Task: t, + RunAt: runAt, + } +} + +func (t *CollectGarbageTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner) error { + oplog := runner.OpLog() + + if err := t.gcOperations(oplog); err != nil { + return fmt.Errorf("collecting garbage: %w", err) + } + + return nil +} + +func (t *CollectGarbageTask) gcOperations(oplog *oplog.OpLog) error { + // snapshotForgottenForFlow returns whether the snapshot associated with the flow is forgotten + snapshotForgottenForFlow := make(map[int64]bool) + if err := oplog.ForAll(func(op *v1.Operation) error { + if snapshotOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok { + snapshotForgottenForFlow[op.FlowId] = snapshotOp.OperationIndexSnapshot.Forgot + } + return nil + }); err != nil { + return fmt.Errorf("identifying forgotten snapshots: %w", err) + } + + forgetIDs := []int64{} + curTime := curTimeMillis() + if err := oplog.ForAll(func(op *v1.Operation) error { + forgot, ok := snapshotForgottenForFlow[op.FlowId] + if !ok { + // no snapshot associated with this flow; check if it's old enough to be gc'd + maxAgeForType := gcHistoryAge.Milliseconds() + if _, isStats := op.Op.(*v1.Operation_OperationStats); isStats { + maxAgeForType = gcHistoryStatsAge.Milliseconds() + } + if curTime-op.UnixTimeStartMs > maxAgeForType { + forgetIDs = append(forgetIDs, op.Id) + } + } else if forgot { + // snapshot is forgotten; this operation is eligible for gc + forgetIDs = append(forgetIDs, op.Id) + } + return nil + }); err != nil { + return fmt.Errorf("identifying gc eligible operations: %w", err) + } + + if err := oplog.Delete(forgetIDs...); err != nil { + return fmt.Errorf("removing gc eligible operations: %w", err) + } + + zap.L().Info("collecting garbage", + zap.Any("operations_removed", len(forgetIDs))) + return nil +} + +func (t *CollectGarbageTask) Cancel(withStatus v1.OperationStatus) error { + return nil +} + +func (t *CollectGarbageTask) OperationId() int64 { + return 0 +} + +type gcOpInfo struct { + id int64 // operation ID + timestamp int64 // unix time milliseconds + isStats bool // true if this is a stats operation +} diff --git a/internal/orchestrator/tasks/taskforget.go b/internal/orchestrator/tasks/taskforget.go new file mode 100644 index 00000000..a69b0cfc --- /dev/null +++ b/internal/orchestrator/tasks/taskforget.go @@ -0,0 +1,110 @@ +package tasks + +import ( + "context" + "fmt" + "time" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/hook" + "github.com/garethgeorge/backrest/internal/oplog/indexutil" + "github.com/hashicorp/go-multierror" + "go.uber.org/zap" +) + +func NewOneoffForgetTask(repoID, planID string, flowID int64, at time.Time) Task { + return &GenericOneoffTask{ + BaseTask: BaseTask{ + TaskName: fmt.Sprintf("forget for plan %q in repo %q", repoID, planID), + TaskRepoID: repoID, + TaskPlanID: planID, + }, + OneoffTask: OneoffTask{ + FlowID: flowID, + RunAt: at, + ProtoOp: &v1.Operation{ + Op: &v1.Operation_OperationForget{}, + }, + }, + Do: func(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error { + op := st.Op + forgetOp := op.GetOperationForget() + if forgetOp == nil { + panic("forget task with non-forget operation") + } + + if err := forgetHelper(ctx, st, taskRunner); err != nil { + taskRunner.ExecuteHooks([]v1.Hook_Condition{ + v1.Hook_CONDITION_ANY_ERROR, + }, hook.HookVars{ + Error: err.Error(), + }) + return err + } + + return nil + }, + } +} + +func forgetHelper(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error { + t := st.Task + + repo, err := taskRunner.GetRepoOrchestrator(t.RepoID()) + if err != nil { + return fmt.Errorf("get repo %q: %w", t.RepoID(), err) + } + + err = repo.UnlockIfAutoEnabled(ctx) + if err != nil { + return fmt.Errorf("auto unlock repo %q: %w", t.RepoID(), err) + } + + plan, err := taskRunner.GetPlan(t.PlanID()) + if err != nil { + return fmt.Errorf("get plan %q: %w", t.PlanID(), err) + } + + forgot, err := repo.Forget(ctx, plan) + if err != nil { + return fmt.Errorf("forget: %w", err) + } + + forgetOp := &v1.Operation_OperationForget{ + OperationForget: &v1.OperationForget{}, + } + st.Op.Op = forgetOp + + forgetOp.OperationForget.Forget = append(forgetOp.OperationForget.Forget, forgot...) + forgetOp.OperationForget.Policy = plan.Retention + + var ops []*v1.Operation + for _, forgot := range forgot { + if e := taskRunner.OpLog().ForEachBySnapshotId(forgot.Id, indexutil.CollectAll(), func(op *v1.Operation) error { + ops = append(ops, op) + return nil + }); e != nil { + err = multierror.Append(err, fmt.Errorf("cleanup snapshot %v: %w", forgot.Id, e)) + } + } + + zap.S().Debugf("found %v snapshots were forgotten, marking this in oplog", len(ops)) + + for _, op := range ops { + if indexOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok { + indexOp.OperationIndexSnapshot.Forgot = true + if e := taskRunner.UpdateOperation(op); err != nil { + err = multierror.Append(err, fmt.Errorf("mark index snapshot %v as forgotten: %w", op.Id, e)) + continue + } + } + } + + if len(forgot) > 0 { + if err := taskRunner.ScheduleTask(NewOneoffPruneTask(t.RepoID(), t.PlanID(), st.Op.FlowId, time.Now(), false), TaskPriorityPrune); err != nil { + return fmt.Errorf("schedule prune task: %w", err) + } + } + + return err +} diff --git a/internal/orchestrator/tasks/taskforgetsnapshot.go b/internal/orchestrator/tasks/taskforgetsnapshot.go new file mode 100644 index 00000000..8d565c9b --- /dev/null +++ b/internal/orchestrator/tasks/taskforgetsnapshot.go @@ -0,0 +1,67 @@ +package tasks + +import ( + "context" + "fmt" + "time" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/hook" +) + +func NewOneoffForgetSnapshotTask(repoID, planID string, flowID int64, at time.Time, snapshotID string) Task { + return &GenericOneoffTask{ + BaseTask: BaseTask{ + TaskName: fmt.Sprintf("forget snapshot %q for plan %q in repo %q", snapshotID, planID, repoID), + TaskRepoID: repoID, + TaskPlanID: planID, + }, + OneoffTask: OneoffTask{ + FlowID: flowID, + RunAt: at, + ProtoOp: &v1.Operation{ + Op: &v1.Operation_OperationForget{}, + }, + }, + Do: func(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error { + op := st.Op + forgetOp := op.GetOperationForget() + if forgetOp == nil { + panic("forget task with non-forget operation") + } + + if err := forgetSnapshotHelper(ctx, st, taskRunner, snapshotID); err != nil { + taskRunner.ExecuteHooks([]v1.Hook_Condition{ + v1.Hook_CONDITION_ANY_ERROR, + }, hook.HookVars{ + Error: err.Error(), + }) + return err + } + return nil + }, + } +} + +func forgetSnapshotHelper(ctx context.Context, st ScheduledTask, taskRunner TaskRunner, snapshotID string) error { + t := st.Task + + repo, err := taskRunner.GetRepoOrchestrator(t.RepoID()) + if err != nil { + return fmt.Errorf("get repo %q: %w", t.RepoID(), err) + } + + err = repo.UnlockIfAutoEnabled(ctx) + if err != nil { + return fmt.Errorf("auto unlock repo %q: %w", t.RepoID(), err) + } + + if err := repo.ForgetSnapshot(ctx, snapshotID); err != nil { + return fmt.Errorf("forget %q: %w", snapshotID, err) + } + + taskRunner.ScheduleTask(NewOneoffIndexSnapshotsTask(t.RepoID(), time.Now()), TaskPriorityIndexSnapshots) + taskRunner.OpLog().Delete(st.Op.Id) + st.Op = nil + return err +} diff --git a/internal/orchestrator/taskindexsnapshots.go b/internal/orchestrator/tasks/taskindexsnapshots.go similarity index 66% rename from internal/orchestrator/taskindexsnapshots.go rename to internal/orchestrator/tasks/taskindexsnapshots.go index 93d01391..dc465ab5 100644 --- a/internal/orchestrator/taskindexsnapshots.go +++ b/internal/orchestrator/tasks/taskindexsnapshots.go @@ -1,4 +1,4 @@ -package orchestrator +package tasks import ( "context" @@ -14,77 +14,54 @@ import ( "go.uber.org/zap" ) -// IndexSnapshotsTask tracks a forget operation. -type IndexSnapshotsTask struct { - orchestrator *Orchestrator // owning orchestrator - repoId string - at *time.Time -} - -var _ Task = &IndexSnapshotsTask{} - -func NewOneoffIndexSnapshotsTask(orchestrator *Orchestrator, repoId string, at time.Time) *IndexSnapshotsTask { - return &IndexSnapshotsTask{ - orchestrator: orchestrator, - repoId: repoId, - at: &at, +func NewOneoffIndexSnapshotsTask(repoID string, at time.Time) Task { + return &GenericOneoffTask{ + BaseTask: BaseTask{ + TaskName: fmt.Sprintf("index snapshots for repo %q", repoID), + TaskRepoID: repoID, + }, + OneoffTask: OneoffTask{ + RunAt: at, + ProtoOp: nil, + }, + Do: func(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error { + if err := indexSnapshotsHelper(ctx, st, taskRunner); err != nil { + taskRunner.ExecuteHooks([]v1.Hook_Condition{ + v1.Hook_CONDITION_ANY_ERROR, + }, hook.HookVars{ + Task: st.Task.Name(), + Error: err.Error(), + }) + return err + } + return nil + }, } } -func (t *IndexSnapshotsTask) Name() string { - return fmt.Sprintf("index snapshots for plan %q", t.repoId) -} - -func (t *IndexSnapshotsTask) Next(now time.Time) *time.Time { - ret := t.at - if ret != nil { - t.at = nil - } - return ret -} - -func (t *IndexSnapshotsTask) Run(ctx context.Context) error { - if err := indexSnapshotsHelper(ctx, t.orchestrator, t.repoId); err != nil { - repo, _ := t.orchestrator.GetRepo(t.repoId) - t.orchestrator.hookExecutor.ExecuteHooks(repo.Config(), nil, []v1.Hook_Condition{ - v1.Hook_CONDITION_ANY_ERROR, - }, hook.HookVars{ - Task: t.Name(), - Error: err.Error(), - }) - return err - } - return nil -} - -func (t *IndexSnapshotsTask) Cancel(withStatus v1.OperationStatus) error { - return nil -} - -func (t *IndexSnapshotsTask) OperationId() int64 { - return 0 -} - // indexSnapshotsHelper indexes all snapshots for a plan. // - If the snapshot is already indexed, it is skipped. // - If the snapshot is not indexed, an index snapshot operation with it's metadata is added. // - If an index snapshot operation is found for a snapshot that is not returned by the repo, it is marked as forgotten. -func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, repoId string) error { - repo, err := orchestrator.GetRepo(repoId) +func indexSnapshotsHelper(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error { + t := st.Task + oplog := taskRunner.OpLog() + + repo, err := taskRunner.GetRepoOrchestrator(t.RepoID()) if err != nil { - return fmt.Errorf("couldn't get repo %q: %w", repoId, err) + return fmt.Errorf("couldn't get repo %q: %w", t.RepoID(), err) } // collect all tracked snapshots for the plan. snapshots, err := repo.Snapshots(ctx) if err != nil { - return fmt.Errorf("get snapshots for repo %q: %w", repoId, err) + return fmt.Errorf("get snapshots for repo %q: %w", t.RepoID(), err) } // collect all current snapshot IDs. - currentIds, err := indexCurrentSnapshotIdsForRepo(orchestrator.OpLog, repoId) + currentIds, err := indexCurrentSnapshotIdsForRepo(taskRunner.OpLog(), t.RepoID()) if err != nil { - return fmt.Errorf("get known snapshot IDs for repo %q: %w", repoId, err) + return fmt.Errorf("get known snapshot IDs for repo %q: %w", t.RepoID(), err) } foundIds := make(map[string]bool) @@ -99,10 +76,15 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, repoI } snapshotProto := protoutil.SnapshotToProto(snapshot) + flowID, err := FlowIDForSnapshotID(taskRunner.OpLog(), snapshot.Id) + if err != nil { + return fmt.Errorf("get flow ID for snapshot %q: %w", snapshot.Id, err) + } planId := planForSnapshot(snapshotProto) indexOps = append(indexOps, &v1.Operation{ - RepoId: repoId, + RepoId: t.RepoID(), PlanId: planId, + FlowId: flowID, UnixTimeStartMs: snapshotProto.UnixTimeMs, UnixTimeEndMs: snapshotProto.UnixTimeMs, Status: v1.OperationStatus_STATUS_SUCCESS, @@ -115,7 +97,7 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, repoI }) } - if err := orchestrator.OpLog.BulkAdd(indexOps); err != nil { + if err := taskRunner.OpLog().BulkAdd(indexOps); err != nil { return fmt.Errorf("BulkAdd snapshot operations: %w", err) } @@ -127,7 +109,7 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, repoI } // mark snapshot forgotten. - op, err := orchestrator.OpLog.Get(opId) + op, err := oplog.Get(opId) if err != nil { // should only be possible in the case of a data race (e.g. operation was somehow deleted). return fmt.Errorf("get operation %v: %w", opId, err) @@ -139,14 +121,14 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, repoI } snapshotOp.OperationIndexSnapshot.Forgot = true - if err := orchestrator.OpLog.Update(op); err != nil { + if err := oplog.Update(op); err != nil { return fmt.Errorf("mark index snapshot operation %v as forgotten: %w", opId, err) } } // Print stats at the end of indexing. zap.L().Debug("indexed snapshots", - zap.String("repo", repoId), + zap.String("repo", t.RepoID()), zap.Duration("duration", time.Since(startTime)), zap.Int("alreadyIndexed", len(foundIds)), zap.Int("newlyAdded", len(indexOps)), diff --git a/internal/orchestrator/tasks/taskprune.go b/internal/orchestrator/tasks/taskprune.go new file mode 100644 index 00000000..8fca81eb --- /dev/null +++ b/internal/orchestrator/tasks/taskprune.go @@ -0,0 +1,180 @@ +package tasks + +import ( + "bytes" + "context" + "fmt" + "sync" + "time" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/hook" + "github.com/garethgeorge/backrest/internal/oplog" + "github.com/garethgeorge/backrest/internal/oplog/indexutil" + "go.uber.org/zap" +) + +type PruneTask struct { + BaseTask + OneoffTask + force bool +} + +func NewOneoffPruneTask(repoID, planID string, flowID int64, at time.Time, force bool) Task { + return &PruneTask{ + BaseTask: BaseTask{ + TaskName: fmt.Sprintf("prune for plan %q in repo %q", planID, repoID), + TaskRepoID: repoID, + TaskPlanID: planID, + }, + OneoffTask: OneoffTask{ + FlowID: flowID, + RunAt: at, + ProtoOp: &v1.Operation{ + Op: &v1.Operation_OperationPrune{}, + }, + }, + force: force, + } +} + +func (t *PruneTask) Next(now time.Time, runner TaskRunner) ScheduledTask { + if t.force { + return t.OneoffTask.Next(now, runner) + } + + shouldRun, err := t.shouldRun(now, runner) + if err != nil { + zap.S().Errorf("task %v failed to check if it should run: %v", t.Name(), err) + return NeverScheduledTask + } + if !shouldRun { + return NeverScheduledTask + } + + return t.OneoffTask.Next(now, runner) +} + +func (t *PruneTask) shouldRun(now time.Time, runner TaskRunner) (bool, error) { + repo, err := runner.GetRepo(t.RepoID()) + if err != nil { + return false, fmt.Errorf("get repo %v: %w", t.RepoID(), err) + } + + nextPruneTime, err := t.getNextPruneTime(runner, repo.PrunePolicy) + if err != nil { + return false, fmt.Errorf("get next prune time: %w", err) + } + + return nextPruneTime.Before(now), nil +} + +func (t *PruneTask) getNextPruneTime(runner TaskRunner, policy *v1.PrunePolicy) (time.Time, error) { + var lastPruneTime time.Time + runner.OpLog().ForEachByRepo(t.RepoID(), indexutil.Reversed(indexutil.CollectAll()), func(op *v1.Operation) error { + if _, ok := op.Op.(*v1.Operation_OperationPrune); ok { + lastPruneTime = time.Unix(0, op.UnixTimeStartMs*int64(time.Millisecond)) + return oplog.ErrStopIteration + } + return nil + }) + + if policy != nil { + return lastPruneTime.Add(time.Duration(policy.MaxFrequencyDays) * 24 * time.Hour), nil + } else { + return lastPruneTime.Add(7 * 24 * time.Hour), nil // default to 7 days. + } +} + +func (t *PruneTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner) error { + op := st.Op + + repo, err := runner.GetRepoOrchestrator(t.RepoID()) + if err != nil { + return fmt.Errorf("couldn't get repo %q: %w", t.RepoID(), err) + } + + err = repo.UnlockIfAutoEnabled(ctx) + if err != nil { + return fmt.Errorf("auto unlock repo %q: %w", t.RepoID(), err) + } + + opPrune := &v1.Operation_OperationPrune{ + OperationPrune: &v1.OperationPrune{}, + } + op.Op = opPrune + + ctx, cancel := context.WithCancel(ctx) + interval := time.NewTicker(1 * time.Second) + defer interval.Stop() + var buf synchronizedBuffer + var wg sync.WaitGroup + wg.Add(1) + go func() { + + defer wg.Done() + for { + select { + case <-interval.C: + output := buf.String() + if len(output) > 8*1024 { // only provide live status upto the first 8K of output. + output = output[:len(output)-8*1024] + } + + if opPrune.OperationPrune.Output != output { + opPrune.OperationPrune.Output = buf.String() + + if err := runner.OpLog().Update(op); err != nil { + zap.L().Error("update prune operation with status output", zap.Error(err)) + } + } + case <-ctx.Done(): + return + } + } + }() + + if err := repo.Prune(ctx, &buf); err != nil { + cancel() + + runner.ExecuteHooks([]v1.Hook_Condition{ + v1.Hook_CONDITION_ANY_ERROR, + }, hook.HookVars{ + Error: err.Error(), + }) + + return fmt.Errorf("prune: %w", err) + } + cancel() + wg.Wait() + + // TODO: it would be best to store the output in separate storage for large status data. + output := buf.String() + if len(output) > 8*1024 { // only save the first 4K of output. + output = output[:len(output)-8*1024] + } + + opPrune.OperationPrune.Output = output + + return nil +} + +// synchronizedBuffer is used for collecting prune command's output +type synchronizedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (w *synchronizedBuffer) Write(p []byte) (n int, err error) { + w.mu.Lock() + defer w.mu.Unlock() + + return w.buf.Write(p) +} + +func (w *synchronizedBuffer) String() string { + w.mu.Lock() + defer w.mu.Unlock() + + return w.buf.String() +} diff --git a/internal/orchestrator/tasks/taskrestore.go b/internal/orchestrator/tasks/taskrestore.go new file mode 100644 index 00000000..7021cd06 --- /dev/null +++ b/internal/orchestrator/tasks/taskrestore.go @@ -0,0 +1,92 @@ +package tasks + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/hook" + "go.uber.org/zap" +) + +func NewOneoffRestoreTask(repoID, planID string, flowID int64, at time.Time, snapshotID, path, target string) Task { + return &GenericOneoffTask{ + BaseTask: BaseTask{ + TaskName: fmt.Sprintf("restore snapshot %q in repo %q", snapshotID, repoID), + TaskRepoID: repoID, + TaskPlanID: planID, + }, + OneoffTask: OneoffTask{ + FlowID: flowID, + RunAt: at, + ProtoOp: &v1.Operation{ + SnapshotId: snapshotID, + Op: &v1.Operation_OperationRestore{}, + }, + }, + Do: func(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error { + if err := restoreHelper(ctx, st, taskRunner, snapshotID, path, target); err != nil { + taskRunner.ExecuteHooks([]v1.Hook_Condition{ + v1.Hook_CONDITION_ANY_ERROR, + }, hook.HookVars{ + Task: st.Task.Name(), + Error: err.Error(), + }) + return err + } + return nil + }, + } +} + +func restoreHelper(ctx context.Context, st ScheduledTask, taskRunner TaskRunner, snapshotID, path, target string) error { + t := st.Task + oplog := taskRunner.OpLog() + op := st.Op + + if snapshotID == "" || path == "" || target == "" { + return errors.New("snapshotID, path, and target are required") + } + + restoreOp := st.Op.GetOperationRestore() + if restoreOp == nil { + return errors.New("operation is not a restore operation") + } + + repo, err := taskRunner.GetRepoOrchestrator(t.RepoID()) + if err != nil { + return fmt.Errorf("couldn't get repo %q: %w", t.RepoID(), err) + } + + var sendWg sync.WaitGroup + lastSent := time.Now() // debounce progress updates, these can endup being very frequent. + summary, err := repo.Restore(ctx, snapshotID, path, target, func(entry *v1.RestoreProgressEntry) { + sendWg.Wait() + if time.Since(lastSent) < 1*time.Second { + return + } + lastSent = time.Now() + + zap.S().Infof("restore progress: %v", entry) + + restoreOp.Status = entry + + sendWg.Add(1) + go func() { + if err := oplog.Update(op); err != nil { + zap.S().Errorf("failed to update oplog with progress for restore: %v", err) + } + sendWg.Done() + }() + }) + + if err != nil { + return fmt.Errorf("restore failed: %w", err) + } + restoreOp.Status = summary + + return nil +} diff --git a/internal/orchestrator/tasks/taskstats.go b/internal/orchestrator/tasks/taskstats.go new file mode 100644 index 00000000..214d0bdd --- /dev/null +++ b/internal/orchestrator/tasks/taskstats.go @@ -0,0 +1,61 @@ +package tasks + +import ( + "context" + "fmt" + "time" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/hook" +) + +func NewOneoffStatsTask(repoID, planID string, at time.Time) Task { + return &GenericOneoffTask{ + BaseTask: BaseTask{ + TaskName: fmt.Sprintf("stats for repo %q", repoID), + TaskRepoID: repoID, + TaskPlanID: planID, + }, + OneoffTask: OneoffTask{ + RunAt: at, + ProtoOp: &v1.Operation{ + Op: &v1.Operation_OperationStats{}, + }, + }, + Do: func(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error { + if err := statsHelper(ctx, st, taskRunner); err != nil { + taskRunner.ExecuteHooks([]v1.Hook_Condition{ + v1.Hook_CONDITION_ANY_ERROR, + }, hook.HookVars{ + Task: st.Task.Name(), + Error: err.Error(), + }) + return err + } + return nil + }, + } +} + +func statsHelper(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error { + t := st.Task + + repo, err := taskRunner.GetRepoOrchestrator(t.RepoID()) + if err != nil { + return fmt.Errorf("get repo %q: %w", t.RepoID(), err) + } + + stats, err := repo.Stats(ctx) + if err != nil { + return fmt.Errorf("get stats: %w", err) + } + + op := st.Op + op.Op = &v1.Operation_OperationStats{ + OperationStats: &v1.OperationStats{ + Stats: stats, + }, + } + + return nil +} diff --git a/internal/orchestrator/taskstats.go b/internal/orchestrator/taskstats.go deleted file mode 100644 index 07bc666b..00000000 --- a/internal/orchestrator/taskstats.go +++ /dev/null @@ -1,88 +0,0 @@ -package orchestrator - -import ( - "context" - "fmt" - "time" - - v1 "github.com/garethgeorge/backrest/gen/go/v1" - "github.com/garethgeorge/backrest/internal/hook" - "go.uber.org/zap" -) - -// StatsTask tracks a restic stats operation. -type StatsTask struct { - TaskWithOperation - planId string - repoId string - at *time.Time -} - -var _ Task = &StatsTask{} - -func NewOneoffStatsTask(orchestrator *Orchestrator, repoId, planId string, at time.Time) *StatsTask { - return &StatsTask{ - TaskWithOperation: TaskWithOperation{ - orch: orchestrator, - }, - planId: planId, - repoId: repoId, - at: &at, - } -} - -func (t *StatsTask) Name() string { - return fmt.Sprintf("stats for repo %q", t.repoId) -} - -func (t *StatsTask) Next(now time.Time) *time.Time { - ret := t.at - if ret != nil { - t.at = nil - - if err := t.setOperation(&v1.Operation{ - PlanId: t.planId, - RepoId: t.repoId, - UnixTimeStartMs: timeToUnixMillis(*ret), - Status: v1.OperationStatus_STATUS_PENDING, - Op: &v1.Operation_OperationStats{}, - }); err != nil { - zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err) - return nil - } - } - return ret -} - -func (t *StatsTask) Run(ctx context.Context) error { - if err := t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error { - repo, err := t.orch.GetRepo(t.repoId) - if err != nil { - return fmt.Errorf("get repo %q: %w", t.repoId, err) - } - - stats, err := repo.Stats(ctx) - if err != nil { - return fmt.Errorf("get stats: %w", err) - } - - op.Op = &v1.Operation_OperationStats{ - OperationStats: &v1.OperationStats{ - Stats: stats, - }, - } - - return err - }); err != nil { - repo, _ := t.orch.GetRepo(t.repoId) - plan, _ := t.orch.GetPlan(t.planId) - _ = t.orch.hookExecutor.ExecuteHooks(repo.Config(), plan, []v1.Hook_Condition{ - v1.Hook_CONDITION_ANY_ERROR, - }, hook.HookVars{ - Task: t.Name(), - Error: err.Error(), - }) - return err - } - return nil -} diff --git a/internal/protoutil/validation.go b/internal/protoutil/validation.go index af10b33b..3ef89887 100644 --- a/internal/protoutil/validation.go +++ b/internal/protoutil/validation.go @@ -13,6 +13,9 @@ func ValidateOperation(op *v1.Operation) error { if op.Id == 0 { return errors.New("operation.id is required") } + if op.FlowId == 0 { + return errors.New("operation.flow_id is required") + } if op.RepoId == "" { return errors.New("operation.repo_id is required") } diff --git a/proto/v1/operations.proto b/proto/v1/operations.proto index cc9388d0..4190db75 100644 --- a/proto/v1/operations.proto +++ b/proto/v1/operations.proto @@ -14,6 +14,8 @@ message OperationList { message Operation { // required, primary ID of the operation. ID is sequential based on creation time of the operation. int64 id = 1; + // flow id groups operations together, e.g. by an execution of a plan. + int64 flow_id = 10; // optional, flow id if associated with a flow // required, repo id if associated with a repo string repo_id = 2; // required, plan id if associated with a plan @@ -101,5 +103,6 @@ message OperationStats { message OperationRunHook { string name = 1; // description of the hook that was run. typically repo/hook_idx or plan/hook_idx. - string output_logref = 2; // logref of the hook's output. + string output_logref = 2; // logref of the hook's output. DEPRECATED. + Hook.Condition condition = 3; // triggering condition of the hook. } diff --git a/proto/v1/service.proto b/proto/v1/service.proto index 2343f5c5..ca473eea 100644 --- a/proto/v1/service.proto +++ b/proto/v1/service.proto @@ -82,6 +82,7 @@ message GetOperationsRequest { string repo_id = 1; string plan_id = 2; string snapshot_id = 4; + int64 flow_id = 6; repeated int64 ids = 5; int64 last_n = 3; // limit to the last n operations } diff --git a/webui/gen/ts/v1/operations_pb.ts b/webui/gen/ts/v1/operations_pb.ts index 0f47879f..5bc0bde3 100644 --- a/webui/gen/ts/v1/operations_pb.ts +++ b/webui/gen/ts/v1/operations_pb.ts @@ -6,7 +6,7 @@ import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf"; import { Message, proto3, protoInt64 } from "@bufbuild/protobuf"; import { BackupProgressEntry, BackupProgressError, RepoStats, ResticSnapshot, RestoreProgressEntry } from "./restic_pb.js"; -import { RetentionPolicy } from "./config_pb.js"; +import { Hook_Condition, RetentionPolicy } from "./config_pb.js"; /** * OperationEventType indicates whether the operation was created or updated @@ -162,6 +162,15 @@ export class Operation extends Message { */ id = protoInt64.zero; + /** + * flow id groups operations together, e.g. by an execution of a plan. + * + * optional, flow id if associated with a flow + * + * @generated from field: int64 flow_id = 10; + */ + flowId = protoInt64.zero; + /** * required, repo id if associated with a repo * @@ -272,6 +281,7 @@ export class Operation extends Message { static readonly typeName = "v1.Operation"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "id", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 10, name: "flow_id", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, { no: 2, name: "repo_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 3, name: "plan_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 8, name: "snapshot_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, @@ -641,12 +651,19 @@ export class OperationRunHook extends Message { name = ""; /** - * logref of the hook's output. + * logref of the hook's output. DEPRECATED. * * @generated from field: string output_logref = 2; */ outputLogref = ""; + /** + * triggering condition of the hook. + * + * @generated from field: v1.Hook.Condition condition = 3; + */ + condition = Hook_Condition.UNKNOWN; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -657,6 +674,7 @@ export class OperationRunHook extends Message { static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "name", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 2, name: "output_logref", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 3, name: "condition", kind: "enum", T: proto3.getEnumType(Hook_Condition) }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): OperationRunHook { diff --git a/webui/gen/ts/v1/service_pb.ts b/webui/gen/ts/v1/service_pb.ts index ec264f72..fd18b000 100644 --- a/webui/gen/ts/v1/service_pb.ts +++ b/webui/gen/ts/v1/service_pb.ts @@ -172,6 +172,11 @@ export class GetOperationsRequest extends Message { */ snapshotId = ""; + /** + * @generated from field: int64 flow_id = 6; + */ + flowId = protoInt64.zero; + /** * @generated from field: repeated int64 ids = 5; */ @@ -195,6 +200,7 @@ export class GetOperationsRequest extends Message { { no: 1, name: "repo_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 2, name: "plan_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 4, name: "snapshot_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 6, name: "flow_id", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, { no: 5, name: "ids", kind: "scalar", T: 3 /* ScalarType.INT64 */, repeated: true }, { no: 3, name: "last_n", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, ]); diff --git a/webui/src/components/OperationRow.tsx b/webui/src/components/OperationRow.tsx index e61ecdaf..b845c530 100644 --- a/webui/src/components/OperationRow.tsx +++ b/webui/src/components/OperationRow.tsx @@ -44,6 +44,8 @@ import { LogDataRequest } from "../../gen/ts/v1/service_pb"; import { MessageInstance } from "antd/es/message/interface"; import { backrestService } from "../api"; import { useShowModal } from "./ModalManager"; +import { proto3 } from "@bufbuild/protobuf"; +import { Hook_Condition } from "../../gen/ts/v1/config_pb"; export const OperationRow = ({ @@ -127,6 +129,7 @@ export const OperationRow = ({ } let body: React.ReactNode | undefined; + let displayMessage = operation.displayMessage; if (operation.op.case === "operationBackup") { const backupOp = operation.op.value; @@ -194,14 +197,18 @@ export const OperationRow = ({ ); } else if (operation.op.case === "operationRunHook") { - body = <> + const hook = operation.op.value; + const triggeringCondition = proto3.getEnumType(Hook_Condition).findNumber(hook.condition); + if (triggeringCondition !== undefined) { + displayMessage += "\ntriggered by condition: " + triggeringCondition.name; + } } const children = []; if (operation.displayMessage) { children.push(<> -
{details.state ? details.state + ": " : null}{operation.displayMessage}
+
{details.state ? details.state + ": " : null}{displayMessage}
); } diff --git a/webui/src/components/OperationTree.tsx b/webui/src/components/OperationTree.tsx index bce76b91..2d56027b 100644 --- a/webui/src/components/OperationTree.tsx +++ b/webui/src/components/OperationTree.tsx @@ -133,6 +133,11 @@ export const OperationTree = ({ const b = node.backup; const details: string[] = []; + if (b.operations.length === 0) { + // this happens when all operations in a backup are deleted; it should be hidden until the deletion propagates to a refresh of the tree layout. + return null; + } + if (b.status === OperationStatus.STATUS_PENDING) { details.push("scheduled, waiting"); } else if (b.status === OperationStatus.STATUS_SYSTEM_CANCELLED) { diff --git a/webui/src/components/SnapshotBrowser.tsx b/webui/src/components/SnapshotBrowser.tsx index 71b5d871..c868c9a3 100644 --- a/webui/src/components/SnapshotBrowser.tsx +++ b/webui/src/components/SnapshotBrowser.tsx @@ -222,7 +222,7 @@ const FileNode = ({ entry }: { entry: LsEntry }) => { ({formatBytes(Number(entry.size))}) ) : null} - {dropdown} + {dropdown ||
} ); }; diff --git a/webui/src/state/oplog.ts b/webui/src/state/oplog.ts index fe2111e4..ed5cece0 100644 --- a/webui/src/state/oplog.ts +++ b/webui/src/state/oplog.ts @@ -9,7 +9,7 @@ import { BackupProgressEntry, ResticSnapshot } from "../../gen/ts/v1/restic_pb"; import _ from "lodash"; import { formatDuration, formatTime } from "../lib/formatting"; import { backrestService } from "../api"; -import { STATS_OPERATION_HISTORY } from "../constants"; +import { STATS_OPERATION_HISTORY, STATUS_OPERATION_HISTORY } from "../constants"; const subscribers: ((event: OperationEvent) => void)[] = []; @@ -57,7 +57,7 @@ export const unsubscribeFromOperations = ( export const getStatusForPlan = async (plan: string) => { const req = new GetOperationsRequest({ planId: plan, - lastN: BigInt(STATS_OPERATION_HISTORY), + lastN: BigInt(STATUS_OPERATION_HISTORY), }); return await getStatus(req); }; @@ -65,7 +65,7 @@ export const getStatusForPlan = async (plan: string) => { export const getStatusForRepo = async (repo: string) => { const req = new GetOperationsRequest({ repoId: repo, - lastN: BigInt(STATS_OPERATION_HISTORY), + lastN: BigInt(STATUS_OPERATION_HISTORY), }); return await getStatus(req); }; @@ -79,12 +79,12 @@ const getStatus = async (req: GetOperationsRequest) => { if (ops.length === 0) { return OperationStatus.STATUS_SUCCESS; } - const snapshotId = ops[0].snapshotId; + const flowId = ops[0].flowId; for (const op of ops) { if (op.status === OperationStatus.STATUS_PENDING) { continue; } - if (op.snapshotId !== snapshotId) { + if (op.flowId !== flowId) { break; } if ( @@ -110,7 +110,7 @@ export enum DisplayType { } export interface BackupInfo { - id: string; // id of the first operation that generated this backup. + id: string; // flow ID of the operations that make up this backup. displayTime: Date; displayType: DisplayType; startTimeMs: number; @@ -132,8 +132,10 @@ export class BackupInfoCollector { event: OperationEventType, info: BackupInfo[] ) => void)[] = []; - private backupByOpId: Map = new Map(); - private backupBySnapshotId: Map = new Map(); + + // backups maps a flow ID to a backup info object. + private operationsByFlowId: Map = new Map(); + private backupsByFlowId: Map = new Map(); /** * @@ -164,6 +166,10 @@ export class BackupInfoCollector { let statusIdx = operations.length - 1; let status = OperationStatus.STATUS_SYSTEM_CANCELLED; while (statusIdx !== -1) { + if (operations[statusIdx].op.case === "operationRunHook") { + statusIdx--; + continue; + } const curStatus = operations[statusIdx].status; if ( shouldHideStatus(status) || @@ -179,17 +185,19 @@ export class BackupInfoCollector { let backupLastStatus = undefined; let snapshotInfo = undefined; let forgotten = false; + let snapshotId = ""; for (const op of operations) { if (op.op.case === "operationBackup") { backupLastStatus = op.op.value.lastStatus; } else if (op.op.case === "operationIndexSnapshot") { snapshotInfo = op.op.value.snapshot; forgotten = op.op.value.forgot || false; + snapshotId = op.op.value.snapshot?.id || ""; } } return { - id: id.toString(16), + id: operations[0].flowId.toString(16), startTimeMs, endTimeMs, displayTime, @@ -199,38 +207,38 @@ export class BackupInfoCollector { backupLastStatus, snapshotInfo, forgotten, - snapshotId: operations[0].snapshotId, + snapshotId: snapshotId, planId: operations[0].planId, repoId: operations[0].repoId, }; } - private addHelper(op: Operation): BackupInfo { - if (op.snapshotId) { - this.backupByOpId.delete(op.id); - - const existing = this.backupBySnapshotId.get(op.snapshotId); - let operations: Operation[]; - if (existing) { - operations = [...existing.operations]; - const opIdx = operations.findIndex((o) => o.id === op.id); - if (opIdx > -1) { - operations[opIdx] = op; - } else { - operations.push(op); - } - } else { - operations = [op]; - } - - const newInfo = this.createBackup(operations); - this.backupBySnapshotId.set(op.snapshotId, newInfo); - return newInfo; + private addOrUpdateHelper(op: Operation) { + const existing = this.operationsByFlowId.get(op.flowId); + if (existing === undefined) { + this.operationsByFlowId.set(op.flowId, [op]); } else { - const newInfo = this.createBackup([op]); - this.backupByOpId.set(op.id, newInfo); - return newInfo; + const idx = existing.findIndex((o) => o.id === op.id); + if (idx === -1) { + existing.push(op); + } else { + existing[idx] = op; + } } + this.backupsByFlowId.delete(op.flowId); + } + + private getBackupInfo(flowId: bigint): BackupInfo | undefined { + let existing = this.backupsByFlowId.get(flowId); + if (existing === undefined) { + const operations = this.operationsByFlowId.get(flowId); + if (!operations) { + return undefined; + } + existing = this.createBackup(operations); + this.backupsByFlowId.set(flowId, existing); + } + return existing; } public addOperation(event: OperationEventType, op: Operation): BackupInfo | null { @@ -238,26 +246,27 @@ export class BackupInfoCollector { this.removeOperation(op); return null; } - const backupInfo = this.addHelper(op); + this.addOrUpdateHelper(op); + const backupInfo = this.getBackupInfo(op.flowId)!; this.listeners.forEach((l) => l(event, [backupInfo])); return backupInfo; } // removeOperaiton is not quite correct from a formal standpoint; but will look correct in the UI. public removeOperation(op: Operation) { - if (op.snapshotId) { - let existing = this.backupBySnapshotId.get(op.snapshotId); - if (existing) { - const operations = existing.operations.filter((o) => o.id !== op.id); - if (operations.length === 0) { - this.backupBySnapshotId.delete(op.snapshotId); - } else { - this.backupBySnapshotId.set(op.snapshotId, this.createBackup(operations)); - } - } - } else { - this.backupByOpId.delete(op.id); + const existing = this.operationsByFlowId.get(op.flowId); + if (existing === undefined) { + return; } + const idx = existing.findIndex((o) => o.id === op.id); + if (idx === -1) { + return; + } + existing.splice(idx, 1); + if (existing.length === 0) { + this.operationsByFlowId.delete(op.flowId); + } + this.backupsByFlowId.delete(op.flowId); // delete the cache for lazy recomputation. this.listeners.forEach((l) => l(OperationEventType.EVENT_DELETED, this.getAll()) @@ -265,43 +274,20 @@ export class BackupInfoCollector { } public bulkAddOperations(ops: Operation[]): BackupInfo[] { - ops = ops.filter(this.filter); - let grouped = _.groupBy(ops, (op) => - op.snapshotId ? op.snapshotId : op.id - ); - - const info: BackupInfo[] = []; - - for (const operations of Object.values(grouped)) { - if (operations[0].snapshotId) { - const existing = this.backupBySnapshotId.get(operations[0].snapshotId); - if (existing) { - const newOps = [...existing.operations, ...operations]; - const newInfo = this.createBackup(newOps); - this.backupBySnapshotId.set(operations[0].snapshotId, newInfo); - info.push(newInfo); - } else { - const newInfo = this.createBackup(operations); - this.backupBySnapshotId.set(operations[0].snapshotId, newInfo); - info.push(newInfo); - } - } else { - const op = operations[0]; - const newInfo = this.createBackup([op]); - this.backupByOpId.set(op.id, newInfo); - info.push(newInfo); - } + for (const op of ops) { + this.addOrUpdateHelper(op); } - + const flowIDs = _.uniq(ops.map((op) => op.flowId)); + const info = flowIDs.map((flowId) => this.getBackupInfo(flowId)!); this.listeners.forEach((l) => l(OperationEventType.EVENT_CREATED, info)); return info; } public getAll(): BackupInfo[] { - const arr = [ - ...this.backupByOpId.values(), - ...this.backupBySnapshotId.values(), - ]; + const arr = [] + for (const key of this.operationsByFlowId.keys()) { + arr.push(this.getBackupInfo(key)!); + } return arr.filter((b) => !b.forgotten && !shouldHideStatus(b.status)); } @@ -325,8 +311,6 @@ export class BackupInfoCollector { export const shouldHideOperation = (operation: Operation) => { return ( operation.op.case === "operationStats" || - (operation.op.case === "operationRunHook" && - operation.status === OperationStatus.STATUS_SUCCESS) || shouldHideStatus(operation.status) ); };