lee/backrest
1
0
Fork 0
mirror of https://github.com/garethgeorge/backrest.git synced 2026-08-28 11:56:28 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: overhaul task interface and introduce 'flow ID' for simpler grouping of operations (#253)

Browse Source
This commit is contained in:
Gareth
2024-04-28 23:05:13 -07:00
committed by GitHub
parent 84b4b68760
commit 7a10bdca7b
44 changed files with 2001 additions and 1768 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gen/go/v1/operations.pb.go
+150 -127
View File
@@ -191,6 +191,8 @@ type Operation struct {
// required, primary ID of the operation. ID is sequential based on creation time of the operation.
Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
// flow id groups operations together, e.g. by an execution of a plan.
FlowId int64 `protobuf:"varint,10,opt,name=flow_id,json=flowId,proto3" json:"flow_id,omitempty"` // optional, flow id if associated with a flow
// required, repo id if associated with a repo
RepoId string `protobuf:"bytes,2,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"`
// required, plan id if associated with a plan
@@ -257,6 +259,13 @@ func (x *Operation) GetId() int64 {
return 0
}
func (x *Operation) GetFlowId() int64 {
if x != nil {
return x.FlowId
}
return 0
}
func (x *Operation) GetRepoId() string {
if x != nil {
return x.RepoId
@@ -809,8 +818,9 @@ type OperationRunHook struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // description of the hook that was run. typically repo/hook_idx or plan/hook_idx.
OutputLogref string `protobuf:"bytes,2,opt,name=output_logref,json=outputLogref,proto3" json:"output_logref,omitempty"` // logref of the hook's output.
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // description of the hook that was run. typically repo/hook_idx or plan/hook_idx.
OutputLogref string `protobuf:"bytes,2,opt,name=output_logref,json=outputLogref,proto3" json:"output_logref,omitempty"` // logref of the hook's output. DEPRECATED.
Condition Hook_Condition `protobuf:"varint,3,opt,name=condition,proto3,enum=v1.Hook_Condition" json:"condition,omitempty"` // triggering condition of the hook.
}
func (x *OperationRunHook) Reset() {
@@ -859,6 +869,13 @@ func (x *OperationRunHook) GetOutputLogref() string {
return ""
}
func (x *OperationRunHook) GetCondition() Hook_Condition {
if x != nil {
return x.Condition
}
return Hook_CONDITION_UNKNOWN
}
var File_v1_operations_proto protoreflect.FileDescriptor
var file_v1_operations_proto_rawDesc = []byte{
@@ -869,127 +886,131 @@ var file_v1_operations_proto_rawDesc = []byte{
0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x0a,
0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b,
0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52,
0x0a, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x9d, 0x06, 0x0a, 0x09,
0x0a, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xb6, 0x06, 0x0a, 0x09,
0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18,
0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70,
0x6f, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f,
0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20,
0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73,
0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09,
0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x06,
0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x76,
0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75,
0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x12, 0x75, 0x6e, 0x69,
0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6d, 0x73, 0x18,
0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x75, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x53,
0x74, 0x61, 0x72, 0x74, 0x4d, 0x73, 0x12, 0x27, 0x0a, 0x10, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x74,
0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x6d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03,
0x52, 0x0d, 0x75, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x45, 0x6e, 0x64, 0x4d, 0x73, 0x12,
0x27, 0x0a, 0x0f, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61,
0x67, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61,
0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x6f, 0x67, 0x72,
0x65, 0x66, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x6f, 0x67, 0x72, 0x65, 0x66,
0x12, 0x40, 0x0a, 0x10, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x62, 0x61,
0x63, 0x6b, 0x75, 0x70, 0x18, 0x64, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e,
0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x48,
0x00, 0x52, 0x0f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b,
0x75, 0x70, 0x12, 0x56, 0x0a, 0x18, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f,
0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x65,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
0x48, 0x00, 0x52, 0x16, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64,
0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x40, 0x0a, 0x10, 0x6f, 0x70,
0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x18, 0x66,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x70, 0x65,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x3d, 0x0a, 0x0f,
0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x75, 0x6e, 0x65, 0x18,
0x67, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x6f, 0x70, 0x65,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, 0x43, 0x0a, 0x11, 0x6f,
0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65,
0x18, 0x68, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x48, 0x00, 0x52, 0x10,
0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65,
0x12, 0x3d, 0x0a, 0x0f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74,
0x61, 0x74, 0x73, 0x18, 0x69, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4f,
0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x48, 0x00, 0x52,
0x0e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12,
0x44, 0x0a, 0x12, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6e,
0x5f, 0x68, 0x6f, 0x6f, 0x6b, 0x18, 0x6a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x76, 0x31,
0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x48, 0x6f, 0x6f,
0x6b, 0x48, 0x00, 0x52, 0x10, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75,
0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x42, 0x04, 0x0a, 0x02, 0x6f, 0x70, 0x22, 0x69, 0x0a, 0x0e, 0x4f,
0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x2a, 0x0a,
0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x76, 0x31,
0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54,
0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x2b, 0x0a, 0x09, 0x6f, 0x70, 0x65,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76,
0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x7c, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b, 0x6c, 0x61, 0x73,
0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17,
0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65,
0x73, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x74, 0x61,
0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x04, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50,
0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x06, 0x65, 0x72,
0x72, 0x6f, 0x72, 0x73, 0x22, 0x82, 0x01, 0x0a, 0x16, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12,
0x2e, 0x0a, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61,
0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12,
0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x67, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52,
0x06, 0x66, 0x6f, 0x72, 0x67, 0x6f, 0x74, 0x12, 0x20, 0x0a, 0x0c, 0x66, 0x6f, 0x72, 0x67, 0x6f,
0x74, 0x5f, 0x62, 0x79, 0x5f, 0x6f, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x66,
0x6f, 0x72, 0x67, 0x6f, 0x74, 0x42, 0x79, 0x4f, 0x70, 0x22, 0x6a, 0x0a, 0x0f, 0x4f, 0x70, 0x65,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2a, 0x0a, 0x06,
0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76,
0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
0x52, 0x06, 0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2b, 0x0a, 0x06, 0x70, 0x6f, 0x6c, 0x69,
0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65,
0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x06, 0x70,
0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, 0x28, 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75,
0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x22,
0x70, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x74,
0x6f, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65,
0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12,
0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x18, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x67,
0x72, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75,
0x73, 0x22, 0x35, 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74,
0x61, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x53, 0x74, 0x61, 0x74,
0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, 0x4b, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x12, 0x12, 0x0a, 0x04,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x6c, 0x6f, 0x67, 0x72, 0x65,
0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x4c,
0x6f, 0x67, 0x72, 0x65, 0x66, 0x2a, 0x60, 0x0a, 0x12, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x45,
0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x11,
0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10,
0x01, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54,
0x45, 0x44, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x44, 0x45,
0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x03, 0x2a, 0xc2, 0x01, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, 0x53,
0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12,
0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e,
0x47, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e,
0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54,
0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x03, 0x12, 0x12,
0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, 0x47,
0x10, 0x07, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52,
0x4f, 0x52, 0x10, 0x04, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53,
0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10,
0x05, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x53, 0x45, 0x52,
0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x06, 0x42, 0x2c, 0x5a, 0x2a,
0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74,
0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74,
0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x6c, 0x6f,
0x77, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x66, 0x6c, 0x6f, 0x77,
0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20,
0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x70,
0x6c, 0x61, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c,
0x61, 0x6e, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73,
0x68, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18,
0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74,
0x75, 0x73, 0x12, 0x2b, 0x0a, 0x12, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f,
0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6d, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f,
0x75, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x72, 0x74, 0x4d, 0x73, 0x12,
0x27, 0x0a, 0x10, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6e, 0x64,
0x5f, 0x6d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x75, 0x6e, 0x69, 0x78, 0x54,
0x69, 0x6d, 0x65, 0x45, 0x6e, 0x64, 0x4d, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x64, 0x69, 0x73, 0x70,
0x6c, 0x61, 0x79, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28,
0x09, 0x52, 0x0e, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67,
0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x6f, 0x67, 0x72, 0x65, 0x66, 0x18, 0x09, 0x20, 0x01, 0x28,
0x09, 0x52, 0x06, 0x6c, 0x6f, 0x67, 0x72, 0x65, 0x66, 0x12, 0x40, 0x0a, 0x10, 0x6f, 0x70, 0x65,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x64, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x70, 0x65, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x56, 0x0a, 0x18, 0x6f,
0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x73,
0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x65, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e,
0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65,
0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x48, 0x00, 0x52, 0x16, 0x6f, 0x70, 0x65,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73,
0x68, 0x6f, 0x74, 0x12, 0x40, 0x0a, 0x10, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x5f, 0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x18, 0x66, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e,
0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x72, 0x67,
0x65, 0x74, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46,
0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x3d, 0x0a, 0x0f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x75, 0x6e, 0x65, 0x18, 0x67, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12,
0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x75,
0x6e, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50,
0x72, 0x75, 0x6e, 0x65, 0x12, 0x43, 0x0a, 0x11, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x5f, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x68, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x14, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65,
0x73, 0x74, 0x6f, 0x72, 0x65, 0x48, 0x00, 0x52, 0x10, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x3d, 0x0a, 0x0f, 0x6f, 0x70, 0x65,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x69, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x48, 0x00, 0x52, 0x0e, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x44, 0x0a, 0x12, 0x6f, 0x70, 0x65, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x68, 0x6f, 0x6f, 0x6b, 0x18, 0x6a,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x48, 0x00, 0x52, 0x10, 0x6f, 0x70,
0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x42, 0x04,
0x0a, 0x02, 0x6f, 0x70, 0x22, 0x69, 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79,
0x70, 0x65, 0x12, 0x2b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18,
0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22,
0x7c, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b,
0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75,
0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63,
0x6b, 0x75, 0x70, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79,
0x52, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x06,
0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76,
0x31, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73,
0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0x82, 0x01,
0x0a, 0x16, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78,
0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x08, 0x73, 0x6e, 0x61, 0x70,
0x73, 0x68, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e,
0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x08,
0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x67,
0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x67, 0x6f, 0x74,
0x12, 0x20, 0x0a, 0x0c, 0x66, 0x6f, 0x72, 0x67, 0x6f, 0x74, 0x5f, 0x62, 0x79, 0x5f, 0x6f, 0x70,
0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x66, 0x6f, 0x72, 0x67, 0x6f, 0x74, 0x42, 0x79,
0x4f, 0x70, 0x22, 0x6a, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46,
0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2a, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x18,
0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69,
0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x67, 0x65,
0x74, 0x12, 0x2b, 0x0a, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e,
0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, 0x28,
0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x75, 0x6e, 0x65,
0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x22, 0x70, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04,
0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68,
0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74,
0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65,
0x73, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74,
0x72, 0x79, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x35, 0x0a, 0x0e, 0x4f, 0x70,
0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x05,
0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x31,
0x2e, 0x52, 0x65, 0x70, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74,
0x73, 0x22, 0x7d, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75,
0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x75, 0x74,
0x70, 0x75, 0x74, 0x5f, 0x6c, 0x6f, 0x67, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
0x52, 0x0c, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x4c, 0x6f, 0x67, 0x72, 0x65, 0x66, 0x12, 0x30,
0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28,
0x0e, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x6f, 0x6f, 0x6b, 0x2e, 0x43, 0x6f, 0x6e, 0x64,
0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e,
0x2a, 0x60, 0x0a, 0x12, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65,
0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f,
0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45,
0x4e, 0x54, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d,
0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12,
0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44,
0x10, 0x03, 0x2a, 0xc2, 0x01, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53,
0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54,
0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x15,
0x0a, 0x11, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e, 0x50, 0x52, 0x4f, 0x47, 0x52,
0x45, 0x53, 0x53, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f,
0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41,
0x54, 0x55, 0x53, 0x5f, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x07, 0x12, 0x10, 0x0a,
0x0c, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x12,
0x1b, 0x0a, 0x17, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d,
0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15,
0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x43, 0x41, 0x4e, 0x43,
0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x06, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75,
0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, 0x72,
0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, 0x67, 0x65, 0x6e, 0x2f,
0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -1025,6 +1046,7 @@ var file_v1_operations_proto_goTypes = []interface{}{
(*RetentionPolicy)(nil), // 15: v1.RetentionPolicy
(*RestoreProgressEntry)(nil), // 16: v1.RestoreProgressEntry
(*RepoStats)(nil), // 17: v1.RepoStats
(Hook_Condition)(0), // 18: v1.Hook.Condition
}
var file_v1_operations_proto_depIdxs = []int32{
3, // 0: v1.OperationList.operations:type_name -> v1.Operation
@@ -1045,11 +1067,12 @@ var file_v1_operations_proto_depIdxs = []int32{
15, // 15: v1.OperationForget.policy:type_name -> v1.RetentionPolicy
16, // 16: v1.OperationRestore.status:type_name -> v1.RestoreProgressEntry
17, // 17: v1.OperationStats.stats:type_name -> v1.RepoStats
18, // [18:18] is the sub-list for method output_type
18, // [18:18] is the sub-list for method input_type
18, // [18:18] is the sub-list for extension type_name
18, // [18:18] is the sub-list for extension extendee
0, // [0:18] is the sub-list for field type_name
18, // 18: v1.OperationRunHook.condition:type_name -> v1.Hook.Condition
19, // [19:19] is the sub-list for method output_type
19, // [19:19] is the sub-list for method input_type
19, // [19:19] is the sub-list for extension type_name
19, // [19:19] is the sub-list for extension extendee
0, // [0:19] is the sub-list for field type_name
}
func init() { file_v1_operations_proto_init() }
gen/go/v1/service.pb.go
+114 -104
View File
@@ -220,6 +220,7 @@ type GetOperationsRequest struct {
RepoId string `protobuf:"bytes,1,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"`
PlanId string `protobuf:"bytes,2,opt,name=plan_id,json=planId,proto3" json:"plan_id,omitempty"`
SnapshotId string `protobuf:"bytes,4,opt,name=snapshot_id,json=snapshotId,proto3" json:"snapshot_id,omitempty"`
FlowId int64 `protobuf:"varint,6,opt,name=flow_id,json=flowId,proto3" json:"flow_id,omitempty"`
Ids []int64 `protobuf:"varint,5,rep,packed,name=ids,proto3" json:"ids,omitempty"`
LastN int64 `protobuf:"varint,3,opt,name=last_n,json=lastN,proto3" json:"last_n,omitempty"` // limit to the last n operations
}
@@ -277,6 +278,13 @@ func (x *GetOperationsRequest) GetSnapshotId() string {
return ""
}
func (x *GetOperationsRequest) GetFlowId() int64 {
if x != nil {
return x.FlowId
}
return 0
}
func (x *GetOperationsRequest) GetIds() []int64 {
if x != nil {
return x.Ids
@@ -685,123 +693,125 @@ var file_v1_service_proto_rawDesc = []byte{
0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07,
0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70,
0x6c, 0x61, 0x6e, 0x49, 0x64, 0x22, 0x92, 0x01, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65,
0x6c, 0x61, 0x6e, 0x49, 0x64, 0x22, 0xab, 0x01, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17,
0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x70, 0x6c, 0x61, 0x6e, 0x5f,
0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x61, 0x6e, 0x49, 0x64,
0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18,
0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49,
0x64, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x03, 0x52, 0x03,
0x69, 0x64, 0x73, 0x12, 0x15, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6e, 0x18, 0x03, 0x20,
0x01, 0x28, 0x03, 0x52, 0x05, 0x6c, 0x61, 0x73, 0x74, 0x4e, 0x22, 0x97, 0x01, 0x0a, 0x16, 0x52,
0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x69, 0x64,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x17,
0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52,
0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73,
0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e,
0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68,
0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06,
0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61,
0x72, 0x67, 0x65, 0x74, 0x22, 0x68, 0x0a, 0x18, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70,
0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61,
0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a,
0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61,
0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x56,
0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69,
0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70,
0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12,
0x25, 0x0a, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b,
0x32, 0x0b, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x65,
0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0x22, 0x0a, 0x0e, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74,
0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0xd3, 0x01, 0x0a, 0x07, 0x4c,
0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79,
0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12,
0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61,
0x74, 0x68, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52,
0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28,
0x03, 0x52, 0x03, 0x67, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06,
0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f,
0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x14,
0x0a, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d,
0x74, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20,
0x01, 0x28, 0x09, 0x52, 0x05, 0x61, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x74,
0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x63, 0x74, 0x69, 0x6d, 0x65,
0x32, 0xa7, 0x08, 0x0a, 0x08, 0x42, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a,
0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70,
0x74, 0x79, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00,
0x12, 0x25, 0x0a, 0x09, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x0a, 0x2e,
0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43,
0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, 0x12, 0x21, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x52, 0x65,
0x70, 0x6f, 0x12, 0x08, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x1a, 0x0a, 0x2e, 0x76,
0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, 0x12, 0x44, 0x0a, 0x12, 0x47, 0x65,
0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73,
0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70,
0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x00, 0x30, 0x01,
0x12, 0x3e, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x76, 0x31,
0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00,
0x12, 0x43, 0x0a, 0x0d, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73,
0x68, 0x6f, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x76, 0x31,
0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x4c,
0x69, 0x73, 0x74, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61,
0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x76, 0x31, 0x2e,
0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65,
0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69,
0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3e, 0x0a, 0x0e, 0x49, 0x6e, 0x64,
0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x12, 0x12, 0x2e, 0x74, 0x79,
0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a,
0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x42, 0x61, 0x63,
0x6b, 0x75, 0x70, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69,
0x64, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01,
0x28, 0x03, 0x52, 0x06, 0x66, 0x6c, 0x6f, 0x77, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x64,
0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x03, 0x52, 0x03, 0x69, 0x64, 0x73, 0x12, 0x15, 0x0a, 0x06,
0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6c, 0x61,
0x73, 0x74, 0x4e, 0x22, 0x97, 0x01, 0x0a, 0x16, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53,
0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17,
0x0a, 0x07, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x06, 0x70, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f,
0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64,
0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49,
0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18,
0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0x68, 0x0a,
0x18, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c,
0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70,
0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f,
0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69,
0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f,
0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28,
0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x56, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x53,
0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x25, 0x0a, 0x07, 0x65, 0x6e, 0x74, 0x72,
0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x76, 0x31, 0x2e, 0x4c,
0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22,
0x22, 0x0a, 0x0e, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
0x72, 0x65, 0x66, 0x22, 0xd3, 0x01, 0x0a, 0x07, 0x4c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12,
0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18,
0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x10, 0x0a, 0x03, 0x75,
0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a,
0x03, 0x67, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x67, 0x69, 0x64, 0x12,
0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73,
0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28,
0x03, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65,
0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a,
0x05, 0x61, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x61, 0x74,
0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01,
0x28, 0x09, 0x52, 0x05, 0x63, 0x74, 0x69, 0x6d, 0x65, 0x32, 0xa7, 0x08, 0x0a, 0x08, 0x42, 0x61,
0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e,
0x66, 0x69, 0x67, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x0a, 0x2e, 0x76, 0x31,
0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00, 0x12, 0x25, 0x0a, 0x09, 0x53, 0x65, 0x74,
0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66,
0x69, 0x67, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x00,
0x12, 0x21, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x52, 0x65, 0x70, 0x6f, 0x12, 0x08, 0x2e, 0x76, 0x31,
0x2e, 0x52, 0x65, 0x70, 0x6f, 0x1a, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69,
0x67, 0x22, 0x00, 0x12, 0x44, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74,
0x79, 0x1a, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x45, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x3e, 0x0a, 0x0d, 0x47, 0x65, 0x74,
0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e,
0x47, 0x65, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0d, 0x4c, 0x69, 0x73,
0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x12, 0x18, 0x2e, 0x76, 0x31, 0x2e,
0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x69, 0x63,
0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x12, 0x52,
0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69,
0x6c, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61,
0x70, 0x73, 0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x1d, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73,
0x68, 0x6f, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x22, 0x00, 0x12, 0x3e, 0x0a, 0x0e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73,
0x68, 0x6f, 0x74, 0x73, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72,
0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x12, 0x2e, 0x74,
0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65,
0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x05, 0x50, 0x72,
0x75, 0x6e, 0x65, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69,
0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22,
0x00, 0x12, 0x35, 0x0a, 0x05, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70,
0x00, 0x12, 0x35, 0x0a, 0x06, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x12, 0x11, 0x2e, 0x76, 0x31,
0x2e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x74,
0x6f, 0x72, 0x65, 0x12, 0x1a, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65,
0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x06, 0x55, 0x6e, 0x6c,
0x6f, 0x63, 0x6b, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69,
0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22,
0x00, 0x12, 0x35, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70,
0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x06, 0x46, 0x6f, 0x72, 0x67,
0x65, 0x74, 0x12, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x6f, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12,
0x3f, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1a, 0x2e, 0x76, 0x31, 0x2e,
0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00,
0x12, 0x36, 0x0a, 0x06, 0x55, 0x6e, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70,
0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74,
0x73, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x35, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63,
0x65, 0x6c, 0x12, 0x11, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34,
0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12,
0x35, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x11, 0x2e, 0x74, 0x79, 0x70, 0x65,
0x73, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x16, 0x2e, 0x67,
0x32, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x12, 0x2e, 0x76, 0x31, 0x2e,
0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11,
0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x75,
0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0c, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x48, 0x69, 0x73, 0x74,
0x6f, 0x72, 0x79, 0x12, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x48, 0x69,
0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67,
0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45,
0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67,
0x73, 0x12, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x42, 0x79,
0x74, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0c, 0x43, 0x6c,
0x65, 0x61, 0x72, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x17, 0x2e, 0x76, 0x31, 0x2e,
0x43, 0x6c, 0x65, 0x61, 0x72, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3b, 0x0a,
0x10, 0x50, 0x61, 0x74, 0x68, 0x41, 0x75, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74,
0x65, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67,
0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x11, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74,
0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x00, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x69,
0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67,
0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, 0x67,
0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x10, 0x50, 0x61, 0x74, 0x68, 0x41, 0x75,
0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x12, 0x2e, 0x74, 0x79, 0x70,
0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x11,
0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73,
0x74, 0x22, 0x00, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62,
0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76,
0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
internal/api/backresthandler.go
+36 -39
View File
@@ -6,7 +6,6 @@ import (
"fmt"
"os"
"path"
"sync"
"time"
"connectrpc.com/connect"
@@ -17,6 +16,8 @@ import (
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/oplog/indexutil"
"github.com/garethgeorge/backrest/internal/orchestrator"
"github.com/garethgeorge/backrest/internal/orchestrator/repo"
"github.com/garethgeorge/backrest/internal/orchestrator/tasks"
"github.com/garethgeorge/backrest/internal/protoutil"
"github.com/garethgeorge/backrest/internal/resticinstaller"
"github.com/garethgeorge/backrest/internal/rotatinglog"
@@ -107,7 +108,7 @@ func (s *BackrestHandler) AddRepo(ctx context.Context, req *connect.Request[v1.R
return nil, fmt.Errorf("failed to find or install restic binary: %w", err)
}
r, err := orchestrator.NewRepoOrchestrator(req.Msg, bin)
r, err := repo.NewRepoOrchestrator(req.Msg, bin)
if err != nil {
return nil, fmt.Errorf("failed to configure repo: %w", err)
}
@@ -127,7 +128,7 @@ func (s *BackrestHandler) AddRepo(ctx context.Context, req *connect.Request[v1.R
// index snapshots for the newly added repository.
zap.L().Debug("scheduling index snapshots task")
s.orchestrator.ScheduleTask(orchestrator.NewOneoffIndexSnapshotsTask(s.orchestrator, req.Msg.Id, time.Now()), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityIndexSnapshots)
s.orchestrator.ScheduleTask(tasks.NewOneoffIndexSnapshotsTask(req.Msg.Id, time.Now()), tasks.TaskPriorityInteractive+tasks.TaskPriorityIndexSnapshots)
zap.L().Debug("done add repo")
return connect.NewResponse(c), nil
@@ -136,7 +137,7 @@ func (s *BackrestHandler) AddRepo(ctx context.Context, req *connect.Request[v1.R
// ListSnapshots implements POST /v1/snapshots
func (s *BackrestHandler) ListSnapshots(ctx context.Context, req *connect.Request[v1.ListSnapshotsRequest]) (*connect.Response[v1.ResticSnapshotList], error) {
query := req.Msg
repo, err := s.orchestrator.GetRepo(query.RepoId)
repo, err := s.orchestrator.GetRepoOrchestrator(query.RepoId)
if err != nil {
return nil, fmt.Errorf("failed to get repo: %w", err)
}
@@ -170,7 +171,7 @@ func (s *BackrestHandler) ListSnapshots(ctx context.Context, req *connect.Reques
func (s *BackrestHandler) ListSnapshotFiles(ctx context.Context, req *connect.Request[v1.ListSnapshotFilesRequest]) (*connect.Response[v1.ListSnapshotFilesResponse], error) {
query := req.Msg
repo, err := s.orchestrator.GetRepo(query.RepoId)
repo, err := s.orchestrator.GetRepoOrchestrator(query.RepoId)
if err != nil {
return nil, fmt.Errorf("failed to get repo: %w", err)
}
@@ -258,6 +259,8 @@ func (s *BackrestHandler) GetOperations(ctx context.Context, req *connect.Reques
err = s.oplog.ForEachByRepo(req.Msg.RepoId, idCollector, opCollector)
} else if req.Msg.SnapshotId != "" {
err = s.oplog.ForEachBySnapshotId(req.Msg.SnapshotId, idCollector, opCollector)
} else if req.Msg.FlowId != 0 {
err = s.oplog.ForEachByFlowId(req.Msg.FlowId, idCollector, opCollector)
} else if len(req.Msg.Ids) > 0 {
ops = make([]*v1.Operation, 0, len(req.Msg.Ids))
for i, id := range req.Msg.Ids {
@@ -285,7 +288,7 @@ func (s *BackrestHandler) IndexSnapshots(ctx context.Context, req *connect.Reque
return nil, fmt.Errorf("failed to get repo %q: %w", req.Msg.Value, err)
}
s.orchestrator.ScheduleTask(orchestrator.NewOneoffIndexSnapshotsTask(s.orchestrator, req.Msg.Value, time.Now()), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityIndexSnapshots)
s.orchestrator.ScheduleTask(tasks.NewOneoffIndexSnapshotsTask(req.Msg.Value, time.Now()), tasks.TaskPriorityInteractive+tasks.TaskPriorityIndexSnapshots)
return connect.NewResponse(&emptypb.Empty{}), nil
}
@@ -295,38 +298,36 @@ func (s *BackrestHandler) Backup(ctx context.Context, req *connect.Request[types
if err != nil {
return nil, fmt.Errorf("failed to get plan %q: %w", req.Msg.Value, err)
}
var wg sync.WaitGroup
wg.Add(1)
s.orchestrator.ScheduleTask(orchestrator.NewOneoffBackupTask(s.orchestrator, plan, time.Now()), orchestrator.TaskPriorityInteractive, func(e error) {
wait := make(chan struct{})
s.orchestrator.ScheduleTask(tasks.NewOneoffBackupTask(plan, time.Now()), tasks.TaskPriorityInteractive, func(e error) {
err = e
wg.Done()
close(wait)
})
wg.Wait()
<-wait
return connect.NewResponse(&emptypb.Empty{}), err
}
func (s *BackrestHandler) Forget(ctx context.Context, req *connect.Request[v1.ForgetRequest]) (*connect.Response[emptypb.Empty], error) {
at := time.Now()
var err error
_, err := s.orchestrator.GetPlan(req.Msg.PlanId)
if err != nil {
return nil, fmt.Errorf("failed to get plan %q: %w", req.Msg.PlanId, err)
}
if req.Msg.SnapshotId != "" && req.Msg.PlanId != "" && req.Msg.RepoId != "" {
wait := make(chan struct{})
s.orchestrator.ScheduleTask(
orchestrator.NewOneoffForgetSnapshotTask(s.orchestrator, req.Msg.RepoId, req.Msg.PlanId, req.Msg.SnapshotId, at),
orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityForget, func(e error) {
tasks.NewOneoffForgetSnapshotTask(req.Msg.RepoId, req.Msg.PlanId, 0, at, req.Msg.SnapshotId),
tasks.TaskPriorityInteractive+tasks.TaskPriorityForget, func(e error) {
err = e
close(wait)
})
<-wait
} else if req.Msg.RepoId != "" && req.Msg.PlanId != "" {
plan, err := s.orchestrator.GetPlan(req.Msg.PlanId)
if err != nil {
return nil, fmt.Errorf("failed to get plan %q: %w", req.Msg.PlanId, err)
}
wait := make(chan struct{})
s.orchestrator.ScheduleTask(
orchestrator.NewOneoffForgetTask(s.orchestrator, plan, "", at),
orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityForget, func(e error) {
tasks.NewOneoffForgetTask(req.Msg.RepoId, req.Msg.PlanId, 0, at),
tasks.TaskPriorityInteractive+tasks.TaskPriorityForget, func(e error) {
err = e
close(wait)
})
@@ -348,13 +349,12 @@ func (s *BackrestHandler) Prune(ctx context.Context, req *connect.Request[types.
}
at := time.Now()
var wg sync.WaitGroup
wg.Add(1)
s.orchestrator.ScheduleTask(orchestrator.NewOneoffPruneTask(s.orchestrator, plan, at, true), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityPrune, func(e error) {
wait := make(chan struct{})
s.orchestrator.ScheduleTask(tasks.NewOneoffPruneTask(plan.Repo, plan.Id, 0, at, true), tasks.TaskPriorityInteractive+tasks.TaskPriorityPrune, func(e error) {
err = e
wg.Done()
close(wait)
})
wg.Wait()
<-wait
return connect.NewResponse(&emptypb.Empty{}), nil
}
@@ -375,19 +375,17 @@ func (s *BackrestHandler) Restore(ctx context.Context, req *connect.Request[v1.R
at := time.Now()
s.orchestrator.ScheduleTask(orchestrator.NewOneoffRestoreTask(s.orchestrator, orchestrator.RestoreTaskOpts{
RepoId: req.Msg.RepoId,
PlanId: req.Msg.PlanId,
SnapshotId: req.Msg.SnapshotId,
Path: req.Msg.Path,
Target: target,
}, at), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityDefault)
flowID, err := tasks.FlowIDForSnapshotID(s.oplog, req.Msg.SnapshotId)
if err != nil {
return nil, fmt.Errorf("failed to get flow ID for snapshot %q: %w", req.Msg.SnapshotId, err)
}
s.orchestrator.ScheduleTask(tasks.NewOneoffRestoreTask(req.Msg.RepoId, req.Msg.PlanId, flowID, at, req.Msg.SnapshotId, req.Msg.Path, target), tasks.TaskPriorityInteractive+tasks.TaskPriorityDefault)
return connect.NewResponse(&emptypb.Empty{}), nil
}
func (s *BackrestHandler) Unlock(ctx context.Context, req *connect.Request[types.StringValue]) (*connect.Response[emptypb.Empty], error) {
repo, err := s.orchestrator.GetRepo(req.Msg.Value)
repo, err := s.orchestrator.GetRepoOrchestrator(req.Msg.Value)
if err != nil {
return nil, fmt.Errorf("failed to get repo %q: %w", req.Msg.Value, err)
}
@@ -402,13 +400,12 @@ func (s *BackrestHandler) Unlock(ctx context.Context, req *connect.Request[types
func (s *BackrestHandler) Stats(ctx context.Context, req *connect.Request[types.StringValue]) (*connect.Response[emptypb.Empty], error) {
at := time.Now()
var err error
var wg sync.WaitGroup
wg.Add(1)
s.orchestrator.ScheduleTask(orchestrator.NewOneoffStatsTask(s.orchestrator, req.Msg.Value, orchestrator.PlanForUnassociatedOperations, at), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityStats, func(e error) {
wait := make(chan struct{})
s.orchestrator.ScheduleTask(tasks.NewOneoffStatsTask(req.Msg.Value, tasks.PlanForUnassociatedOperations, at), tasks.TaskPriorityInteractive+tasks.TaskPriorityStats, func(e error) {
err = e
wg.Done()
close(wait)
})
wg.Wait()
<-wait
return connect.NewResponse(&emptypb.Empty{}), err
}
internal/api/backresthandler_test.go
+14 -13
View File
@@ -134,15 +134,14 @@ func TestBackup(t *testing.T) {
}
// Wait for the index snapshot operation to appear in the oplog.
var snapshotId string
var snapshotOp *v1.Operation
if err := retry(t, 10, 2*time.Second, func() error {
operations := getOperations(t, sut.oplog)
if index := slices.IndexFunc(operations, func(op *v1.Operation) bool {
_, ok := op.GetOp().(*v1.Operation_OperationIndexSnapshot)
return op.Status == v1.OperationStatus_STATUS_SUCCESS && ok
}); index != -1 {
op := operations[index]
snapshotId = op.SnapshotId
snapshotOp = operations[index]
return nil
}
return errors.New("snapshot not indexed")
@@ -150,7 +149,7 @@ func TestBackup(t *testing.T) {
t.Fatalf("Couldn't find snapshot in oplog")
}
if snapshotId == "" {
if snapshotOp.SnapshotId == "" {
t.Fatalf("snapshotId must be set")
}
@@ -162,8 +161,8 @@ func TestBackup(t *testing.T) {
return op.Status == v1.OperationStatus_STATUS_SUCCESS && ok
}); index != -1 {
op := operations[index]
if op.SnapshotId != snapshotId {
t.Fatalf("Snapshot ID mismatch on forget operation")
if op.FlowId != snapshotOp.FlowId {
t.Fatalf("Flow ID mismatch on forget operation")
}
return nil
}
@@ -460,15 +459,14 @@ func TestRestore(t *testing.T) {
}
// Wait for the index snapshot operation to appear in the oplog.
var snapshotId string
var snapshotOp *v1.Operation
if err := retry(t, 10, 2*time.Second, func() error {
operations := getOperations(t, sut.oplog)
if index := slices.IndexFunc(operations, func(op *v1.Operation) bool {
_, ok := op.GetOp().(*v1.Operation_OperationIndexSnapshot)
return op.Status == v1.OperationStatus_STATUS_SUCCESS && ok
}); index != -1 {
op := operations[index]
snapshotId = op.SnapshotId
snapshotOp = operations[index]
return nil
}
return errors.New("snapshot not indexed")
@@ -476,14 +474,14 @@ func TestRestore(t *testing.T) {
t.Fatalf("Couldn't find snapshot in oplog")
}
if snapshotId == "" {
if snapshotOp.SnapshotId == "" {
t.Fatalf("snapshotId must be set")
}
restoreTarget := t.TempDir()
_, err = sut.handler.Restore(context.Background(), connect.NewRequest(&v1.RestoreSnapshotRequest{
SnapshotId: snapshotId,
SnapshotId: snapshotOp.SnapshotId,
PlanId: "test",
RepoId: "local",
Target: restoreTarget,
@@ -500,8 +498,11 @@ func TestRestore(t *testing.T) {
return op.Status == v1.OperationStatus_STATUS_SUCCESS && ok
}); index != -1 {
op := operations[index]
if op.SnapshotId != snapshotId {
t.Fatalf("Snapshot ID mismatch on restore operation")
if op.FlowId != snapshotOp.FlowId {
t.Errorf("Flow ID mismatch on restore operation")
}
if op.SnapshotId != snapshotOp.SnapshotId {
t.Errorf("Snapshot ID mismatch on restore operation")
}
return nil
}
internal/hook/hook.go
+7 -4
View File
@@ -35,11 +35,12 @@ func NewHookExecutor(oplog *oplog.OpLog, bigOutputStore *rotatinglog.RotatingLog
// ExecuteHooks schedules tasks for the hooks subscribed to the given event. The vars map is used to substitute variables
// Hooks are pulled both from the provided plan and from the repo config.
func (e *HookExecutor) ExecuteHooks(repo *v1.Repo, plan *v1.Plan, events []v1.Hook_Condition, vars HookVars) error {
func (e *HookExecutor) ExecuteHooks(flowID int64, repo *v1.Repo, plan *v1.Plan, events []v1.Hook_Condition, vars HookVars) error {
operationBase := v1.Operation{
Status: v1.OperationStatus_STATUS_INPROGRESS,
PlanId: plan.GetId(),
RepoId: repo.GetId(),
FlowId: flowID,
}
vars.Repo = repo
@@ -59,10 +60,11 @@ func (e *HookExecutor) ExecuteHooks(repo *v1.Repo, plan *v1.Plan, events []v1.Ho
operation.UnixTimeStartMs = curTimeMs()
operation.Op = &v1.Operation_OperationRunHook{
OperationRunHook: &v1.OperationRunHook{
Name: name,
Name: name,
Condition: event,
},
}
zap.L().Info("running hook", zap.String("plan", plan.Id), zap.Int64("opId", operation.Id), zap.String("hook", name))
zap.L().Info("running hook", zap.String("plan", repo.Id), zap.Int64("opId", operation.Id), zap.String("hook", name))
if err := e.executeHook(operation, h, event, vars); err != nil {
zap.S().Errorf("error on repo hook %v on condition %v: %v", idx, event.String(), err)
if isHaltingError(err) {
@@ -84,7 +86,8 @@ func (e *HookExecutor) ExecuteHooks(repo *v1.Repo, plan *v1.Plan, events []v1.Ho
operation.UnixTimeStartMs = curTimeMs()
operation.Op = &v1.Operation_OperationRunHook{
OperationRunHook: &v1.OperationRunHook{
Name: name,
Name: name,
Condition: event,
},
}
internal/ioutil/ioutil.go
+13
View File
@@ -110,3 +110,16 @@ func (w *OutputCapturer) Bytes() []byte {
return buf.Bytes()
}
type SynchronizedWriter struct {
mu sync.Mutex
W io.Writer
}
var _ io.Writer = &SynchronizedWriter{}
func (w *SynchronizedWriter) Write(p []byte) (n int, err error) {
w.mu.Lock()
defer w.mu.Unlock()
return w.W.Write(p)
}
internal/oplog/migrations.go
+110
View File
@@ -0,0 +1,110 @@
package oplog
import (
"errors"
"fmt"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/oplog/serializationutil"
"go.etcd.io/bbolt"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
)
var migrations = []func(*OpLog, *bbolt.Tx) error{
migration001FlowID,
}
var CurrentVersion = int64(len(migrations))
func ApplyMigrations(oplog *OpLog, tx *bbolt.Tx) error {
var version int64
versionBytes := tx.Bucket(SystemBucket).Get([]byte("version"))
if versionBytes == nil {
version = 0
} else {
v, err := serializationutil.Btoi(versionBytes)
if err != nil {
return fmt.Errorf("couldn't parse version: %w", err)
}
version = v
}
startMigration := int(version)
if startMigration < 0 {
startMigration = 0
}
for idx := startMigration; idx < len(migrations); idx += 1 {
zap.L().Info("applying oplog migration", zap.Int("migration_no", idx))
if err := migrations[idx](oplog, tx); err != nil {
zap.L().Error("failed to apply migration", zap.Int("migration_no", idx), zap.Error(err))
return fmt.Errorf("couldn't apply migration %d: %w", idx, err)
}
}
if err := tx.Bucket(SystemBucket).Put([]byte("version"), serializationutil.Itob(CurrentVersion)); err != nil {
return fmt.Errorf("couldn't update version: %w", err)
}
return nil
}
func transformOperations(oplog *OpLog, tx *bbolt.Tx, f func(op *v1.Operation) error) error {
opLogBucket := tx.Bucket(OpLogBucket)
if opLogBucket == nil {
return errors.New("oplog bucket not found")
}
c := opLogBucket.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
op := &v1.Operation{}
if err := proto.Unmarshal(v, op); err != nil {
return fmt.Errorf("unmarshal operation: %w", err)
}
copy := proto.Clone(op).(*v1.Operation)
err := f(copy)
if err != nil {
return err
}
if proto.Equal(copy, op) {
continue
}
if _, err := oplog.deleteOperationHelper(tx, op.Id); err != nil {
return fmt.Errorf("delete operation: %w", err)
}
if err := oplog.addOperationHelper(tx, copy); err != nil {
return fmt.Errorf("create operation: %w", err)
}
}
return nil
}
// migration001FlowID sets the flow ID for operations that are missing it.
// All operations with the same snapshot ID will have the same flow ID.
func migration001FlowID(oplog *OpLog, tx *bbolt.Tx) error {
snapshotIdToFlow := make(map[string]int64)
return transformOperations(oplog, tx, func(op *v1.Operation) error {
if op.FlowId != 0 {
return nil
}
if op.SnapshotId == "" {
op.FlowId = op.Id
return nil
}
if flowId, ok := snapshotIdToFlow[op.SnapshotId]; ok {
op.FlowId = flowId
} else {
snapshotIdToFlow[op.SnapshotId] = op.Id
op.FlowId = op.Id
}
return nil
})
}
internal/oplog/oplog.go
+30 -3
View File
@@ -34,6 +34,7 @@ var (
OpLogBucket = []byte("oplog.log") // oplog stores existant operations.
RepoIndexBucket = []byte("oplog.repo_idx") // repo_index tracks IDs of operations affecting a given repo
PlanIndexBucket = []byte("oplog.plan_idx") // plan_index tracks IDs of operations affecting a given plan
FlowIdIndexBucket = []byte("oplog.flow_id_idx") // flow_id_index tracks IDs of operations affecting a given flow
SnapshotIndexBucket = []byte("oplog.snapshot_idx") // snapshot_index tracks IDs of operations affecting a given snapshot
)
@@ -63,13 +64,17 @@ func NewOpLog(databasePath string) (*OpLog, error) {
if err := db.Update(func(tx *bolt.Tx) error {
// Create the buckets if they don't exist
for _, bucket := range [][]byte{
SystemBucket, OpLogBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket,
SystemBucket, OpLogBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket, FlowIdIndexBucket,
} {
if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
return fmt.Errorf("creating bucket %s: %s", string(bucket), err)
}
}
if err := ApplyMigrations(o, tx); err != nil {
return fmt.Errorf("applying migrations: %w", err)
}
return nil
}); err != nil {
return nil, err
@@ -236,7 +241,7 @@ func (o *OpLog) getOperationHelper(b *bolt.Bucket, id int64) (*v1.Operation, err
return &op, nil
}
func (o *OpLog) nextOperationId(b *bolt.Bucket, unixTimeMs int64) (int64, error) {
func (o *OpLog) nextID(b *bolt.Bucket, unixTimeMs int64) (int64, error) {
seq, err := b.NextSequence()
if err != nil {
return 0, fmt.Errorf("next sequence: %w", err)
@@ -248,12 +253,16 @@ func (o *OpLog) addOperationHelper(tx *bolt.Tx, op *v1.Operation) error {
b := tx.Bucket(OpLogBucket)
if op.Id == 0 {
var err error
op.Id, err = o.nextOperationId(b, time.Now().UnixMilli())
op.Id, err = o.nextID(b, time.Now().UnixMilli())
if err != nil {
return fmt.Errorf("create next operation ID: %w", err)
}
}
if op.FlowId == 0 {
op.FlowId = op.Id
}
if err := protoutil.ValidateOperation(op); err != nil {
return fmt.Errorf("validating operation: %w", err)
}
@@ -283,6 +292,11 @@ func (o *OpLog) addOperationHelper(tx *bolt.Tx, op *v1.Operation) error {
return fmt.Errorf("error adding operation to snapshot index: %w", err)
}
}
if op.FlowId != 0 {
if err := indexutil.IndexByteValue(tx.Bucket(FlowIdIndexBucket), serializationutil.Itob(op.FlowId), op.Id); err != nil {
return fmt.Errorf("error adding operation to flow index: %w", err)
}
}
return nil
}
@@ -313,6 +327,12 @@ func (o *OpLog) deleteOperationHelper(tx *bolt.Tx, id int64) (*v1.Operation, err
}
}
if prevValue.FlowId != 0 {
if err := indexutil.IndexRemoveByteValue(tx.Bucket(FlowIdIndexBucket), serializationutil.Itob(prevValue.FlowId), id); err != nil {
return nil, fmt.Errorf("removing operation %v from flow index: %w", id, err)
}
}
if err := b.Delete(serializationutil.Itob(id)); err != nil {
return nil, fmt.Errorf("deleting operation %v from bucket: %w", id, err)
}
@@ -356,6 +376,13 @@ func (o *OpLog) ForEachBySnapshotId(snapshotId string, collector indexutil.Colle
})
}
func (o *OpLog) ForEachByFlowId(flowId int64, collector indexutil.Collector, do func(op *v1.Operation) error) error {
return o.db.View(func(tx *bolt.Tx) error {
ids := collector(indexutil.IndexSearchByteValue(tx.Bucket(FlowIdIndexBucket), serializationutil.Itob(flowId)))
return o.forOpsByIds(tx, ids, do)
})
}
func (o *OpLog) forOpsByIds(tx *bolt.Tx, ids []int64, do func(*v1.Operation) error) error {
b := tx.Bucket(OpLogBucket)
for _, id := range ids {
internal/oplog/oplog_test.go
+36
View File
@@ -215,6 +215,42 @@ func TestListOperation(t *testing.T) {
}
}
func TestListByFlowId(t *testing.T) {
t.Parallel()
log, err := NewOpLog(t.TempDir() + "/test.boltdb")
if err != nil {
t.Fatalf("error creating oplog: %s", err)
}
t.Cleanup(func() { log.Close() })
op := &v1.Operation{
UnixTimeStartMs: 1234,
PlanId: "plan1",
RepoId: "repo1",
FlowId: 1,
Op: &v1.Operation_OperationBackup{},
}
if err := log.Add(op); err != nil {
t.Fatalf("error adding operation: %s", err)
}
var ops []*v1.Operation
if err := log.ForEachByFlowId(1, indexutil.CollectAll(), func(op *v1.Operation) error {
ops = append(ops, op)
return nil
}); err != nil {
t.Fatalf("error listing operations: %s", err)
}
if len(ops) != 1 {
t.Fatalf("want 1 operation, got %d", len(ops))
}
if ops[0].Id != op.Id {
t.Errorf("want operation ID %d, got %d", op.Id, ops[0].Id)
}
}
func TestBigIO(t *testing.T) {
t.Parallel()
internal/orchestrator/logging/logging.go
+24
View File
@@ -0,0 +1,24 @@
package logging
import (
"context"
"io"
)
type contextKey int
const (
contextKeyLogWriter contextKey = iota
)
func WriterFromContext(ctx context.Context) io.Writer {
writer, ok := ctx.Value(contextKeyLogWriter).(io.Writer)
if !ok {
return nil
}
return writer
}
func ContextWithWriter(ctx context.Context, logger io.Writer) context.Context {
return context.WithValue(ctx, contextKeyLogWriter, logger)
}
internal/orchestrator/orchestrator.go
+219 -89
View File
@@ -1,18 +1,22 @@
package orchestrator
import (
"bytes"
"context"
"errors"
"fmt"
"slices"
"sync"
"sync/atomic"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/config"
"github.com/garethgeorge/backrest/internal/hook"
"github.com/garethgeorge/backrest/internal/ioutil"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/orchestrator/logging"
"github.com/garethgeorge/backrest/internal/orchestrator/repo"
"github.com/garethgeorge/backrest/internal/orchestrator/tasks"
"github.com/garethgeorge/backrest/internal/queue"
"github.com/garethgeorge/backrest/internal/rotatinglog"
"go.uber.org/zap"
@@ -23,32 +27,35 @@ var ErrRepoNotFound = errors.New("repo not found")
var ErrRepoInitializationFailed = errors.New("repo initialization failed")
var ErrPlanNotFound = errors.New("plan not found")
const PlanForUnassociatedOperations = "_unassociated_"
const (
TaskPriorityDefault = 0
TaskPriorityInteractive = 10
TaskPriorityIndexSnapshots = 101
TaskPriorityForget = 102
TaskPriorityPrune = 103
TaskPriorityHook = 1000 // runs before any other task.
TaskPriorityStats = -1 // very low priority.
)
// Orchestrator is responsible for managing repos and backups.
type Orchestrator struct {
mu sync.Mutex
config *v1.Config
OpLog *oplog.OpLog
repoPool *resticRepoPool
taskQueue *queue.TimePriorityQueue[scheduledTask]
taskQueue *queue.TimePriorityQueue[stContainer]
hookExecutor *hook.HookExecutor
logStore *rotatinglog.RotatingLog
// cancelNotify is a list of channels that are notified when a task should be cancelled.
cancelNotify []chan int64
// now for the purpose of testing; used by Run() to get the current time.
now func() time.Time
}
runningTask atomic.Pointer[taskExecutionInfo]
type stContainer struct {
tasks.ScheduledTask
configModno int32
callbacks []func(error)
}
func (st stContainer) Eq(other stContainer) bool {
return st.ScheduledTask.Eq(other.ScheduledTask)
}
func (st stContainer) Less(other stContainer) bool {
return st.ScheduledTask.Less(other.ScheduledTask)
}
func NewOrchestrator(resticBin string, cfg *v1.Config, oplog *oplog.OpLog, logStore *rotatinglog.RotatingLog) (*Orchestrator, error) {
@@ -61,7 +68,7 @@ func NewOrchestrator(resticBin string, cfg *v1.Config, oplog *oplog.OpLog, logSt
config: cfg,
// repoPool created with a memory store to ensure the config is updated in an atomic operation with the repo pool's config value.
repoPool: newResticRepoPool(resticBin, &config.MemoryStore{Config: cfg}),
taskQueue: queue.NewTimePriorityQueue[scheduledTask](),
taskQueue: queue.NewTimePriorityQueue[stContainer](),
hookExecutor: hook.NewHookExecutor(oplog, logStore),
logStore: logStore,
}
@@ -81,7 +88,7 @@ func NewOrchestrator(resticBin string, cfg *v1.Config, oplog *oplog.OpLog, logSt
}
for _, repoId := range incompleteOpRepos {
repo, err := o.GetRepo(repoId)
repo, err := o.GetRepoOrchestrator(repoId)
if err != nil {
if errors.Is(err, ErrRepoNotFound) {
zap.L().Warn("repo not found for incomplete operation. Possibly just deleted.", zap.String("repo", repoId))
@@ -100,6 +107,8 @@ func NewOrchestrator(resticBin string, cfg *v1.Config, oplog *oplog.OpLog, logSt
return nil, fmt.Errorf("apply initial config: %w", err)
}
zap.L().Info("orchestrator created")
return o, nil
}
@@ -112,14 +121,14 @@ func (o *Orchestrator) curTime() time.Time {
func (o *Orchestrator) ApplyConfig(cfg *v1.Config) error {
o.mu.Lock()
defer o.mu.Unlock()
o.config = cfg
// Update the config provided to the repo pool which is cached and diffed separately.
if err := o.repoPool.configProvider.Update(cfg); err != nil {
o.mu.Unlock()
return fmt.Errorf("failed to update repo pool config: %w", err)
}
o.mu.Unlock()
return o.ScheduleDefaultTasks(cfg)
}
@@ -128,35 +137,41 @@ func (o *Orchestrator) ScheduleDefaultTasks(config *v1.Config) error {
zap.L().Info("scheduling default tasks, waiting for task queue reset.")
removedTasks := o.taskQueue.Reset()
for _, t := range removedTasks {
if err := t.task.Cancel(v1.OperationStatus_STATUS_SYSTEM_CANCELLED); err != nil {
zap.L().Error("failed to cancel queued task", zap.String("task", t.task.Name()), zap.Error(err))
if t.Op == nil {
continue
}
if err := o.cancelHelper(t.Op, v1.OperationStatus_STATUS_SYSTEM_CANCELLED); err != nil {
zap.L().Error("failed to cancel queued task", zap.String("task", t.Task.Name()), zap.Error(err))
} else {
zap.L().Debug("queued task cancelled due to config change", zap.String("task", t.task.Name()))
zap.L().Debug("queued task cancelled due to config change", zap.String("task", t.Task.Name()))
}
}
zap.L().Info("reset task queue, scheduling new task set.")
// Requeue tasks that are affected by the config change.
o.ScheduleTask(&CollectGarbageTask{
orchestrator: o,
}, TaskPriorityDefault)
if err := o.ScheduleTask(tasks.NewCollectGarbageTask(), tasks.TaskPriorityDefault); err != nil {
return fmt.Errorf("schedule collect garbage task: %w", err)
}
for _, plan := range config.Plans {
if plan.Disabled {
continue
}
t, err := NewScheduledBackupTask(o, plan)
t, err := tasks.NewScheduledBackupTask(plan)
if err != nil {
return fmt.Errorf("schedule backup task for plan %q: %w", plan.Id, err)
}
o.ScheduleTask(t, TaskPriorityDefault)
if err := o.ScheduleTask(t, tasks.TaskPriorityDefault); err != nil {
return fmt.Errorf("schedule backup task for plan %q: %w", plan.Id, err)
}
}
return nil
}
func (o *Orchestrator) GetRepo(repoId string) (repo *RepoOrchestrator, err error) {
func (o *Orchestrator) GetRepoOrchestrator(repoId string) (repo *repo.RepoOrchestrator, err error) {
o.mu.Lock()
defer o.mu.Unlock()
@@ -167,130 +182,245 @@ func (o *Orchestrator) GetRepo(repoId string) (repo *RepoOrchestrator, err error
return r, nil
}
func (o *Orchestrator) GetPlan(planId string) (*v1.Plan, error) {
func (o *Orchestrator) GetRepo(repoId string) (*v1.Repo, error) {
o.mu.Lock()
defer o.mu.Unlock()
if o.config.Plans == nil {
return nil, ErrPlanNotFound
for _, r := range o.config.Repos {
if r.GetId() == repoId {
return r, nil
}
}
return nil, fmt.Errorf("get repo %q: %w", repoId, ErrRepoNotFound)
}
func (o *Orchestrator) GetPlan(planId string) (*v1.Plan, error) {
o.mu.Lock()
defer o.mu.Unlock()
for _, p := range o.config.Plans {
if p.Id == planId {
return p, nil
}
}
return nil, ErrPlanNotFound
return nil, fmt.Errorf("get plan %q: %w", planId, ErrPlanNotFound)
}
func (o *Orchestrator) CancelOperation(operationId int64, status v1.OperationStatus) error {
o.mu.Lock()
defer o.mu.Unlock()
// note: if the task is running the requested status will not be set.
if running := o.runningTask.Load(); running != nil && running.operationId == operationId {
running.cancel()
for _, c := range o.cancelNotify {
select {
case c <- operationId:
default:
}
}
o.mu.Unlock()
allTasks := o.taskQueue.GetAll()
idx := slices.IndexFunc(allTasks, func(t scheduledTask) bool {
return t.task.OperationId() == operationId
idx := slices.IndexFunc(allTasks, func(t stContainer) bool {
return t.Op != nil && t.Op.GetId() == operationId
})
if idx == -1 {
return nil
}
t := allTasks[idx]
o.taskQueue.Remove(t)
if err := t.task.Cancel(status); err != nil {
return fmt.Errorf("cancel task %q: %w", t.task.Name(), err)
if err := o.cancelHelper(t.Op, status); err != nil {
return fmt.Errorf("cancel operation: %w", err)
}
if nextTime := t.task.Next(t.runAt.Add(1 * time.Second)); nextTime != nil {
t.runAt = *nextTime
o.taskQueue.Enqueue(*nextTime, t.priority, t)
o.taskQueue.Remove(t)
if err := o.scheduleTaskHelper(t.Task, tasks.TaskPriorityDefault, t.RunAt); err != nil {
return fmt.Errorf("reschedule cancelled task: %w", err)
}
return nil
}
func (o *Orchestrator) cancelHelper(op *v1.Operation, status v1.OperationStatus) error {
op.Status = status
op.UnixTimeEndMs = time.Now().UnixMilli()
if err := o.OpLog.Update(op); err != nil {
return fmt.Errorf("update cancelled operation: %w", err)
}
return nil
}
// Run is the main orchestration loop. Cancel the context to stop the loop.
func (o *Orchestrator) Run(mainCtx context.Context) {
func (o *Orchestrator) Run(ctx context.Context) {
zap.L().Info("starting orchestrator loop")
// subscribe to cancel notifications.
o.mu.Lock()
cancelNotifyChan := make(chan int64, 10) // buffered to queue up cancel notifications.
o.cancelNotify = append(o.cancelNotify, cancelNotifyChan)
o.mu.Unlock()
defer func() {
o.mu.Lock()
if idx := slices.Index(o.cancelNotify, cancelNotifyChan); idx != -1 {
o.cancelNotify = slices.Delete(o.cancelNotify, idx, idx+1)
}
o.mu.Unlock()
}()
for {
if mainCtx.Err() != nil {
if ctx.Err() != nil {
zap.L().Info("shutting down orchestrator loop, context cancelled.")
break
}
t := o.taskQueue.Dequeue(mainCtx)
if t.task == nil {
t := o.taskQueue.Dequeue(ctx)
if t.Task == nil {
continue
}
zap.L().Info("running task", zap.String("task", t.task.Name()))
zap.L().Info("running task", zap.String("task", t.Task.Name()))
taskCtx, cancel := context.WithCancel(mainCtx)
logs := bytes.NewBuffer(nil)
taskCtx, cancelTaskCtx := context.WithCancel(ctx)
taskCtx = logging.ContextWithWriter(taskCtx, &ioutil.SynchronizedWriter{W: logs})
if swapped := o.runningTask.CompareAndSwap(nil, &taskExecutionInfo{
operationId: t.task.OperationId(),
cancel: cancel,
}); !swapped {
zap.L().Fatal("failed to start task, another task is already running. Was Run() called twice?")
}
go func() {
for {
select {
case <-taskCtx.Done():
return
case opID := <-cancelNotifyChan:
if t.Op != nil && opID == t.Op.GetId() {
cancelTaskCtx()
}
}
}
}()
start := time.Now()
err := t.task.Run(taskCtx)
runner := newTaskRunnerImpl(o, t.Task, t.Op)
op := t.Op
if op != nil {
op.UnixTimeStartMs = time.Now().UnixMilli()
if op.Status == v1.OperationStatus_STATUS_PENDING || op.Status == v1.OperationStatus_STATUS_UNKNOWN {
op.Status = v1.OperationStatus_STATUS_INPROGRESS
}
if op.Id != 0 {
if err := o.OpLog.Update(op); err != nil {
zap.S().Errorf("failed to add operation to oplog: %w", err)
}
} else {
if err := o.OpLog.Add(op); err != nil {
zap.S().Errorf("failed to add operation to oplog: %w", err)
}
}
}
err := t.Task.Run(taskCtx, t.ScheduledTask, runner)
if op != nil {
// write logs to log storage for this task.
if logs.Len() > 0 {
ref, err := o.logStore.Write(logs.Bytes())
if err != nil {
zap.S().Errorf("failed to write logs for task %q to log store: %v", t.Task.Name(), err)
} else {
op.Logref = ref
}
}
if err != nil {
if taskCtx.Err() != nil {
// task was cancelled
op.Status = v1.OperationStatus_STATUS_USER_CANCELLED
} else {
op.Status = v1.OperationStatus_STATUS_ERROR
}
op.DisplayMessage = err.Error()
}
op.UnixTimeEndMs = time.Now().UnixMilli()
if op.Status == v1.OperationStatus_STATUS_INPROGRESS {
op.Status = v1.OperationStatus_STATUS_SUCCESS
}
if e := o.OpLog.Update(op); e != nil {
zap.S().Errorf("failed to update operation in oplog: %v", e)
}
}
if err != nil {
zap.L().Error("task failed", zap.String("task", t.task.Name()), zap.Error(err), zap.Duration("duration", time.Since(start)))
zap.L().Error("task failed", zap.String("task", t.Task.Name()), zap.Error(err), zap.Duration("duration", time.Since(start)))
} else {
zap.L().Info("task finished", zap.String("task", t.task.Name()), zap.Duration("duration", time.Since(start)))
}
o.runningTask.Store(nil)
for _, cb := range t.callbacks {
cb(err)
zap.L().Info("task finished", zap.String("task", t.Task.Name()), zap.Duration("duration", time.Since(start)))
}
if t.config == o.config {
o.mu.Lock()
if t.configModno == o.config.Modno {
// Only reschedule tasks if the config hasn't changed since the task was scheduled.
o.ScheduleTask(t.task, t.priority)
if err := o.ScheduleTask(t.Task, tasks.TaskPriorityDefault); err != nil {
zap.L().Error("reschedule task", zap.String("task", t.Task.Name()), zap.Error(err))
}
}
o.mu.Unlock()
cancelTaskCtx()
go func() {
for _, cb := range t.callbacks {
cb(err)
}
}()
}
}
func (o *Orchestrator) ScheduleTask(t Task, priority int, callbacks ...func(error)) {
nextRun := t.Next(o.curTime())
if nextRun == nil {
return
// ScheduleTask schedules a task to run at the next available time.
// note that o.mu must not be held when calling this function.
func (o *Orchestrator) ScheduleTask(t tasks.Task, priority int, callbacks ...func(error)) error {
return o.scheduleTaskHelper(t, priority, o.curTime(), callbacks...)
}
func (o *Orchestrator) scheduleTaskHelper(t tasks.Task, priority int, curTime time.Time, callbacks ...func(error)) error {
nextRun := t.Next(curTime, newTaskRunnerImpl(o, t, nil))
if nextRun.Eq(tasks.NeverScheduledTask) {
return nil
}
zap.L().Info("scheduling task", zap.String("task", t.Name()), zap.String("runAt", nextRun.Format(time.RFC3339)))
o.taskQueue.Enqueue(*nextRun, priority, scheduledTask{
task: t,
runAt: *nextRun,
priority: priority,
callbacks: callbacks,
config: o.config,
})
nextRun.Task = t
stc := stContainer{
ScheduledTask: nextRun,
configModno: o.config.Modno,
callbacks: callbacks,
}
if stc.Op != nil {
stc.Op.PlanId = t.PlanID()
stc.Op.RepoId = t.RepoID()
stc.Op.Status = v1.OperationStatus_STATUS_PENDING
stc.Op.UnixTimeStartMs = nextRun.RunAt.UnixMilli()
if err := o.OpLog.Add(nextRun.Op); err != nil {
return fmt.Errorf("add operation to oplog: %w", err)
}
}
zap.L().Info("scheduling task", zap.String("task", t.Name()), zap.String("runAt", nextRun.RunAt.Format(time.RFC3339)))
o.taskQueue.Enqueue(nextRun.RunAt, priority, stc)
return nil
}
// resticRepoPool caches restic repos.
type resticRepoPool struct {
mu sync.Mutex
resticPath string
repos map[string]*RepoOrchestrator
repos map[string]*repo.RepoOrchestrator
configProvider config.ConfigStore
}
func newResticRepoPool(resticPath string, configProvider config.ConfigStore) *resticRepoPool {
return &resticRepoPool{
resticPath: resticPath,
repos: make(map[string]*RepoOrchestrator),
repos: make(map[string]*repo.RepoOrchestrator),
configProvider: configProvider,
}
}
func (rp *resticRepoPool) GetRepo(repoId string) (repo *RepoOrchestrator, err error) {
func (rp *resticRepoPool) GetRepo(repoId string) (*repo.RepoOrchestrator, error) {
cfg, err := rp.configProvider.Get()
if err != nil {
return nil, fmt.Errorf("failed to get config: %w", err)
@@ -315,19 +445,19 @@ func (rp *resticRepoPool) GetRepo(repoId string) (repo *RepoOrchestrator, err er
}
// Check if we already have a repo for this id, if we do return it.
repo, ok := rp.repos[repoId]
if ok && proto.Equal(repo.repoConfig, repoProto) {
return repo, nil
r, ok := rp.repos[repoId]
if ok && proto.Equal(r.Config(), repoProto) {
return r, nil
}
delete(rp.repos, repoId)
// Otherwise create a new repo.
repo, err = NewRepoOrchestrator(repoProto, rp.resticPath)
r, err = repo.NewRepoOrchestrator(repoProto, rp.resticPath)
if err != nil {
return nil, err
}
rp.repos[repoId] = repo
return repo, nil
rp.repos[repoId] = r
return r, nil
}
type taskExecutionInfo struct {
internal/orchestrator/orchestrator_test.go
+48 -39
View File
@@ -6,35 +6,44 @@ import (
"testing"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/config"
"github.com/garethgeorge/backrest/internal/orchestrator/tasks"
)
type testTask struct {
tasks.BaseTask
onRun func() error
onNext func(curTime time.Time) *time.Time
}
func (t *testTask) Name() string {
return "test"
var _ tasks.Task = &testTask{}
func newTestTask(onRun func() error, onNext func(curTime time.Time) *time.Time) tasks.Task {
return &testTask{
BaseTask: tasks.BaseTask{
TaskName: "test task",
TaskRepoID: "repo",
TaskPlanID: "plan",
},
onRun: onRun,
onNext: onNext,
}
}
func (t *testTask) Next(curTime time.Time, runner tasks.TaskRunner) tasks.ScheduledTask {
at := t.onNext(curTime)
if at == nil {
return tasks.NeverScheduledTask
}
return tasks.ScheduledTask{
Task: t,
RunAt: *at,
}
}
func (t *testTask) Next(now time.Time) *time.Time {
return t.onNext(now)
}
func (t *testTask) Run(ctx context.Context) error {
func (t *testTask) Run(ctx context.Context, st tasks.ScheduledTask, runner tasks.TaskRunner) error {
return t.onRun()
}
func (t *testTask) Cancel(withStatus v1.OperationStatus) error {
return nil
}
func (t *testTask) OperationId() int64 {
return 0
}
func TestTaskScheduling(t *testing.T) {
t.Parallel()
@@ -49,17 +58,17 @@ func TestTaskScheduling(t *testing.T) {
var wg sync.WaitGroup
wg.Add(1)
task := &testTask{
onRun: func() error {
task := newTestTask(
func() error {
wg.Done()
cancel()
return nil
},
onNext: func(t time.Time) *time.Time {
func(t time.Time) *time.Time {
t = t.Add(10 * time.Millisecond)
return &t
},
}
)
wg.Add(1)
go func() {
@@ -68,7 +77,7 @@ func TestTaskScheduling(t *testing.T) {
}()
// Act
orch.ScheduleTask(task, TaskPriorityDefault)
orch.ScheduleTask(task, tasks.TaskPriorityDefault)
// Assert passes if all tasks run and the orchestrator exists when cancelled.
wg.Wait()
@@ -98,22 +107,22 @@ func TestTaskRescheduling(t *testing.T) {
count := 0
ranTimes := 0
orch.ScheduleTask(&testTask{
onNext: func(t time.Time) *time.Time {
if count < 10 {
count += 1
return &t
}
return nil
},
onRun: func() error {
orch.ScheduleTask(newTestTask(
func() error {
ranTimes += 1
if ranTimes == 10 {
cancel()
}
return nil
},
}, TaskPriorityDefault)
func(t time.Time) *time.Time {
if count < 10 {
count += 1
return &t
}
return nil
},
), tasks.TaskPriorityDefault)
wg.Wait()
@@ -157,8 +166,12 @@ func TestSchedulerWait(t *testing.T) {
ran := make(chan struct{})
didRun := false
orch.ScheduleTask(&testTask{
onNext: func(t time.Time) *time.Time {
orch.ScheduleTask(newTestTask(
func() error {
close(ran)
return nil
},
func(t time.Time) *time.Time {
if didRun {
return nil
}
@@ -166,11 +179,7 @@ func TestSchedulerWait(t *testing.T) {
didRun = true
return &t
},
onRun: func() error {
close(ran)
return nil
},
}, TaskPriorityDefault)
), tasks.TaskPriorityDefault)
// Act
go orch.Run(context.Background())
@@ -192,7 +201,7 @@ func TestSchedulerWait(t *testing.T) {
t.Fatalf("should never run")
return nil
},
}, TaskPriorityDefault)
}, tasks.TaskPriorityDefault)
select {
case <-time.NewTimer(200 * time.Millisecond).C:
internal/orchestrator/repo/logging.go
+22
View File
@@ -0,0 +1,22 @@
package repo
import (
"context"
"github.com/garethgeorge/backrest/internal/ioutil"
"github.com/garethgeorge/backrest/internal/orchestrator/logging"
"github.com/garethgeorge/backrest/pkg/restic"
)
// pipeResticLogsToWriter sets the restic logger to write to the provided writer.
// returns a new context with the logger set and a function to flush the logs.
func forwardResticLogs(ctx context.Context) (context.Context, func()) {
capture := ioutil.NewOutputCapturer(8 * 1024)
return restic.ContextWithLogger(ctx, capture), func() {
writer := logging.WriterFromContext(ctx)
if writer == nil {
return
}
_, _ = writer.Write(capture.Bytes())
}
}
internal/orchestrator/repo.go → internal/orchestrator/repo/repo.go
+29 -4
View File
@@ -1,4 +1,4 @@
package orchestrator
package repo
import (
"context"
@@ -15,7 +15,6 @@ import (
"github.com/garethgeorge/backrest/pkg/restic"
"github.com/google/shlex"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
)
// RepoOrchestrator is responsible for managing a single repo.
@@ -70,10 +69,16 @@ func NewRepoOrchestrator(repoConfig *v1.Repo, resticPath string) (*RepoOrchestra
}
func (r *RepoOrchestrator) Init(ctx context.Context) error {
ctx, flush := forwardResticLogs(ctx)
defer flush()
return r.repo.Init(ctx)
}
func (r *RepoOrchestrator) Snapshots(ctx context.Context) ([]*restic.Snapshot, error) {
ctx, flush := forwardResticLogs(ctx)
defer flush()
snapshots, err := r.repo.Snapshots(ctx)
if err != nil {
return nil, fmt.Errorf("get snapshots for repo %v: %w", r.repoConfig.Id, err)
@@ -83,6 +88,9 @@ func (r *RepoOrchestrator) Snapshots(ctx context.Context) ([]*restic.Snapshot, e
}
func (r *RepoOrchestrator) SnapshotsForPlan(ctx context.Context, plan *v1.Plan) ([]*restic.Snapshot, error) {
ctx, flush := forwardResticLogs(ctx)
defer flush()
snapshots, err := r.repo.Snapshots(ctx, restic.WithFlags("--tag", tagForPlan(plan)))
if err != nil {
return nil, fmt.Errorf("get snapshots for plan %q: %w", plan.Id, err)
@@ -96,6 +104,7 @@ func (r *RepoOrchestrator) Backup(ctx context.Context, plan *v1.Plan, progressCa
r.mu.Lock()
defer r.mu.Unlock()
if !r.initialized {
if err := r.repo.Init(ctx, restic.WithEnviron()); err != nil {
return nil, fmt.Errorf("failed to initialize repo: %w", err)
@@ -134,6 +143,8 @@ func (r *RepoOrchestrator) Backup(ctx context.Context, plan *v1.Plan, progressCa
opts = append(opts, restic.WithFlags(args...))
}
ctx, flush := forwardResticLogs(ctx)
defer flush()
summary, err := r.repo.Backup(ctx, plan.Paths, progressCallback, opts...)
if err != nil {
return summary, fmt.Errorf("failed to backup: %w", err)
@@ -144,6 +155,9 @@ func (r *RepoOrchestrator) Backup(ctx context.Context, plan *v1.Plan, progressCa
}
func (r *RepoOrchestrator) ListSnapshotFiles(ctx context.Context, snapshotId string, path string) ([]*v1.LsEntry, error) {
ctx, flush := forwardResticLogs(ctx)
defer flush()
_, entries, err := r.repo.ListDirectory(ctx, snapshotId, path)
if err != nil {
return nil, fmt.Errorf("failed to list snapshot files: %w", err)
@@ -160,6 +174,8 @@ func (r *RepoOrchestrator) ListSnapshotFiles(ctx context.Context, snapshotId str
func (r *RepoOrchestrator) Forget(ctx context.Context, plan *v1.Plan) ([]*v1.ResticSnapshot, error) {
r.mu.Lock()
defer r.mu.Unlock()
ctx, flush := forwardResticLogs(ctx)
defer flush()
policy := plan.Retention
if policy == nil {
@@ -190,6 +206,8 @@ func (r *RepoOrchestrator) Forget(ctx context.Context, plan *v1.Plan) ([]*v1.Res
func (r *RepoOrchestrator) ForgetSnapshot(ctx context.Context, snapshotId string) error {
r.mu.Lock()
defer r.mu.Unlock()
ctx, flush := forwardResticLogs(ctx)
defer flush()
r.l.Debug("forget snapshot with ID", zap.String("snapshot", snapshotId), zap.String("repo", r.repoConfig.Id))
return r.repo.ForgetSnapshot(ctx, snapshotId)
@@ -198,6 +216,8 @@ func (r *RepoOrchestrator) ForgetSnapshot(ctx context.Context, snapshotId string
func (r *RepoOrchestrator) Prune(ctx context.Context, output io.Writer) error {
r.mu.Lock()
defer r.mu.Unlock()
ctx, flush := forwardResticLogs(ctx)
defer flush()
policy := r.repoConfig.PrunePolicy
if policy == nil {
@@ -224,6 +244,8 @@ func (r *RepoOrchestrator) Prune(ctx context.Context, output io.Writer) error {
func (r *RepoOrchestrator) Restore(ctx context.Context, snapshotId string, path string, target string, progressCallback func(event *v1.RestoreProgressEntry)) (*v1.RestoreProgressEntry, error) {
r.mu.Lock()
defer r.mu.Unlock()
ctx, flush := forwardResticLogs(ctx)
defer flush()
r.l.Debug("restore snapshot", zap.String("snapshot", snapshotId), zap.String("target", target))
@@ -250,9 +272,10 @@ func (r *RepoOrchestrator) UnlockIfAutoEnabled(ctx context.Context) error {
if !r.repoConfig.AutoUnlock {
return nil
}
r.mu.Lock()
defer r.mu.Unlock()
ctx, flush := forwardResticLogs(ctx)
defer flush()
zap.L().Debug("auto-unlocking repo", zap.String("repo", r.repoConfig.Id))
@@ -272,6 +295,8 @@ func (r *RepoOrchestrator) Unlock(ctx context.Context) error {
func (r *RepoOrchestrator) Stats(ctx context.Context) (*v1.RepoStats, error) {
r.mu.Lock()
defer r.mu.Unlock()
ctx, flush := forwardResticLogs(ctx)
defer flush()
r.l.Debug("getting repo stats", zap.String("repo", r.repoConfig.Id))
stats, err := r.repo.Stats(ctx)
@@ -286,7 +311,7 @@ func (r *RepoOrchestrator) Config() *v1.Repo {
if r == nil {
return nil
}
return proto.Clone(r.repoConfig).(*v1.Repo)
return r.repoConfig
}
func tagForPlan(plan *v1.Plan) string {
internal/orchestrator/repo_test.go → internal/orchestrator/repo/repo_test.go
+1 -1
View File
@@ -1,4 +1,4 @@
package orchestrator
package repo
import (
"context"
internal/orchestrator/scheduledtaskheap.go
-231
View File
@@ -1,231 +0,0 @@
package orchestrator
import (
"container/heap"
"context"
"sync"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
)
var taskQueueDefaultPollInterval = 3 * time.Minute
type taskQueue struct {
dequeueMu sync.Mutex
mu sync.Mutex
heap scheduledTaskHeapByTime
notify chan struct{}
ready scheduledTaskHeapByPriorityThenTime
pollInterval time.Duration
Now func() time.Time
}
func newTaskQueue(now func() time.Time) taskQueue {
return taskQueue{
heap: scheduledTaskHeapByTime{},
ready: scheduledTaskHeapByPriorityThenTime{},
pollInterval: taskQueueDefaultPollInterval,
Now: now,
}
}
func (t *taskQueue) curTime() time.Time {
if t.Now != nil {
return t.Now()
}
return time.Now()
}
func (t *taskQueue) Push(tasks ...scheduledTask) {
t.mu.Lock()
defer t.mu.Unlock()
for _, task := range tasks {
task := task
if task.task == nil {
panic("task cannot be nil")
}
heap.Push(&t.heap, &task)
}
if t.notify != nil {
t.notify <- struct{}{}
}
}
func (t *taskQueue) Reset() []*scheduledTask {
t.mu.Lock()
defer t.mu.Unlock()
oldTasks := t.heap.tasks
oldTasks = append(oldTasks, t.ready.tasks...)
t.heap.tasks = nil
t.ready.tasks = nil
if t.notify != nil {
t.notify <- struct{}{}
}
return oldTasks
}
func (t *taskQueue) Dequeue(ctx context.Context) *scheduledTask {
t.dequeueMu.Lock()
defer t.dequeueMu.Unlock()
t.mu.Lock()
defer t.mu.Unlock()
t.notify = make(chan struct{}, 10)
defer func() {
close(t.notify)
t.notify = nil
}()
for {
first, ok := t.heap.Peek().(*scheduledTask)
if !ok { // no tasks in heap.
if t.ready.Len() > 0 {
return heap.Pop(&t.ready).(*scheduledTask)
}
t.mu.Unlock()
select {
case <-ctx.Done():
t.mu.Lock()
return nil
case <-t.notify:
}
t.mu.Lock()
continue
}
now := t.curTime()
// if there's a task in the ready queue AND the first task isn't ready yet then immediately return the ready task.
ready, ok := t.ready.Peek().(*scheduledTask)
if ok && now.Before(first.runAt) {
heap.Pop(&t.ready)
return ready
}
t.mu.Unlock()
d := first.runAt.Sub(now)
if t.pollInterval > 0 && d > t.pollInterval {
// A poll interval may be set to work around clock changes
// e.g. when a laptop wakes from sleep or the system clock is adjusted.
d = t.pollInterval
}
timer := time.NewTimer(d)
select {
case <-timer.C:
t.mu.Lock()
if t.heap.Len() == 0 {
break
}
for {
first, ok := t.heap.Peek().(*scheduledTask)
if !ok {
break
}
if first.runAt.After(t.curTime()) {
// task is not yet ready to run
break
}
heap.Pop(&t.heap) // remove the task from the heap
heap.Push(&t.ready, first)
}
if t.ready.Len() == 0 {
break
}
return heap.Pop(&t.ready).(*scheduledTask)
case <-t.notify: // new task was added, loop again to ensure we have the earliest task.
t.mu.Lock()
if !timer.Stop() {
<-timer.C
}
case <-ctx.Done():
t.mu.Lock()
if !timer.Stop() {
<-timer.C
}
return nil
}
}
}
type scheduledTask struct {
task Task
runAt time.Time
priority int
callbacks []func(error)
config *v1.Config
}
func (s *scheduledTask) Less(other *scheduledTask) bool {
if s.priority != other.priority {
return s.priority > other.priority
}
return s.runAt.Before(other.runAt)
}
func (s scheduledTask) Eq(other scheduledTask) bool {
return s.task == other.task && s.runAt.Equal(other.runAt) && s.priority == other.priority && s.config == other.config
}
type scheduledTaskHeap struct {
tasks []*scheduledTask
comparator func(i, j *scheduledTask) bool
}
func (h *scheduledTaskHeap) Len() int {
return len(h.tasks)
}
func (h *scheduledTaskHeap) Swap(i, j int) {
h.tasks[i], h.tasks[j] = h.tasks[j], h.tasks[i]
}
func (h *scheduledTaskHeap) Push(x interface{}) {
h.tasks = append(h.tasks, x.(*scheduledTask))
}
func (h *scheduledTaskHeap) Pop() interface{} {
old := h.tasks
n := len(old)
x := old[n-1]
h.tasks = old[0 : n-1]
return x
}
func (h *scheduledTaskHeap) Peek() interface{} {
if len(h.tasks) == 0 {
return nil
}
return h.tasks[0]
}
type scheduledTaskHeapByTime struct {
scheduledTaskHeap
}
var _ heap.Interface = &scheduledTaskHeapByTime{}
func (h *scheduledTaskHeapByTime) Less(i, j int) bool {
return h.tasks[i].runAt.Before(h.tasks[j].runAt)
}
type scheduledTaskHeapByPriorityThenTime struct {
scheduledTaskHeap
}
var _ heap.Interface = &scheduledTaskHeapByPriorityThenTime{}
func (h *scheduledTaskHeapByPriorityThenTime) Less(i, j int) bool {
if h.tasks[i].priority != h.tasks[j].priority {
return h.tasks[i].priority > h.tasks[j].priority
}
return h.tasks[i].runAt.Before(h.tasks[j].runAt)
}
internal/orchestrator/task.go
-137
View File
@@ -1,137 +0,0 @@
package orchestrator
import (
"context"
"errors"
"fmt"
"sync/atomic"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/ioutil"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/pkg/restic"
"github.com/hashicorp/go-multierror"
"go.uber.org/zap"
)
type Task interface {
Name() string // huamn readable name for this task.
Next(now time.Time) *time.Time // when this task would like to be run.
Run(ctx context.Context) error // run the task.
Cancel(withStatus v1.OperationStatus) error // informat the task that it's scheduled execution will be skipped (either STATUS_USER_CANCELLED or STATUS_SYSTEM_CANCELLED).
OperationId() int64 // the id of the operation associated with this task (if any).
}
type TaskWithOperation struct {
orch *Orchestrator
op *v1.Operation
running atomic.Bool
}
func (t *TaskWithOperation) OperationId() int64 {
if t.op == nil {
return 0
}
return t.op.Id
}
func (t *TaskWithOperation) setOperation(op *v1.Operation) error {
if t.op != nil {
return errors.New("task already has an operation")
}
if err := t.orch.OpLog.Add(op); err != nil {
return fmt.Errorf("task failed to add operation to oplog: %v", err)
}
t.op = op
return nil
}
func (t *TaskWithOperation) runWithOpAndContext(ctx context.Context, do func(ctx context.Context, op *v1.Operation) error) error {
if t.op == nil {
return errors.New("task has no operation, a call to setOperation first is required")
}
if t.running.Load() {
return errors.New("task is already running")
}
t.running.Store(true)
defer t.running.Store(false)
defer func() {
t.op = nil
}()
return WithOperation(t.orch.OpLog, t.op, func() error {
capture := ioutil.NewOutputCapturer(32_000) // 32k of logs
ctx = restic.ContextWithLogger(ctx, capture)
err := do(ctx, t.op)
if bytes := capture.Bytes(); len(bytes) > 0 {
ref, e := t.orch.logStore.Write(bytes)
if e != nil {
errors.Join(err, fmt.Errorf("failed to write log to logstore: %w", e))
}
t.op.Logref = ref
zap.S().Debugf("wrote operation log to %v", ref)
}
return err
})
}
// Cancel marks a task as cancelled. Note that, unintuitively, it is actually an error to call cancel on a running task.
func (t *TaskWithOperation) Cancel(withStatus v1.OperationStatus) error {
if t.running.Load() {
return errors.New("cannot cancel a running task") // should never happen.
}
if t.op == nil {
return nil
}
t.op.Status = withStatus
t.op.UnixTimeEndMs = curTimeMillis()
if err := t.orch.OpLog.Update(t.op); err != nil {
return fmt.Errorf("failed to update operation %v in oplog: %w", t.op.Id, err)
}
t.op = nil
return nil
}
// WithOperation is a utility that creates an operation to track the function's execution.
// timestamps are automatically added and the status is automatically updated if an error occurs.
func WithOperation(oplog *oplog.OpLog, op *v1.Operation, do func() error) error {
op.UnixTimeStartMs = curTimeMillis() // update the start time from the planned time to the actual time.
if op.Status == v1.OperationStatus_STATUS_PENDING || op.Status == v1.OperationStatus_STATUS_UNKNOWN {
op.Status = v1.OperationStatus_STATUS_INPROGRESS
}
if op.Id != 0 {
if err := oplog.Update(op); err != nil {
return fmt.Errorf("failed to add operation to oplog: %w", err)
}
} else {
if err := oplog.Add(op); err != nil {
return fmt.Errorf("failed to add operation to oplog: %w", err)
}
}
err := do()
if err != nil {
op.Status = v1.OperationStatus_STATUS_ERROR
op.DisplayMessage = err.Error()
}
op.UnixTimeEndMs = curTimeMillis()
if op.Status == v1.OperationStatus_STATUS_INPROGRESS {
op.Status = v1.OperationStatus_STATUS_SUCCESS
}
if e := oplog.Update(op); e != nil {
return multierror.Append(err, fmt.Errorf("failed to update operation in oplog: %w", e))
}
return err
}
func timeToUnixMillis(t time.Time) int64 {
return t.Unix()*1000 + int64(t.Nanosecond()/1000000)
}
func curTimeMillis() int64 {
return timeToUnixMillis(time.Now())
}
internal/orchestrator/taskcollectgarbage.go
-134
View File
@@ -1,134 +0,0 @@
package orchestrator
import (
"context"
"fmt"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"go.uber.org/zap"
)
const (
gcStartupDelay = 5 * time.Second
gcInterval = 24 * time.Hour
// keep operations that are eligible for gc for 30 days OR up to a limit of 100 for any one plan.
// an operation is eligible for gc if:
// - it has no snapshot associated with it
// - it has a forgotten snapshot associated with it
gcHistoryAge = 30 * 24 * time.Hour
gcHistoryMaxCount = 1000
// keep stats operations for 1 year (they're small and useful for long term trends)
gcHistoryStatsAge = 365 * 24 * time.Hour
)
type CollectGarbageTask struct {
orchestrator *Orchestrator // owning orchestrator
firstRun bool
}
var _ Task = &CollectGarbageTask{}
func (t *CollectGarbageTask) Name() string {
return "collect garbage"
}
func (t *CollectGarbageTask) Next(now time.Time) *time.Time {
if !t.firstRun {
t.firstRun = true
runAt := now.Add(gcStartupDelay)
return &runAt
}
runAt := now.Add(gcInterval)
return &runAt
}
func (t *CollectGarbageTask) Run(ctx context.Context) error {
if err := t.gcOperations(); err != nil {
return fmt.Errorf("collecting garbage: %w", err)
}
return nil
}
func (t *CollectGarbageTask) gcOperations() error {
oplog := t.orchestrator.OpLog
// pass 1: identify forgotten snapshots.
snapshotIsForgotten := make(map[string]bool)
if err := oplog.ForAll(func(op *v1.Operation) error {
if snapshotOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok {
if snapshotOp.OperationIndexSnapshot.Forgot {
snapshotIsForgotten[snapshotOp.OperationIndexSnapshot.Snapshot.Id] = true
}
}
return nil
}); err != nil {
return fmt.Errorf("identifying forgotten snapshots: %w", err)
}
// pass 2: identify operations that are gc eligible
// - any operation that has no snapshot associated with it
// - any operation that has a forgotten snapshot associated with it
operationsByPlan := make(map[string][]gcOpInfo)
if err := oplog.ForAll(func(op *v1.Operation) error {
if op.SnapshotId == "" || snapshotIsForgotten[op.SnapshotId] {
_, isStats := op.Op.(*v1.Operation_OperationStats)
operationsByPlan[op.PlanId] = append(operationsByPlan[op.PlanId], gcOpInfo{
id: op.Id,
timestamp: op.UnixTimeStartMs,
isStats: isStats,
})
}
return nil
}); err != nil {
return fmt.Errorf("identifying gc eligible operations: %w", err)
}
var gcOps []int64
curTime := curTimeMillis()
for _, opInfos := range operationsByPlan {
if len(opInfos) >= gcHistoryMaxCount {
for _, opInfo := range opInfos[:len(opInfos)-gcHistoryMaxCount] {
gcOps = append(gcOps, opInfo.id)
}
opInfos = opInfos[len(opInfos)-gcHistoryMaxCount:]
}
// check if each operation timestamp is old.
for _, opInfo := range opInfos {
maxAgeForType := gcHistoryAge.Milliseconds()
if opInfo.isStats {
maxAgeForType = gcHistoryStatsAge.Milliseconds()
}
if curTime-opInfo.timestamp > maxAgeForType {
gcOps = append(gcOps, opInfo.id)
}
}
}
// pass 3: remove gc eligible operations
if err := oplog.Delete(gcOps...); err != nil {
return fmt.Errorf("removing gc eligible operations: %w", err)
}
zap.L().Info("collecting garbage",
zap.Int("forgotten_snapshots", len(snapshotIsForgotten)),
zap.Any("operations_removed", len(gcOps)))
return nil
}
func (t *CollectGarbageTask) Cancel(withStatus v1.OperationStatus) error {
return nil
}
func (t *CollectGarbageTask) OperationId() int64 {
return 0
}
type gcOpInfo struct {
id int64 // operation ID
timestamp int64 // unix time milliseconds
isStats bool // true if this is a stats operation
}
internal/orchestrator/taskforget.go
-121
View File
@@ -1,121 +0,0 @@
package orchestrator
import (
"context"
"fmt"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/hook"
"github.com/garethgeorge/backrest/internal/oplog/indexutil"
"github.com/hashicorp/go-multierror"
"go.uber.org/zap"
)
// ForgetTask tracks a forget operation.
type ForgetTask struct {
name string
TaskWithOperation
plan *v1.Plan
linkSnapshot string // snapshot to link the task to.
at *time.Time
}
var _ Task = &ForgetTask{}
func NewOneoffForgetTask(orchestrator *Orchestrator, plan *v1.Plan, linkSnapshot string, at time.Time) *ForgetTask {
return &ForgetTask{
TaskWithOperation: TaskWithOperation{
orch: orchestrator,
},
plan: plan,
at: &at,
linkSnapshot: linkSnapshot,
}
}
func (t *ForgetTask) Name() string {
return fmt.Sprintf("forget for plan %q", t.plan.Id)
}
func (t *ForgetTask) Next(now time.Time) *time.Time {
ret := t.at
if ret != nil {
t.at = nil
if err := t.setOperation(&v1.Operation{
PlanId: t.plan.Id,
RepoId: t.plan.Repo,
SnapshotId: t.linkSnapshot,
UnixTimeStartMs: timeToUnixMillis(*ret),
Status: v1.OperationStatus_STATUS_PENDING,
Op: &v1.Operation_OperationForget{},
}); err != nil {
zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err)
return nil
}
}
return ret
}
func (t *ForgetTask) Run(ctx context.Context) error {
if err := t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error {
forgetOp := &v1.Operation_OperationForget{
OperationForget: &v1.OperationForget{},
}
op.Op = forgetOp
var err error
repo, err := t.orch.GetRepo(t.plan.Repo)
if err != nil {
return fmt.Errorf("get repo %q: %w", t.plan.Repo, err)
}
err = repo.UnlockIfAutoEnabled(ctx)
if err != nil {
return fmt.Errorf("auto unlock repo %q: %w", t.plan.Repo, err)
}
forgot, err := repo.Forget(ctx, t.plan)
if err != nil {
return fmt.Errorf("forget: %w", err)
}
forgetOp.OperationForget.Forget = append(forgetOp.OperationForget.Forget, forgot...)
forgetOp.OperationForget.Policy = t.plan.Retention
var ops []*v1.Operation
for _, forgot := range forgot {
if e := t.orch.OpLog.ForEachBySnapshotId(forgot.Id, indexutil.CollectAll(), func(op *v1.Operation) error {
ops = append(ops, op)
return nil
}); e != nil {
err = multierror.Append(err, fmt.Errorf("cleanup snapshot %v: %w", forgot.Id, e))
}
}
for _, op := range ops {
if indexOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok {
indexOp.OperationIndexSnapshot.Forgot = true
if e := t.orch.OpLog.Update(op); err != nil {
err = multierror.Append(err, fmt.Errorf("mark index snapshot %v as forgotten: %w", op.Id, e))
continue
}
}
}
if len(forgot) > 0 {
t.orch.ScheduleTask(NewOneoffPruneTask(t.orch, t.plan, time.Now(), false), TaskPriorityPrune)
}
return err
}); err != nil {
repo, _ := t.orch.GetRepo(t.plan.Repo)
_ = t.orch.hookExecutor.ExecuteHooks(repo.Config(), t.plan, []v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Task: t.Name(),
Error: err.Error(),
SnapshotId: t.linkSnapshot,
})
}
return nil
}
internal/orchestrator/taskforgetsnapshot.go
-94
View File
@@ -1,94 +0,0 @@
package orchestrator
import (
"context"
"fmt"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/oplog/indexutil"
"github.com/hashicorp/go-multierror"
"go.uber.org/zap"
)
// ForgetTask tracks a forget operation.
type ForgetSnapshotTask struct {
TaskWithOperation
repoId string
planId string
forgetSnapshot string
at *time.Time
}
var _ Task = &ForgetSnapshotTask{}
func NewOneoffForgetSnapshotTask(orchestrator *Orchestrator, repoId, planId, forgetSnapshot string, at time.Time) *ForgetSnapshotTask {
return &ForgetSnapshotTask{
TaskWithOperation: TaskWithOperation{
orch: orchestrator,
},
repoId: repoId,
planId: planId,
at: &at,
forgetSnapshot: forgetSnapshot,
}
}
func (t *ForgetSnapshotTask) Name() string {
return fmt.Sprintf("forget snapshot %q", t.forgetSnapshot)
}
func (t *ForgetSnapshotTask) Next(now time.Time) *time.Time {
ret := t.at
if ret != nil {
t.at = nil
if err := t.setOperation(&v1.Operation{
PlanId: t.planId,
RepoId: t.repoId,
UnixTimeStartMs: timeToUnixMillis(*ret),
Status: v1.OperationStatus_STATUS_PENDING,
Op: &v1.Operation_OperationForget{},
}); err != nil {
zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err)
return nil
}
}
return ret
}
func (t *ForgetSnapshotTask) Run(ctx context.Context) error {
id := t.op.Id
if err := t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error {
repo, err := t.orch.GetRepo(t.repoId)
if err != nil {
return fmt.Errorf("get repo %q: %w", t.repoId, err)
}
// Find snapshot to forget
var ops []*v1.Operation
t.orch.OpLog.ForEachBySnapshotId(t.forgetSnapshot, indexutil.CollectAll(), func(op *v1.Operation) error {
ops = append(ops, op)
return nil
})
for _, op := range ops {
if indexOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok {
err := repo.ForgetSnapshot(ctx, op.SnapshotId)
if err != nil {
return fmt.Errorf("forget %q: %w", op.SnapshotId, err)
}
indexOp.OperationIndexSnapshot.Forgot = true
if e := t.orch.OpLog.Update(op); err != nil {
err = multierror.Append(err, fmt.Errorf("mark index snapshot %v as forgotten: %w", op.Id, e))
continue
}
}
}
return err
}); err != nil {
return err
}
return t.orch.OpLog.Delete(id)
}
internal/orchestrator/taskprune.go
-203
View File
@@ -1,203 +0,0 @@
package orchestrator
import (
"bytes"
"context"
"fmt"
"sync"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/hook"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/oplog/indexutil"
"go.uber.org/zap"
)
// PruneTask tracks a forget operation.
type PruneTask struct {
TaskWithOperation
plan *v1.Plan
at *time.Time
force bool
}
var _ Task = &PruneTask{}
func NewOneoffPruneTask(orchestrator *Orchestrator, plan *v1.Plan, at time.Time, force bool) *PruneTask {
return &PruneTask{
TaskWithOperation: TaskWithOperation{
orch: orchestrator,
},
plan: plan,
at: &at,
force: force, // overrides the PrunePolicy's MaxFrequencyDays
}
}
func (t *PruneTask) Name() string {
return fmt.Sprintf("prune for plan %q", t.plan.Id)
}
func (t *PruneTask) Next(now time.Time) *time.Time {
shouldRun, err := t.shouldRun(now)
if err != nil {
zap.S().Errorf("task %v failed to check if it should run: %v", t.Name(), err)
}
if !shouldRun {
return nil
}
ret := t.at
if ret != nil {
t.at = nil
if err := t.setOperation(&v1.Operation{
PlanId: t.plan.Id,
RepoId: t.plan.Repo,
UnixTimeStartMs: timeToUnixMillis(*ret),
Status: v1.OperationStatus_STATUS_PENDING,
Op: &v1.Operation_OperationPrune{},
}); err != nil {
zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err)
return nil
}
}
return ret
}
func (t *PruneTask) shouldRun(now time.Time) (bool, error) {
if t.force {
return true, nil
}
repo, err := t.orch.GetRepo(t.plan.Repo)
if err != nil {
return false, fmt.Errorf("get repo %v: %w", t.plan.Repo, err)
}
nextPruneTime, err := t.getNextPruneTime(repo, repo.repoConfig.PrunePolicy)
if err != nil {
return false, fmt.Errorf("get next prune time: %w", err)
}
return nextPruneTime.Before(now), nil
}
func (t *PruneTask) getNextPruneTime(repo *RepoOrchestrator, policy *v1.PrunePolicy) (time.Time, error) {
var lastPruneTime time.Time
t.orch.OpLog.ForEachByRepo(t.plan.Repo, indexutil.Reversed(indexutil.CollectAll()), func(op *v1.Operation) error {
if _, ok := op.Op.(*v1.Operation_OperationPrune); ok {
lastPruneTime = time.Unix(0, op.UnixTimeStartMs*int64(time.Millisecond))
return oplog.ErrStopIteration
}
return nil
})
if repo.repoConfig.PrunePolicy != nil {
return lastPruneTime.Add(time.Duration(repo.repoConfig.PrunePolicy.MaxFrequencyDays) * 24 * time.Hour), nil
} else {
return lastPruneTime.Add(7 * 24 * time.Hour), nil // default to 7 days.
}
}
func (t *PruneTask) Run(ctx context.Context) error {
if err := t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error {
repo, err := t.orch.GetRepo(t.plan.Repo)
if err != nil {
return fmt.Errorf("get repo %v: %w", t.plan.Repo, err)
}
err = repo.UnlockIfAutoEnabled(ctx)
if err != nil {
return fmt.Errorf("auto unlock repo %q: %w", t.plan.Repo, err)
}
opPrune := &v1.Operation_OperationPrune{
OperationPrune: &v1.OperationPrune{},
}
op.Op = opPrune
ctx, cancel := context.WithCancel(ctx)
interval := time.NewTicker(1 * time.Second)
defer interval.Stop()
var buf synchronizedBuffer
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-interval.C:
output := buf.String()
if len(output) > 8*1024 { // only provide live status upto the first 8K of output.
output = output[:len(output)-8*1024]
}
if opPrune.OperationPrune.Output != output {
opPrune.OperationPrune.Output = buf.String()
if err := t.orch.OpLog.Update(op); err != nil {
zap.L().Error("update prune operation with status output", zap.Error(err))
}
}
case <-ctx.Done():
return
}
}
}()
if err := repo.Prune(ctx, &buf); err != nil {
cancel()
return fmt.Errorf("prune: %w", err)
}
cancel()
wg.Wait()
// TODO: it would be best to store the output in separate storage for large status data.
output := buf.String()
if len(output) > 8*1024 { // only save the first 4K of output.
output = output[:len(output)-8*1024]
}
op.Op = &v1.Operation_OperationPrune{
OperationPrune: &v1.OperationPrune{
Output: output,
},
}
return nil
}); err != nil {
repo, _ := t.orch.GetRepo(t.plan.Repo)
_ = t.orch.hookExecutor.ExecuteHooks(repo.Config(), t.plan, []v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Task: t.Name(),
Error: err.Error(),
})
return err
}
t.orch.ScheduleTask(NewOneoffStatsTask(t.orch, t.plan.Repo, t.plan.Id, time.Now()), TaskPriorityStats)
return nil
}
// synchronizedBuffer is used for collecting prune command's output
type synchronizedBuffer struct {
mu sync.Mutex
buf bytes.Buffer
}
func (w *synchronizedBuffer) Write(p []byte) (n int, err error) {
w.mu.Lock()
defer w.mu.Unlock()
return w.buf.Write(p)
}
func (w *synchronizedBuffer) String() string {
w.mu.Lock()
defer w.mu.Unlock()
return w.buf.String()
}
internal/orchestrator/taskrestore.go
-126
View File
@@ -1,126 +0,0 @@
package orchestrator
import (
"context"
"errors"
"fmt"
"sync"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/hook"
"go.uber.org/zap"
)
type RestoreTaskOpts struct {
RepoId string // optional
PlanId string // optional
SnapshotId string // required
Path string // required
Target string // required
}
// RestoreTask tracks a forget operation.
type RestoreTask struct {
TaskWithOperation
restoreOpts RestoreTaskOpts
at *time.Time
}
var _ Task = &RestoreTask{}
func NewOneoffRestoreTask(orchestrator *Orchestrator, opts RestoreTaskOpts, at time.Time) *RestoreTask {
return &RestoreTask{
TaskWithOperation: TaskWithOperation{
orch: orchestrator,
},
restoreOpts: opts,
at: &at,
}
}
func (t *RestoreTask) Name() string {
return fmt.Sprintf("restore snapshot %v in repo %v", t.restoreOpts.SnapshotId, t.restoreOpts.RepoId)
}
func (t *RestoreTask) Next(now time.Time) *time.Time {
ret := t.at
if ret != nil {
t.at = nil
if err := t.setOperation(&v1.Operation{
PlanId: t.restoreOpts.PlanId,
RepoId: t.restoreOpts.RepoId,
SnapshotId: t.restoreOpts.SnapshotId,
UnixTimeStartMs: timeToUnixMillis(*ret),
Status: v1.OperationStatus_STATUS_PENDING,
Op: &v1.Operation_OperationRestore{},
}); err != nil {
zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err)
return nil
}
}
return ret
}
func (t *RestoreTask) Run(ctx context.Context) error {
if t.restoreOpts.SnapshotId == "" || t.restoreOpts.Path == "" || t.restoreOpts.Target == "" {
return errors.New("snapshotId, path, and target are required")
}
if err := t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error {
forgetOp := &v1.Operation_OperationRestore{
OperationRestore: &v1.OperationRestore{
Path: t.restoreOpts.Path,
Target: t.restoreOpts.Target,
},
}
op.Op = forgetOp
op.UnixTimeStartMs = curTimeMillis()
repo, err := t.orch.GetRepo(t.restoreOpts.RepoId)
if err != nil {
return fmt.Errorf("couldn't get repo %q: %w", t.restoreOpts.RepoId, err)
}
var sendWg sync.WaitGroup
lastSent := time.Now() // debounce progress updates, these can endup being very frequent.
summary, err := repo.Restore(ctx, t.restoreOpts.SnapshotId, t.restoreOpts.Path, t.restoreOpts.Target, func(entry *v1.RestoreProgressEntry) {
sendWg.Wait()
if time.Since(lastSent) < 1*time.Second {
return
}
lastSent = time.Now()
zap.S().Infof("restore progress: %v", entry)
forgetOp.OperationRestore.Status = entry
sendWg.Add(1)
go func() {
if err := t.orch.OpLog.Update(op); err != nil {
zap.S().Errorf("failed to update oplog with progress for restore: %v", err)
}
sendWg.Done()
}()
})
if err != nil {
return fmt.Errorf("restore failed: %w", err)
}
forgetOp.OperationRestore.Status = summary
return nil
}); err != nil {
if t.restoreOpts.RepoId != "" {
repo, _ := t.orch.GetRepo(t.restoreOpts.RepoId)
_ = t.orch.hookExecutor.ExecuteHooks(repo.Config(), nil, []v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Task: t.Name(),
Error: err.Error(),
SnapshotId: t.restoreOpts.SnapshotId,
})
}
return err
}
return nil
}
internal/orchestrator/taskrunnerimpl.go
+104
View File
@@ -0,0 +1,104 @@
package orchestrator
import (
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/hook"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/orchestrator/repo"
"github.com/garethgeorge/backrest/internal/orchestrator/tasks"
)
// taskRunnerImpl is an implementation of TaskRunner for the default orchestrator.
type taskRunnerImpl struct {
orchestrator *Orchestrator
t tasks.Task
op *v1.Operation
repo *v1.Repo // cache, populated on first call to Repo()
plan *v1.Plan // cache, populated on first call to Plan()
}
var _ tasks.TaskRunner = &taskRunnerImpl{}
func (t *taskRunnerImpl) FindRepo() (*v1.Repo, error) {
if t.repo != nil {
return t.repo, nil
}
var err error
t.repo, err = t.orchestrator.GetRepo(t.t.RepoID())
return t.repo, err
}
func (t *taskRunnerImpl) FindPlan() (*v1.Plan, error) {
if t.plan != nil {
return t.plan, nil
}
var err error
t.plan, err = t.orchestrator.GetPlan(t.t.PlanID())
return t.plan, err
}
func newTaskRunnerImpl(orchestrator *Orchestrator, task tasks.Task, op *v1.Operation) *taskRunnerImpl {
return &taskRunnerImpl{
orchestrator: orchestrator,
t: task,
op: op,
}
}
func (t *taskRunnerImpl) CreateOperation(op *v1.Operation) error {
return t.orchestrator.OpLog.Add(op)
}
func (t *taskRunnerImpl) UpdateOperation(op *v1.Operation) error {
return t.orchestrator.OpLog.Update(op)
}
func (t *taskRunnerImpl) Orchestrator() *Orchestrator {
return t.orchestrator
}
func (t *taskRunnerImpl) OpLog() *oplog.OpLog {
return t.orchestrator.OpLog
}
func (t *taskRunnerImpl) ExecuteHooks(events []v1.Hook_Condition, vars hook.HookVars) error {
repoID := t.t.RepoID()
planID := t.t.PlanID()
var repo *v1.Repo
var plan *v1.Plan
if repoID != "" {
var err error
repo, err = t.FindRepo()
if err != nil {
return err
}
}
if planID != "" {
var err error
plan, err = t.FindPlan()
if err != nil {
return err
}
}
var flowID int64
if t.op != nil {
flowID = t.op.FlowId
}
return t.orchestrator.hookExecutor.ExecuteHooks(flowID, repo, plan, events, vars)
}
func (t *taskRunnerImpl) GetRepo(repoID string) (*v1.Repo, error) {
return t.orchestrator.GetRepo(repoID)
}
func (t *taskRunnerImpl) GetPlan(planID string) (*v1.Plan, error) {
return t.orchestrator.GetPlan(planID)
}
func (t *taskRunnerImpl) GetRepoOrchestrator(repoID string) (*repo.RepoOrchestrator, error) {
return t.orchestrator.GetRepoOrchestrator(repoID)
}
func (t *taskRunnerImpl) ScheduleTask(task tasks.Task, priority int) error {
return t.orchestrator.ScheduleTask(task, priority)
}
internal/orchestrator/tasks/flowidutil.go
+24
View File
@@ -0,0 +1,24 @@
package tasks
import (
"fmt"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/oplog/indexutil"
)
// FlowIDForSnapshotID returns the flow ID associated with the backup task that created snapshot ID or 0 if not found.
func FlowIDForSnapshotID(oplog *oplog.OpLog, snapshotID string) (int64, error) {
var flowID int64
if err := oplog.ForEachBySnapshotId(snapshotID, indexutil.CollectAll(), func(op *v1.Operation) error {
if _, ok := op.Op.(*v1.Operation_OperationBackup); !ok {
return nil
}
flowID = op.FlowId
return nil
}); err != nil {
return 0, fmt.Errorf("get flow id for snapshot %q: %w", snapshotID, err)
}
return flowID, nil
}
internal/orchestrator/tasks/loggingutil.go
+26
View File
@@ -0,0 +1,26 @@
package tasks
import (
"context"
"github.com/garethgeorge/backrest/internal/orchestrator/logging"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
// LoggerFromContext returns a logger from the context, or the global logger if none is found.
// this is somewhat expensive, it should be called once per task.
func Logger(ctx context.Context, task Task) *zap.Logger {
writer := logging.WriterFromContext(ctx)
if writer == nil {
return zap.L()
}
p := zap.NewProductionEncoderConfig()
p.EncodeTime = zapcore.ISO8601TimeEncoder
fe := zapcore.NewJSONEncoder(p)
l := zap.New(zapcore.NewTee(
zap.L().Core(),
zapcore.NewCore(fe, zapcore.AddSync(writer), zapcore.DebugLevel),
))
return l
}
internal/orchestrator/tasks/task.go
+156
View File
@@ -0,0 +1,156 @@
package tasks
import (
"context"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/hook"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/orchestrator/repo"
"google.golang.org/protobuf/proto"
)
var NeverScheduledTask = ScheduledTask{}
const (
PlanForUnassociatedOperations = "_unassociated_"
TaskPriorityStats = -1
TaskPriorityDefault = 0
TaskPriorityInteractive = 1 << 1
TaskPriorityIndexSnapshots = 1 << 2
TaskPriorityForget = 1 << 3
TaskPriorityPrune = 1 << 4
)
// TaskRunner is an interface for running tasks. It is used by tasks to create operations and write logs.
type TaskRunner interface {
// CreateOperation creates the operation in storage and sets the operation ID in the task.
CreateOperation(*v1.Operation) error
// UpdateOperation updates the operation in storage. It must be called after CreateOperation.
UpdateOperation(*v1.Operation) error
// ExecuteHooks
ExecuteHooks(events []v1.Hook_Condition, vars hook.HookVars) error
// OpLog returns the oplog for the operations.
OpLog() *oplog.OpLog
// GetRepo returns the repo with the given ID.
GetRepo(repoID string) (*v1.Repo, error)
// GetPlan returns the plan with the given ID.
GetPlan(planID string) (*v1.Plan, error)
// GetRepoOrchestrator returns the orchestrator for the repo with the given ID.
GetRepoOrchestrator(repoID string) (*repo.RepoOrchestrator, error)
// ScheduleTask schedules a task to run at a specific time.
ScheduleTask(task Task, priority int) error
}
// ScheduledTask is a task that is scheduled to run at a specific time.
type ScheduledTask struct {
Task Task // the task to run
RunAt time.Time // the time at which the task should be run.
Op *v1.Operation // operation associated with this execution of the task.
configModno int32 // used by to detect if the config has changed since this task was scheduled.
callbacks []func(error) // callbacks to run after the task is run.
}
func (s ScheduledTask) Eq(other ScheduledTask) bool {
return s.Task == other.Task && s.RunAt.Equal(other.RunAt)
}
func (s ScheduledTask) Less(other ScheduledTask) bool {
if s.RunAt.Equal(other.RunAt) {
return s.Task.Name() < other.Task.Name()
}
return s.RunAt.Before(other.RunAt)
}
func (s ScheduledTask) cancel(oplog *oplog.OpLog) error {
if s.Op == nil {
return nil
}
opCopy := proto.Clone(s.Op).(*v1.Operation)
opCopy.Status = v1.OperationStatus_STATUS_SYSTEM_CANCELLED
opCopy.DisplayMessage = "operation cancelled"
opCopy.UnixTimeEndMs = curTimeMillis()
if err := oplog.Update(opCopy); err != nil {
return err
}
return nil
}
// Task is a task that can be scheduled to run at a specific time.
type Task interface {
Name() string // human readable name for this task.
Next(now time.Time, runner TaskRunner) ScheduledTask // returns the next scheduled task.
Run(ctx context.Context, st ScheduledTask, runner TaskRunner) error // run the task.
PlanID() string // the ID of the plan this task is associated with.
RepoID() string // the ID of the repo this task is associated with.
}
type BaseTask struct {
TaskName string
TaskPlanID string
TaskRepoID string
}
func (b BaseTask) Name() string {
return b.TaskName
}
func (b BaseTask) PlanID() string {
return b.TaskPlanID
}
func (b BaseTask) RepoID() string {
return b.TaskRepoID
}
type OneoffTask struct {
BaseTask
RunAt time.Time
FlowID int64 // the ID of the flow this task is associated with.
DidSchedule bool
ProtoOp *v1.Operation // the prototype operation for this class of task.
}
func (o *OneoffTask) Next(now time.Time, runner TaskRunner) ScheduledTask {
if o.DidSchedule {
return NeverScheduledTask
}
o.DidSchedule = true
var op *v1.Operation
if o.ProtoOp != nil {
op = proto.Clone(o.ProtoOp).(*v1.Operation)
op.PlanId = o.PlanID()
op.RepoId = o.RepoID()
op.FlowId = o.FlowID
op.UnixTimeStartMs = timeToUnixMillis(o.RunAt) // TODO: this should be updated before Run is called.
op.Status = v1.OperationStatus_STATUS_PENDING
}
return ScheduledTask{
RunAt: o.RunAt,
Op: op,
}
}
type GenericOneoffTask struct {
BaseTask
OneoffTask
Do func(ctx context.Context, st ScheduledTask, runner TaskRunner) error
}
func (g *GenericOneoffTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner) error {
return g.Do(ctx, st, runner)
}
func timeToUnixMillis(t time.Time) int64 {
return t.Unix()*1000 + int64(t.Nanosecond()/1000000)
}
func curTimeMillis() int64 {
return timeToUnixMillis(time.Now())
}
internal/orchestrator/taskbackup.go → internal/orchestrator/tasks/taskbackup.go
+50 -65
View File
@@ -1,4 +1,4 @@
package orchestrator
package tasks
import (
"context"
@@ -20,26 +20,24 @@ var maxBackupErrorHistoryLength = 20 // arbitrary limit on the number of file re
// BackupTask is a scheduled backup operation.
type BackupTask struct {
name string
TaskWithOperation
plan *v1.Plan
BaseTask
scheduler func(curTime time.Time) *time.Time
}
var _ Task = &BackupTask{}
func NewScheduledBackupTask(orchestrator *Orchestrator, plan *v1.Plan) (*BackupTask, error) {
func NewScheduledBackupTask(plan *v1.Plan) (*BackupTask, error) {
sched, err := cronexpr.ParseInLocation(plan.Cron, time.Now().Location().String())
if err != nil {
return nil, fmt.Errorf("failed to parse schedule %q: %w", plan.Cron, err)
}
return &BackupTask{
name: fmt.Sprintf("backup for plan %q", plan.Id),
TaskWithOperation: TaskWithOperation{
orch: orchestrator,
BaseTask: BaseTask{
TaskName: fmt.Sprintf("backup for plan %q", plan.Id),
TaskRepoID: plan.Repo,
TaskPlanID: plan.Id,
},
plan: plan,
scheduler: func(curTime time.Time) *time.Time {
next := sched.Next(curTime)
return &next
@@ -47,14 +45,14 @@ func NewScheduledBackupTask(orchestrator *Orchestrator, plan *v1.Plan) (*BackupT
}, nil
}
func NewOneoffBackupTask(orchestrator *Orchestrator, plan *v1.Plan, at time.Time) *BackupTask {
func NewOneoffBackupTask(plan *v1.Plan, at time.Time) *BackupTask {
didOnce := false
return &BackupTask{
name: fmt.Sprintf("onetime backup for plan %q", plan.Id),
TaskWithOperation: TaskWithOperation{
orch: orchestrator,
BaseTask: BaseTask{
TaskName: fmt.Sprintf("backup for plan %q", plan.Id),
TaskRepoID: plan.Repo,
TaskPlanID: plan.Id,
},
plan: plan,
scheduler: func(curTime time.Time) *time.Time {
if didOnce {
return nil
@@ -65,72 +63,55 @@ func NewOneoffBackupTask(orchestrator *Orchestrator, plan *v1.Plan, at time.Time
}
}
func (t *BackupTask) Name() string {
return t.name
}
func (t *BackupTask) Next(now time.Time) *time.Time {
func (t *BackupTask) Next(now time.Time, runner TaskRunner) ScheduledTask {
next := t.scheduler(now)
if next == nil {
return nil
return NeverScheduledTask
}
if err := t.setOperation(&v1.Operation{
PlanId: t.plan.Id,
RepoId: t.plan.Repo,
UnixTimeStartMs: timeToUnixMillis(*next),
Status: v1.OperationStatus_STATUS_PENDING,
Op: &v1.Operation_OperationBackup{},
}); err != nil {
zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err)
return ScheduledTask{
Task: t,
RunAt: *next,
Op: &v1.Operation{
PlanId: t.PlanID(),
RepoId: t.RepoID(),
UnixTimeStartMs: (*next).UnixMilli(),
Status: v1.OperationStatus_STATUS_PENDING,
Op: &v1.Operation_OperationBackup{},
},
}
return next
}
func (t *BackupTask) Run(ctx context.Context) error {
return t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error {
return backupHelper(ctx, t, t.orch, t.plan, op)
})
}
func (t *BackupTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner) error {
l := Logger(ctx, st.Task)
// backupHelper does a backup.
func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan *v1.Plan, op *v1.Operation) error {
startTime := time.Now()
op := st.Op
backupOp := &v1.Operation_OperationBackup{
OperationBackup: &v1.OperationBackup{},
}
op.Op = backupOp
zap.L().Info("Starting backup", zap.String("plan", plan.Id), zap.Int64("opId", op.Id))
repo, err := orchestrator.GetRepo(plan.Repo)
repo, err := runner.GetRepoOrchestrator(t.RepoID())
if err != nil {
return fmt.Errorf("couldn't get repo %q: %w", plan.Repo, err)
return err
}
// Run start hooks e.g. preflight checks and backup start notifications.
if err := orchestrator.hookExecutor.ExecuteHooks(repo.Config(), plan, []v1.Hook_Condition{
plan, err := runner.GetPlan(t.PlanID())
if err != nil {
return err
}
if err := runner.ExecuteHooks([]v1.Hook_Condition{
v1.Hook_CONDITION_SNAPSHOT_START,
}, hook.HookVars{
Task: t.Name(),
}); err != nil {
}, hook.HookVars{}); err != nil {
var cancelErr *hook.HookErrorRequestCancel
if errors.As(err, &cancelErr) {
op.Status = v1.OperationStatus_STATUS_USER_CANCELLED // user visible cancelled status
op.DisplayMessage = err.Error()
return nil
}
// If the snapshot start hook fails we trigger error notification hooks.
retErr := fmt.Errorf("hook failed: %w", err)
_ = orchestrator.hookExecutor.ExecuteHooks(repo.Config(), plan, []v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Task: t.Name(),
Error: retErr.Error(),
})
return retErr
return fmt.Errorf("hook failed: %w", err)
}
var sendWg sync.WaitGroup
@@ -172,13 +153,12 @@ func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan
sendWg.Add(1)
go func() {
if err := orchestrator.OpLog.Update(op); err != nil {
if err := runner.UpdateOperation(op); err != nil {
zap.S().Errorf("failed to update oplog with progress for backup: %v", err)
}
sendWg.Done()
}()
})
sendWg.Wait()
if summary == nil {
@@ -193,16 +173,17 @@ func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan
if err != nil {
vars.Error = err.Error()
if !errors.Is(err, restic.ErrPartialBackup) {
_ = orchestrator.hookExecutor.ExecuteHooks(repo.Config(), plan, []v1.Hook_Condition{
v1.Hook_CONDITION_SNAPSHOT_ERROR, v1.Hook_CONDITION_ANY_ERROR,
runner.ExecuteHooks([]v1.Hook_Condition{
v1.Hook_CONDITION_SNAPSHOT_ERROR,
v1.Hook_CONDITION_ANY_ERROR,
}, vars)
return fmt.Errorf("repo.Backup for repo %q: %w", plan.Repo, err)
return err
}
op.Status = v1.OperationStatus_STATUS_WARNING
op.DisplayMessage = "Partial backup, some files may not have been read completely."
}
orchestrator.hookExecutor.ExecuteHooks(repo.Config(), plan, []v1.Hook_Condition{
runner.ExecuteHooks([]v1.Hook_Condition{
v1.Hook_CONDITION_SNAPSHOT_END,
}, vars)
@@ -212,14 +193,18 @@ func backupHelper(ctx context.Context, t Task, orchestrator *Orchestrator, plan
return fmt.Errorf("expected a final backup progress entry, got nil")
}
zap.L().Info("Backup complete", zap.String("plan", plan.Id), zap.Duration("duration", time.Since(startTime)), zap.Any("summary", backupOp.OperationBackup.LastStatus))
l.Info("backup complete", zap.String("plan", plan.Id), zap.Duration("duration", time.Since(startTime)), zap.Any("summary", backupOp.OperationBackup.LastStatus))
// schedule followup tasks
at := time.Now()
if _, ok := plan.Retention.GetPolicy().(*v1.RetentionPolicy_PolicyKeepAll); plan.Retention != nil && !ok {
orchestrator.ScheduleTask(NewOneoffForgetTask(orchestrator, plan, op.SnapshotId, at), TaskPriorityForget)
if err := runner.ScheduleTask(NewOneoffForgetTask(t.RepoID(), t.PlanID(), op.FlowId, at), TaskPriorityForget); err != nil {
return fmt.Errorf("failed to schedule forget task: %w", err)
}
}
if err := runner.ScheduleTask(NewOneoffIndexSnapshotsTask(t.RepoID(), at), TaskPriorityIndexSnapshots); err != nil {
return fmt.Errorf("failed to schedule index snapshots task: %w", err)
}
orchestrator.ScheduleTask(NewOneoffIndexSnapshotsTask(orchestrator, plan.Repo, at), TaskPriorityIndexSnapshots)
return nil
}
internal/orchestrator/tasks/taskcollectgarbage.go
+123
View File
@@ -0,0 +1,123 @@
package tasks
import (
"context"
"fmt"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/oplog"
"go.uber.org/zap"
)
const (
gcStartupDelay = 5 * time.Second
gcInterval = 24 * time.Hour
// keep operations that are eligible for gc for 30 days OR up to a limit of 100 for any one plan.
// an operation is eligible for gc if:
// - it has no snapshot associated with it
// - it has a forgotten snapshot associated with it
gcHistoryAge = 30 * 24 * time.Hour
gcHistoryMaxCount = 1000
// keep stats operations for 1 year (they're small and useful for long term trends)
gcHistoryStatsAge = 365 * 24 * time.Hour
)
type CollectGarbageTask struct {
BaseTask
firstRun bool
}
func NewCollectGarbageTask() *CollectGarbageTask {
return &CollectGarbageTask{
BaseTask: BaseTask{
TaskName: "collect garbage",
},
}
}
var _ Task = &CollectGarbageTask{}
func (t *CollectGarbageTask) Next(now time.Time, runner TaskRunner) ScheduledTask {
if !t.firstRun {
t.firstRun = true
runAt := now.Add(gcStartupDelay)
return ScheduledTask{
Task: t,
RunAt: runAt,
}
}
runAt := now.Add(gcInterval)
return ScheduledTask{
Task: t,
RunAt: runAt,
}
}
func (t *CollectGarbageTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner) error {
oplog := runner.OpLog()
if err := t.gcOperations(oplog); err != nil {
return fmt.Errorf("collecting garbage: %w", err)
}
return nil
}
func (t *CollectGarbageTask) gcOperations(oplog *oplog.OpLog) error {
// snapshotForgottenForFlow returns whether the snapshot associated with the flow is forgotten
snapshotForgottenForFlow := make(map[int64]bool)
if err := oplog.ForAll(func(op *v1.Operation) error {
if snapshotOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok {
snapshotForgottenForFlow[op.FlowId] = snapshotOp.OperationIndexSnapshot.Forgot
}
return nil
}); err != nil {
return fmt.Errorf("identifying forgotten snapshots: %w", err)
}
forgetIDs := []int64{}
curTime := curTimeMillis()
if err := oplog.ForAll(func(op *v1.Operation) error {
forgot, ok := snapshotForgottenForFlow[op.FlowId]
if !ok {
// no snapshot associated with this flow; check if it's old enough to be gc'd
maxAgeForType := gcHistoryAge.Milliseconds()
if _, isStats := op.Op.(*v1.Operation_OperationStats); isStats {
maxAgeForType = gcHistoryStatsAge.Milliseconds()
}
if curTime-op.UnixTimeStartMs > maxAgeForType {
forgetIDs = append(forgetIDs, op.Id)
}
} else if forgot {
// snapshot is forgotten; this operation is eligible for gc
forgetIDs = append(forgetIDs, op.Id)
}
return nil
}); err != nil {
return fmt.Errorf("identifying gc eligible operations: %w", err)
}
if err := oplog.Delete(forgetIDs...); err != nil {
return fmt.Errorf("removing gc eligible operations: %w", err)
}
zap.L().Info("collecting garbage",
zap.Any("operations_removed", len(forgetIDs)))
return nil
}
func (t *CollectGarbageTask) Cancel(withStatus v1.OperationStatus) error {
return nil
}
func (t *CollectGarbageTask) OperationId() int64 {
return 0
}
type gcOpInfo struct {
id int64 // operation ID
timestamp int64 // unix time milliseconds
isStats bool // true if this is a stats operation
}
internal/orchestrator/tasks/taskforget.go
+110
View File
@@ -0,0 +1,110 @@
package tasks
import (
"context"
"fmt"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/hook"
"github.com/garethgeorge/backrest/internal/oplog/indexutil"
"github.com/hashicorp/go-multierror"
"go.uber.org/zap"
)
func NewOneoffForgetTask(repoID, planID string, flowID int64, at time.Time) Task {
return &GenericOneoffTask{
BaseTask: BaseTask{
TaskName: fmt.Sprintf("forget for plan %q in repo %q", repoID, planID),
TaskRepoID: repoID,
TaskPlanID: planID,
},
OneoffTask: OneoffTask{
FlowID: flowID,
RunAt: at,
ProtoOp: &v1.Operation{
Op: &v1.Operation_OperationForget{},
},
},
Do: func(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error {
op := st.Op
forgetOp := op.GetOperationForget()
if forgetOp == nil {
panic("forget task with non-forget operation")
}
if err := forgetHelper(ctx, st, taskRunner); err != nil {
taskRunner.ExecuteHooks([]v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Error: err.Error(),
})
return err
}
return nil
},
}
}
func forgetHelper(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error {
t := st.Task
repo, err := taskRunner.GetRepoOrchestrator(t.RepoID())
if err != nil {
return fmt.Errorf("get repo %q: %w", t.RepoID(), err)
}
err = repo.UnlockIfAutoEnabled(ctx)
if err != nil {
return fmt.Errorf("auto unlock repo %q: %w", t.RepoID(), err)
}
plan, err := taskRunner.GetPlan(t.PlanID())
if err != nil {
return fmt.Errorf("get plan %q: %w", t.PlanID(), err)
}
forgot, err := repo.Forget(ctx, plan)
if err != nil {
return fmt.Errorf("forget: %w", err)
}
forgetOp := &v1.Operation_OperationForget{
OperationForget: &v1.OperationForget{},
}
st.Op.Op = forgetOp
forgetOp.OperationForget.Forget = append(forgetOp.OperationForget.Forget, forgot...)
forgetOp.OperationForget.Policy = plan.Retention
var ops []*v1.Operation
for _, forgot := range forgot {
if e := taskRunner.OpLog().ForEachBySnapshotId(forgot.Id, indexutil.CollectAll(), func(op *v1.Operation) error {
ops = append(ops, op)
return nil
}); e != nil {
err = multierror.Append(err, fmt.Errorf("cleanup snapshot %v: %w", forgot.Id, e))
}
}
zap.S().Debugf("found %v snapshots were forgotten, marking this in oplog", len(ops))
for _, op := range ops {
if indexOp, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok {
indexOp.OperationIndexSnapshot.Forgot = true
if e := taskRunner.UpdateOperation(op); err != nil {
err = multierror.Append(err, fmt.Errorf("mark index snapshot %v as forgotten: %w", op.Id, e))
continue
}
}
}
if len(forgot) > 0 {
if err := taskRunner.ScheduleTask(NewOneoffPruneTask(t.RepoID(), t.PlanID(), st.Op.FlowId, time.Now(), false), TaskPriorityPrune); err != nil {
return fmt.Errorf("schedule prune task: %w", err)
}
}
return err
}
internal/orchestrator/tasks/taskforgetsnapshot.go
+67
View File
@@ -0,0 +1,67 @@
package tasks
import (
"context"
"fmt"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/hook"
)
func NewOneoffForgetSnapshotTask(repoID, planID string, flowID int64, at time.Time, snapshotID string) Task {
return &GenericOneoffTask{
BaseTask: BaseTask{
TaskName: fmt.Sprintf("forget snapshot %q for plan %q in repo %q", snapshotID, planID, repoID),
TaskRepoID: repoID,
TaskPlanID: planID,
},
OneoffTask: OneoffTask{
FlowID: flowID,
RunAt: at,
ProtoOp: &v1.Operation{
Op: &v1.Operation_OperationForget{},
},
},
Do: func(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error {
op := st.Op
forgetOp := op.GetOperationForget()
if forgetOp == nil {
panic("forget task with non-forget operation")
}
if err := forgetSnapshotHelper(ctx, st, taskRunner, snapshotID); err != nil {
taskRunner.ExecuteHooks([]v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Error: err.Error(),
})
return err
}
return nil
},
}
}
func forgetSnapshotHelper(ctx context.Context, st ScheduledTask, taskRunner TaskRunner, snapshotID string) error {
t := st.Task
repo, err := taskRunner.GetRepoOrchestrator(t.RepoID())
if err != nil {
return fmt.Errorf("get repo %q: %w", t.RepoID(), err)
}
err = repo.UnlockIfAutoEnabled(ctx)
if err != nil {
return fmt.Errorf("auto unlock repo %q: %w", t.RepoID(), err)
}
if err := repo.ForgetSnapshot(ctx, snapshotID); err != nil {
return fmt.Errorf("forget %q: %w", snapshotID, err)
}
taskRunner.ScheduleTask(NewOneoffIndexSnapshotsTask(t.RepoID(), time.Now()), TaskPriorityIndexSnapshots)
taskRunner.OpLog().Delete(st.Op.Id)
st.Op = nil
return err
}
internal/orchestrator/taskindexsnapshots.go → internal/orchestrator/tasks/taskindexsnapshots.go
+42 -60
View File
@@ -1,4 +1,4 @@
package orchestrator
package tasks
import (
"context"
@@ -14,77 +14,54 @@ import (
"go.uber.org/zap"
)
// IndexSnapshotsTask tracks a forget operation.
type IndexSnapshotsTask struct {
orchestrator *Orchestrator // owning orchestrator
repoId string
at *time.Time
}
var _ Task = &IndexSnapshotsTask{}
func NewOneoffIndexSnapshotsTask(orchestrator *Orchestrator, repoId string, at time.Time) *IndexSnapshotsTask {
return &IndexSnapshotsTask{
orchestrator: orchestrator,
repoId: repoId,
at: &at,
func NewOneoffIndexSnapshotsTask(repoID string, at time.Time) Task {
return &GenericOneoffTask{
BaseTask: BaseTask{
TaskName: fmt.Sprintf("index snapshots for repo %q", repoID),
TaskRepoID: repoID,
},
OneoffTask: OneoffTask{
RunAt: at,
ProtoOp: nil,
},
Do: func(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error {
if err := indexSnapshotsHelper(ctx, st, taskRunner); err != nil {
taskRunner.ExecuteHooks([]v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Task: st.Task.Name(),
Error: err.Error(),
})
return err
}
return nil
},
}
}
func (t *IndexSnapshotsTask) Name() string {
return fmt.Sprintf("index snapshots for plan %q", t.repoId)
}
func (t *IndexSnapshotsTask) Next(now time.Time) *time.Time {
ret := t.at
if ret != nil {
t.at = nil
}
return ret
}
func (t *IndexSnapshotsTask) Run(ctx context.Context) error {
if err := indexSnapshotsHelper(ctx, t.orchestrator, t.repoId); err != nil {
repo, _ := t.orchestrator.GetRepo(t.repoId)
t.orchestrator.hookExecutor.ExecuteHooks(repo.Config(), nil, []v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Task: t.Name(),
Error: err.Error(),
})
return err
}
return nil
}
func (t *IndexSnapshotsTask) Cancel(withStatus v1.OperationStatus) error {
return nil
}
func (t *IndexSnapshotsTask) OperationId() int64 {
return 0
}
// indexSnapshotsHelper indexes all snapshots for a plan.
// - If the snapshot is already indexed, it is skipped.
// - If the snapshot is not indexed, an index snapshot operation with it's metadata is added.
// - If an index snapshot operation is found for a snapshot that is not returned by the repo, it is marked as forgotten.
func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, repoId string) error {
repo, err := orchestrator.GetRepo(repoId)
func indexSnapshotsHelper(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error {
t := st.Task
oplog := taskRunner.OpLog()
repo, err := taskRunner.GetRepoOrchestrator(t.RepoID())
if err != nil {
return fmt.Errorf("couldn't get repo %q: %w", repoId, err)
return fmt.Errorf("couldn't get repo %q: %w", t.RepoID(), err)
}
// collect all tracked snapshots for the plan.
snapshots, err := repo.Snapshots(ctx)
if err != nil {
return fmt.Errorf("get snapshots for repo %q: %w", repoId, err)
return fmt.Errorf("get snapshots for repo %q: %w", t.RepoID(), err)
}
// collect all current snapshot IDs.
currentIds, err := indexCurrentSnapshotIdsForRepo(orchestrator.OpLog, repoId)
currentIds, err := indexCurrentSnapshotIdsForRepo(taskRunner.OpLog(), t.RepoID())
if err != nil {
return fmt.Errorf("get known snapshot IDs for repo %q: %w", repoId, err)
return fmt.Errorf("get known snapshot IDs for repo %q: %w", t.RepoID(), err)
}
foundIds := make(map[string]bool)
@@ -99,10 +76,15 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, repoI
}
snapshotProto := protoutil.SnapshotToProto(snapshot)
flowID, err := FlowIDForSnapshotID(taskRunner.OpLog(), snapshot.Id)
if err != nil {
return fmt.Errorf("get flow ID for snapshot %q: %w", snapshot.Id, err)
}
planId := planForSnapshot(snapshotProto)
indexOps = append(indexOps, &v1.Operation{
RepoId: repoId,
RepoId: t.RepoID(),
PlanId: planId,
FlowId: flowID,
UnixTimeStartMs: snapshotProto.UnixTimeMs,
UnixTimeEndMs: snapshotProto.UnixTimeMs,
Status: v1.OperationStatus_STATUS_SUCCESS,
@@ -115,7 +97,7 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, repoI
})
}
if err := orchestrator.OpLog.BulkAdd(indexOps); err != nil {
if err := taskRunner.OpLog().BulkAdd(indexOps); err != nil {
return fmt.Errorf("BulkAdd snapshot operations: %w", err)
}
@@ -127,7 +109,7 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, repoI
}
// mark snapshot forgotten.
op, err := orchestrator.OpLog.Get(opId)
op, err := oplog.Get(opId)
if err != nil {
// should only be possible in the case of a data race (e.g. operation was somehow deleted).
return fmt.Errorf("get operation %v: %w", opId, err)
@@ -139,14 +121,14 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, repoI
}
snapshotOp.OperationIndexSnapshot.Forgot = true
if err := orchestrator.OpLog.Update(op); err != nil {
if err := oplog.Update(op); err != nil {
return fmt.Errorf("mark index snapshot operation %v as forgotten: %w", opId, err)
}
}
// Print stats at the end of indexing.
zap.L().Debug("indexed snapshots",
zap.String("repo", repoId),
zap.String("repo", t.RepoID()),
zap.Duration("duration", time.Since(startTime)),
zap.Int("alreadyIndexed", len(foundIds)),
zap.Int("newlyAdded", len(indexOps)),
internal/orchestrator/tasks/taskprune.go
+180
View File
@@ -0,0 +1,180 @@
package tasks
import (
"bytes"
"context"
"fmt"
"sync"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/hook"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/oplog/indexutil"
"go.uber.org/zap"
)
type PruneTask struct {
BaseTask
OneoffTask
force bool
}
func NewOneoffPruneTask(repoID, planID string, flowID int64, at time.Time, force bool) Task {
return &PruneTask{
BaseTask: BaseTask{
TaskName: fmt.Sprintf("prune for plan %q in repo %q", planID, repoID),
TaskRepoID: repoID,
TaskPlanID: planID,
},
OneoffTask: OneoffTask{
FlowID: flowID,
RunAt: at,
ProtoOp: &v1.Operation{
Op: &v1.Operation_OperationPrune{},
},
},
force: force,
}
}
func (t *PruneTask) Next(now time.Time, runner TaskRunner) ScheduledTask {
if t.force {
return t.OneoffTask.Next(now, runner)
}
shouldRun, err := t.shouldRun(now, runner)
if err != nil {
zap.S().Errorf("task %v failed to check if it should run: %v", t.Name(), err)
return NeverScheduledTask
}
if !shouldRun {
return NeverScheduledTask
}
return t.OneoffTask.Next(now, runner)
}
func (t *PruneTask) shouldRun(now time.Time, runner TaskRunner) (bool, error) {
repo, err := runner.GetRepo(t.RepoID())
if err != nil {
return false, fmt.Errorf("get repo %v: %w", t.RepoID(), err)
}
nextPruneTime, err := t.getNextPruneTime(runner, repo.PrunePolicy)
if err != nil {
return false, fmt.Errorf("get next prune time: %w", err)
}
return nextPruneTime.Before(now), nil
}
func (t *PruneTask) getNextPruneTime(runner TaskRunner, policy *v1.PrunePolicy) (time.Time, error) {
var lastPruneTime time.Time
runner.OpLog().ForEachByRepo(t.RepoID(), indexutil.Reversed(indexutil.CollectAll()), func(op *v1.Operation) error {
if _, ok := op.Op.(*v1.Operation_OperationPrune); ok {
lastPruneTime = time.Unix(0, op.UnixTimeStartMs*int64(time.Millisecond))
return oplog.ErrStopIteration
}
return nil
})
if policy != nil {
return lastPruneTime.Add(time.Duration(policy.MaxFrequencyDays) * 24 * time.Hour), nil
} else {
return lastPruneTime.Add(7 * 24 * time.Hour), nil // default to 7 days.
}
}
func (t *PruneTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunner) error {
op := st.Op
repo, err := runner.GetRepoOrchestrator(t.RepoID())
if err != nil {
return fmt.Errorf("couldn't get repo %q: %w", t.RepoID(), err)
}
err = repo.UnlockIfAutoEnabled(ctx)
if err != nil {
return fmt.Errorf("auto unlock repo %q: %w", t.RepoID(), err)
}
opPrune := &v1.Operation_OperationPrune{
OperationPrune: &v1.OperationPrune{},
}
op.Op = opPrune
ctx, cancel := context.WithCancel(ctx)
interval := time.NewTicker(1 * time.Second)
defer interval.Stop()
var buf synchronizedBuffer
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-interval.C:
output := buf.String()
if len(output) > 8*1024 { // only provide live status upto the first 8K of output.
output = output[:len(output)-8*1024]
}
if opPrune.OperationPrune.Output != output {
opPrune.OperationPrune.Output = buf.String()
if err := runner.OpLog().Update(op); err != nil {
zap.L().Error("update prune operation with status output", zap.Error(err))
}
}
case <-ctx.Done():
return
}
}
}()
if err := repo.Prune(ctx, &buf); err != nil {
cancel()
runner.ExecuteHooks([]v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Error: err.Error(),
})
return fmt.Errorf("prune: %w", err)
}
cancel()
wg.Wait()
// TODO: it would be best to store the output in separate storage for large status data.
output := buf.String()
if len(output) > 8*1024 { // only save the first 4K of output.
output = output[:len(output)-8*1024]
}
opPrune.OperationPrune.Output = output
return nil
}
// synchronizedBuffer is used for collecting prune command's output
type synchronizedBuffer struct {
mu sync.Mutex
buf bytes.Buffer
}
func (w *synchronizedBuffer) Write(p []byte) (n int, err error) {
w.mu.Lock()
defer w.mu.Unlock()
return w.buf.Write(p)
}
func (w *synchronizedBuffer) String() string {
w.mu.Lock()
defer w.mu.Unlock()
return w.buf.String()
}
internal/orchestrator/tasks/taskrestore.go
+92
View File
@@ -0,0 +1,92 @@
package tasks
import (
"context"
"errors"
"fmt"
"sync"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/hook"
"go.uber.org/zap"
)
func NewOneoffRestoreTask(repoID, planID string, flowID int64, at time.Time, snapshotID, path, target string) Task {
return &GenericOneoffTask{
BaseTask: BaseTask{
TaskName: fmt.Sprintf("restore snapshot %q in repo %q", snapshotID, repoID),
TaskRepoID: repoID,
TaskPlanID: planID,
},
OneoffTask: OneoffTask{
FlowID: flowID,
RunAt: at,
ProtoOp: &v1.Operation{
SnapshotId: snapshotID,
Op: &v1.Operation_OperationRestore{},
},
},
Do: func(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error {
if err := restoreHelper(ctx, st, taskRunner, snapshotID, path, target); err != nil {
taskRunner.ExecuteHooks([]v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Task: st.Task.Name(),
Error: err.Error(),
})
return err
}
return nil
},
}
}
func restoreHelper(ctx context.Context, st ScheduledTask, taskRunner TaskRunner, snapshotID, path, target string) error {
t := st.Task
oplog := taskRunner.OpLog()
op := st.Op
if snapshotID == "" || path == "" || target == "" {
return errors.New("snapshotID, path, and target are required")
}
restoreOp := st.Op.GetOperationRestore()
if restoreOp == nil {
return errors.New("operation is not a restore operation")
}
repo, err := taskRunner.GetRepoOrchestrator(t.RepoID())
if err != nil {
return fmt.Errorf("couldn't get repo %q: %w", t.RepoID(), err)
}
var sendWg sync.WaitGroup
lastSent := time.Now() // debounce progress updates, these can endup being very frequent.
summary, err := repo.Restore(ctx, snapshotID, path, target, func(entry *v1.RestoreProgressEntry) {
sendWg.Wait()
if time.Since(lastSent) < 1*time.Second {
return
}
lastSent = time.Now()
zap.S().Infof("restore progress: %v", entry)
restoreOp.Status = entry
sendWg.Add(1)
go func() {
if err := oplog.Update(op); err != nil {
zap.S().Errorf("failed to update oplog with progress for restore: %v", err)
}
sendWg.Done()
}()
})
if err != nil {
return fmt.Errorf("restore failed: %w", err)
}
restoreOp.Status = summary
return nil
}
internal/orchestrator/tasks/taskstats.go
+61
View File
@@ -0,0 +1,61 @@
package tasks
import (
"context"
"fmt"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/hook"
)
func NewOneoffStatsTask(repoID, planID string, at time.Time) Task {
return &GenericOneoffTask{
BaseTask: BaseTask{
TaskName: fmt.Sprintf("stats for repo %q", repoID),
TaskRepoID: repoID,
TaskPlanID: planID,
},
OneoffTask: OneoffTask{
RunAt: at,
ProtoOp: &v1.Operation{
Op: &v1.Operation_OperationStats{},
},
},
Do: func(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error {
if err := statsHelper(ctx, st, taskRunner); err != nil {
taskRunner.ExecuteHooks([]v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Task: st.Task.Name(),
Error: err.Error(),
})
return err
}
return nil
},
}
}
func statsHelper(ctx context.Context, st ScheduledTask, taskRunner TaskRunner) error {
t := st.Task
repo, err := taskRunner.GetRepoOrchestrator(t.RepoID())
if err != nil {
return fmt.Errorf("get repo %q: %w", t.RepoID(), err)
}
stats, err := repo.Stats(ctx)
if err != nil {
return fmt.Errorf("get stats: %w", err)
}
op := st.Op
op.Op = &v1.Operation_OperationStats{
OperationStats: &v1.OperationStats{
Stats: stats,
},
}
return nil
}
internal/orchestrator/taskstats.go
-88
View File
@@ -1,88 +0,0 @@
package orchestrator
import (
"context"
"fmt"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/hook"
"go.uber.org/zap"
)
// StatsTask tracks a restic stats operation.
type StatsTask struct {
TaskWithOperation
planId string
repoId string
at *time.Time
}
var _ Task = &StatsTask{}
func NewOneoffStatsTask(orchestrator *Orchestrator, repoId, planId string, at time.Time) *StatsTask {
return &StatsTask{
TaskWithOperation: TaskWithOperation{
orch: orchestrator,
},
planId: planId,
repoId: repoId,
at: &at,
}
}
func (t *StatsTask) Name() string {
return fmt.Sprintf("stats for repo %q", t.repoId)
}
func (t *StatsTask) Next(now time.Time) *time.Time {
ret := t.at
if ret != nil {
t.at = nil
if err := t.setOperation(&v1.Operation{
PlanId: t.planId,
RepoId: t.repoId,
UnixTimeStartMs: timeToUnixMillis(*ret),
Status: v1.OperationStatus_STATUS_PENDING,
Op: &v1.Operation_OperationStats{},
}); err != nil {
zap.S().Errorf("task %v failed to add operation to oplog: %v", t.Name(), err)
return nil
}
}
return ret
}
func (t *StatsTask) Run(ctx context.Context) error {
if err := t.runWithOpAndContext(ctx, func(ctx context.Context, op *v1.Operation) error {
repo, err := t.orch.GetRepo(t.repoId)
if err != nil {
return fmt.Errorf("get repo %q: %w", t.repoId, err)
}
stats, err := repo.Stats(ctx)
if err != nil {
return fmt.Errorf("get stats: %w", err)
}
op.Op = &v1.Operation_OperationStats{
OperationStats: &v1.OperationStats{
Stats: stats,
},
}
return err
}); err != nil {
repo, _ := t.orch.GetRepo(t.repoId)
plan, _ := t.orch.GetPlan(t.planId)
_ = t.orch.hookExecutor.ExecuteHooks(repo.Config(), plan, []v1.Hook_Condition{
v1.Hook_CONDITION_ANY_ERROR,
}, hook.HookVars{
Task: t.Name(),
Error: err.Error(),
})
return err
}
return nil
}
internal/protoutil/validation.go
+3
View File
@@ -13,6 +13,9 @@ func ValidateOperation(op *v1.Operation) error {
if op.Id == 0 {
return errors.New("operation.id is required")
}
if op.FlowId == 0 {
return errors.New("operation.flow_id is required")
}
if op.RepoId == "" {
return errors.New("operation.repo_id is required")
}
proto/v1/operations.proto
+4 -1
View File
@@ -14,6 +14,8 @@ message OperationList {
message Operation {
// required, primary ID of the operation. ID is sequential based on creation time of the operation.
int64 id = 1;
// flow id groups operations together, e.g. by an execution of a plan.
int64 flow_id = 10; // optional, flow id if associated with a flow
// required, repo id if associated with a repo
string repo_id = 2;
// required, plan id if associated with a plan
@@ -101,5 +103,6 @@ message OperationStats {
message OperationRunHook {
string name = 1; // description of the hook that was run. typically repo/hook_idx or plan/hook_idx.
string output_logref = 2; // logref of the hook's output.
string output_logref = 2; // logref of the hook's output. DEPRECATED.
Hook.Condition condition = 3; // triggering condition of the hook.
}
proto/v1/service.proto
+1
View File
@@ -82,6 +82,7 @@ message GetOperationsRequest {
string repo_id = 1;
string plan_id = 2;
string snapshot_id = 4;
int64 flow_id = 6;
repeated int64 ids = 5;
int64 last_n = 3; // limit to the last n operations
}
webui/gen/ts/v1/operations_pb.ts
+20 -2
View File
@@ -6,7 +6,7 @@
import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf";
import { Message, proto3, protoInt64 } from "@bufbuild/protobuf";
import { BackupProgressEntry, BackupProgressError, RepoStats, ResticSnapshot, RestoreProgressEntry } from "./restic_pb.js";
import { RetentionPolicy } from "./config_pb.js";
import { Hook_Condition, RetentionPolicy } from "./config_pb.js";
/**
* OperationEventType indicates whether the operation was created or updated
@@ -162,6 +162,15 @@ export class Operation extends Message<Operation> {
*/
id = protoInt64.zero;
/**
* flow id groups operations together, e.g. by an execution of a plan.
*
* optional, flow id if associated with a flow
*
* @generated from field: int64 flow_id = 10;
*/
flowId = protoInt64.zero;
/**
* required, repo id if associated with a repo
*
@@ -272,6 +281,7 @@ export class Operation extends Message<Operation> {
static readonly typeName = "v1.Operation";
static readonly fields: FieldList = proto3.util.newFieldList(() => [
{ no: 1, name: "id", kind: "scalar", T: 3 /* ScalarType.INT64 */ },
{ no: 10, name: "flow_id", kind: "scalar", T: 3 /* ScalarType.INT64 */ },
{ no: 2, name: "repo_id", kind: "scalar", T: 9 /* ScalarType.STRING */ },
{ no: 3, name: "plan_id", kind: "scalar", T: 9 /* ScalarType.STRING */ },
{ no: 8, name: "snapshot_id", kind: "scalar", T: 9 /* ScalarType.STRING */ },
@@ -641,12 +651,19 @@ export class OperationRunHook extends Message<OperationRunHook> {
name = "";
/**
* logref of the hook's output.
* logref of the hook's output. DEPRECATED.
*
* @generated from field: string output_logref = 2;
*/
outputLogref = "";
/**
* triggering condition of the hook.
*
* @generated from field: v1.Hook.Condition condition = 3;
*/
condition = Hook_Condition.UNKNOWN;
constructor(data?: PartialMessage<OperationRunHook>) {
super();
proto3.util.initPartial(data, this);
@@ -657,6 +674,7 @@ export class OperationRunHook extends Message<OperationRunHook> {
static readonly fields: FieldList = proto3.util.newFieldList(() => [
{ no: 1, name: "name", kind: "scalar", T: 9 /* ScalarType.STRING */ },
{ no: 2, name: "output_logref", kind: "scalar", T: 9 /* ScalarType.STRING */ },
{ no: 3, name: "condition", kind: "enum", T: proto3.getEnumType(Hook_Condition) },
]);
static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): OperationRunHook {
webui/gen/ts/v1/service_pb.ts
+6
View File
@@ -172,6 +172,11 @@ export class GetOperationsRequest extends Message<GetOperationsRequest> {
*/
snapshotId = "";
/**
* @generated from field: int64 flow_id = 6;
*/
flowId = protoInt64.zero;
/**
* @generated from field: repeated int64 ids = 5;
*/
@@ -195,6 +200,7 @@ export class GetOperationsRequest extends Message<GetOperationsRequest> {
{ no: 1, name: "repo_id", kind: "scalar", T: 9 /* ScalarType.STRING */ },
{ no: 2, name: "plan_id", kind: "scalar", T: 9 /* ScalarType.STRING */ },
{ no: 4, name: "snapshot_id", kind: "scalar", T: 9 /* ScalarType.STRING */ },
{ no: 6, name: "flow_id", kind: "scalar", T: 3 /* ScalarType.INT64 */ },
{ no: 5, name: "ids", kind: "scalar", T: 3 /* ScalarType.INT64 */, repeated: true },
{ no: 3, name: "last_n", kind: "scalar", T: 3 /* ScalarType.INT64 */ },
]);
webui/src/components/OperationRow.tsx
+9 -2
View File
@@ -44,6 +44,8 @@ import { LogDataRequest } from "../../gen/ts/v1/service_pb";
import { MessageInstance } from "antd/es/message/interface";
import { backrestService } from "../api";
import { useShowModal } from "./ModalManager";
import { proto3 } from "@bufbuild/protobuf";
import { Hook_Condition } from "../../gen/ts/v1/config_pb";
export const OperationRow = ({
@@ -127,6 +129,7 @@ export const OperationRow = ({
}
let body: React.ReactNode | undefined;
let displayMessage = operation.displayMessage;
if (operation.op.case === "operationBackup") {
const backupOp = operation.op.value;
@@ -194,14 +197,18 @@ export const OperationRow = ({
</>
);
} else if (operation.op.case === "operationRunHook") {
body = <></>
const hook = operation.op.value;
const triggeringCondition = proto3.getEnumType(Hook_Condition).findNumber(hook.condition);
if (triggeringCondition !== undefined) {
displayMessage += "\ntriggered by condition: " + triggeringCondition.name;
}
}
const children = [];
if (operation.displayMessage) {
children.push(<>
<pre>{details.state ? details.state + ": " : null}{operation.displayMessage}</pre>
<pre>{details.state ? details.state + ": " : null}{displayMessage}</pre>
</>);
}
webui/src/components/OperationTree.tsx
+5
View File
@@ -133,6 +133,11 @@ export const OperationTree = ({
const b = node.backup;
const details: string[] = [];
if (b.operations.length === 0) {
// this happens when all operations in a backup are deleted; it should be hidden until the deletion propagates to a refresh of the tree layout.
return null;
}
if (b.status === OperationStatus.STATUS_PENDING) {
details.push("scheduled, waiting");
} else if (b.status === OperationStatus.STATUS_SYSTEM_CANCELLED) {
webui/src/components/SnapshotBrowser.tsx
+1 -1
View File
@@ -222,7 +222,7 @@ const FileNode = ({ entry }: { entry: LsEntry }) => {
({formatBytes(Number(entry.size))})
</span>
) : null}
{dropdown}
{dropdown || <div style={{ width: 16 }}></div>}
</Space>
);
};
webui/src/state/oplog.ts
+64 -80
View File
@@ -9,7 +9,7 @@ import { BackupProgressEntry, ResticSnapshot } from "../../gen/ts/v1/restic_pb";
import _ from "lodash";
import { formatDuration, formatTime } from "../lib/formatting";
import { backrestService } from "../api";
import { STATS_OPERATION_HISTORY } from "../constants";
import { STATS_OPERATION_HISTORY, STATUS_OPERATION_HISTORY } from "../constants";
const subscribers: ((event: OperationEvent) => void)[] = [];
@@ -57,7 +57,7 @@ export const unsubscribeFromOperations = (
export const getStatusForPlan = async (plan: string) => {
const req = new GetOperationsRequest({
planId: plan,
lastN: BigInt(STATS_OPERATION_HISTORY),
lastN: BigInt(STATUS_OPERATION_HISTORY),
});
return await getStatus(req);
};
@@ -65,7 +65,7 @@ export const getStatusForPlan = async (plan: string) => {
export const getStatusForRepo = async (repo: string) => {
const req = new GetOperationsRequest({
repoId: repo,
lastN: BigInt(STATS_OPERATION_HISTORY),
lastN: BigInt(STATUS_OPERATION_HISTORY),
});
return await getStatus(req);
};
@@ -79,12 +79,12 @@ const getStatus = async (req: GetOperationsRequest) => {
if (ops.length === 0) {
return OperationStatus.STATUS_SUCCESS;
}
const snapshotId = ops[0].snapshotId;
const flowId = ops[0].flowId;
for (const op of ops) {
if (op.status === OperationStatus.STATUS_PENDING) {
continue;
}
if (op.snapshotId !== snapshotId) {
if (op.flowId !== flowId) {
break;
}
if (
@@ -110,7 +110,7 @@ export enum DisplayType {
}
export interface BackupInfo {
id: string; // id of the first operation that generated this backup.
id: string; // flow ID of the operations that make up this backup.
displayTime: Date;
displayType: DisplayType;
startTimeMs: number;
@@ -132,8 +132,10 @@ export class BackupInfoCollector {
event: OperationEventType,
info: BackupInfo[]
) => void)[] = [];
private backupByOpId: Map<bigint, BackupInfo> = new Map();
private backupBySnapshotId: Map<string, BackupInfo> = new Map();
// backups maps a flow ID to a backup info object.
private operationsByFlowId: Map<bigint, Operation[]> = new Map();
private backupsByFlowId: Map<bigint, BackupInfo> = new Map();
/**
*
@@ -164,6 +166,10 @@ export class BackupInfoCollector {
let statusIdx = operations.length - 1;
let status = OperationStatus.STATUS_SYSTEM_CANCELLED;
while (statusIdx !== -1) {
if (operations[statusIdx].op.case === "operationRunHook") {
statusIdx--;
continue;
}
const curStatus = operations[statusIdx].status;
if (
shouldHideStatus(status) ||
@@ -179,17 +185,19 @@ export class BackupInfoCollector {
let backupLastStatus = undefined;
let snapshotInfo = undefined;
let forgotten = false;
let snapshotId = "";
for (const op of operations) {
if (op.op.case === "operationBackup") {
backupLastStatus = op.op.value.lastStatus;
} else if (op.op.case === "operationIndexSnapshot") {
snapshotInfo = op.op.value.snapshot;
forgotten = op.op.value.forgot || false;
snapshotId = op.op.value.snapshot?.id || "";
}
}
return {
id: id.toString(16),
id: operations[0].flowId.toString(16),
startTimeMs,
endTimeMs,
displayTime,
@@ -199,38 +207,38 @@ export class BackupInfoCollector {
backupLastStatus,
snapshotInfo,
forgotten,
snapshotId: operations[0].snapshotId,
snapshotId: snapshotId,
planId: operations[0].planId,
repoId: operations[0].repoId,
};
}
private addHelper(op: Operation): BackupInfo {
if (op.snapshotId) {
this.backupByOpId.delete(op.id);
const existing = this.backupBySnapshotId.get(op.snapshotId);
let operations: Operation[];
if (existing) {
operations = [...existing.operations];
const opIdx = operations.findIndex((o) => o.id === op.id);
if (opIdx > -1) {
operations[opIdx] = op;
} else {
operations.push(op);
}
} else {
operations = [op];
}
const newInfo = this.createBackup(operations);
this.backupBySnapshotId.set(op.snapshotId, newInfo);
return newInfo;
private addOrUpdateHelper(op: Operation) {
const existing = this.operationsByFlowId.get(op.flowId);
if (existing === undefined) {
this.operationsByFlowId.set(op.flowId, [op]);
} else {
const newInfo = this.createBackup([op]);
this.backupByOpId.set(op.id, newInfo);
return newInfo;
const idx = existing.findIndex((o) => o.id === op.id);
if (idx === -1) {
existing.push(op);
} else {
existing[idx] = op;
}
}
this.backupsByFlowId.delete(op.flowId);
}
private getBackupInfo(flowId: bigint): BackupInfo | undefined {
let existing = this.backupsByFlowId.get(flowId);
if (existing === undefined) {
const operations = this.operationsByFlowId.get(flowId);
if (!operations) {
return undefined;
}
existing = this.createBackup(operations);
this.backupsByFlowId.set(flowId, existing);
}
return existing;
}
public addOperation(event: OperationEventType, op: Operation): BackupInfo | null {
@@ -238,26 +246,27 @@ export class BackupInfoCollector {
this.removeOperation(op);
return null;
}
const backupInfo = this.addHelper(op);
this.addOrUpdateHelper(op);
const backupInfo = this.getBackupInfo(op.flowId)!;
this.listeners.forEach((l) => l(event, [backupInfo]));
return backupInfo;
}
// removeOperaiton is not quite correct from a formal standpoint; but will look correct in the UI.
public removeOperation(op: Operation) {
if (op.snapshotId) {
let existing = this.backupBySnapshotId.get(op.snapshotId);
if (existing) {
const operations = existing.operations.filter((o) => o.id !== op.id);
if (operations.length === 0) {
this.backupBySnapshotId.delete(op.snapshotId);
} else {
this.backupBySnapshotId.set(op.snapshotId, this.createBackup(operations));
}
}
} else {
this.backupByOpId.delete(op.id);
const existing = this.operationsByFlowId.get(op.flowId);
if (existing === undefined) {
return;
}
const idx = existing.findIndex((o) => o.id === op.id);
if (idx === -1) {
return;
}
existing.splice(idx, 1);
if (existing.length === 0) {
this.operationsByFlowId.delete(op.flowId);
}
this.backupsByFlowId.delete(op.flowId); // delete the cache for lazy recomputation.
this.listeners.forEach((l) =>
l(OperationEventType.EVENT_DELETED, this.getAll())
@@ -265,43 +274,20 @@ export class BackupInfoCollector {
}
public bulkAddOperations(ops: Operation[]): BackupInfo[] {
ops = ops.filter(this.filter);
let grouped = _.groupBy(ops, (op) =>
op.snapshotId ? op.snapshotId : op.id
);
const info: BackupInfo[] = [];
for (const operations of Object.values(grouped)) {
if (operations[0].snapshotId) {
const existing = this.backupBySnapshotId.get(operations[0].snapshotId);
if (existing) {
const newOps = [...existing.operations, ...operations];
const newInfo = this.createBackup(newOps);
this.backupBySnapshotId.set(operations[0].snapshotId, newInfo);
info.push(newInfo);
} else {
const newInfo = this.createBackup(operations);
this.backupBySnapshotId.set(operations[0].snapshotId, newInfo);
info.push(newInfo);
}
} else {
const op = operations[0];
const newInfo = this.createBackup([op]);
this.backupByOpId.set(op.id, newInfo);
info.push(newInfo);
}
for (const op of ops) {
this.addOrUpdateHelper(op);
}
const flowIDs = _.uniq(ops.map((op) => op.flowId));
const info = flowIDs.map((flowId) => this.getBackupInfo(flowId)!);
this.listeners.forEach((l) => l(OperationEventType.EVENT_CREATED, info));
return info;
}
public getAll(): BackupInfo[] {
const arr = [
...this.backupByOpId.values(),
...this.backupBySnapshotId.values(),
];
const arr = []
for (const key of this.operationsByFlowId.keys()) {
arr.push(this.getBackupInfo(key)!);
}
return arr.filter((b) => !b.forgotten && !shouldHideStatus(b.status));
}
@@ -325,8 +311,6 @@ export class BackupInfoCollector {
export const shouldHideOperation = (operation: Operation) => {
return (
operation.op.case === "operationStats" ||
(operation.op.case === "operationRunHook" &&
operation.status === OperationStatus.STATUS_SUCCESS) ||
shouldHideStatus(operation.status)
);
};