lee/backrest
1
0
Fork 0
mirror of https://github.com/garethgeorge/backrest.git synced 2025-12-15 10:05:42 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

chore: refactor storage layer

Browse Source
This commit is contained in:
garethgeorge
2023-11-15 18:14:46 -08:00
parent 4c6f042250
commit b22e006ffb
12 changed files with 271 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/resticui/resticui.go
View File

@@ -13,7 +13,7 @@ import (
"github.com/garethgeorge/resticui/internal/api"
"github.com/garethgeorge/resticui/internal/config"
"github.com/garethgeorge/resticui/internal/oplog"
"github.com/garethgeorge/resticui/internal/database/oplog"
"github.com/garethgeorge/resticui/internal/orchestrator"
static "github.com/garethgeorge/resticui/webui"
"github.com/mattn/go-colorable"

171
gen/go/v1/operations.pb.go
View File

@@ -187,6 +187,7 @@ type Operation struct {
// Types that are assignable to Op:
//
// *Operation_OperationBackup
// *Operation_OperationIndexSnapshot
Op isOperation_Op `protobuf_oneof:"op"`
}
@@ -285,6 +286,13 @@ func (x *Operation) GetOperationBackup() *OperationBackup {
return nil
}
func (x *Operation) GetOperationIndexSnapshot() *OperationIndexSnapshot {
if x, ok := x.GetOp().(*Operation_OperationIndexSnapshot); ok {
return x.OperationIndexSnapshot
}
return nil
}
type isOperation_Op interface {
isOperation_Op()
}
@@ -293,8 +301,14 @@ type Operation_OperationBackup struct {
OperationBackup *OperationBackup `protobuf:"bytes,100,opt,name=operation_backup,json=operationBackup,proto3,oneof"`
}
type Operation_OperationIndexSnapshot struct {
OperationIndexSnapshot *OperationIndexSnapshot `protobuf:"bytes,101,opt,name=operation_index_snapshot,json=operationIndexSnapshot,proto3,oneof"`
}
func (*Operation_OperationBackup) isOperation_Op() {}
func (*Operation_OperationIndexSnapshot) isOperation_Op() {}
// OperationEvent is used in the wireformat to stream operation changes to clients
type OperationEvent struct {
state protoimpl.MessageState
@@ -398,6 +412,54 @@ func (x *OperationBackup) GetLastStatus() *BackupProgressEntry {
return nil
}
// OperationIndexSnapshot tracks that a snapshot was detected by resticui.
type OperationIndexSnapshot struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Snapshot *ResticSnapshot `protobuf:"bytes,2,opt,name=snapshot,proto3" json:"snapshot,omitempty"`
}
func (x *OperationIndexSnapshot) Reset() {
*x = OperationIndexSnapshot{}
if protoimpl.UnsafeEnabled {
mi := &file_v1_operations_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *OperationIndexSnapshot) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*OperationIndexSnapshot) ProtoMessage() {}
func (x *OperationIndexSnapshot) ProtoReflect() protoreflect.Message {
mi := &file_v1_operations_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use OperationIndexSnapshot.ProtoReflect.Descriptor instead.
func (*OperationIndexSnapshot) Descriptor() ([]byte, []int) {
return file_v1_operations_proto_rawDescGZIP(), []int{4}
}
func (x *OperationIndexSnapshot) GetSnapshot() *ResticSnapshot {
if x != nil {
return x.Snapshot
}
return nil
}
var File_v1_operations_proto protoreflect.FileDescriptor
var file_v1_operations_proto_rawDesc = []byte{
@@ -407,7 +469,7 @@ var file_v1_operations_proto_rawDesc = []byte{
0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x0a, 0x6f,
0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32,
0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a,
0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xc1, 0x02, 0x0a, 0x09, 0x4f,
0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x99, 0x03, 0x0a, 0x09, 0x4f,
0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01,
0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f,
0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49,
@@ -427,35 +489,45 @@ var file_v1_operations_proto_rawDesc = []byte{
0x69, 0x6f, 0x6e, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x64, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x13, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42,
0x61, 0x63, 0x6b, 0x75, 0x70, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x42, 0x04, 0x0a, 0x02, 0x6f, 0x70, 0x22, 0x69,
0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74,
0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16,
0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65,
0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x2b, 0x0a, 0x09,
0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09,
0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x4b, 0x0a, 0x0f, 0x4f, 0x70, 0x65,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b,
0x6c, 0x61, 0x73, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x72, 0x6f,
0x67, 0x72, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x6c, 0x61, 0x73, 0x74,
0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2a, 0x4d, 0x0a, 0x12, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x11, 0x0a, 0x0d,
0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12,
0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44,
0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41,
0x54, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x76, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54,
0x55, 0x53, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e,
0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01,
0x12, 0x15, 0x0a, 0x11, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e, 0x50, 0x52, 0x4f,
0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55,
0x53, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, 0x53,
0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x42, 0x2e, 0x5a,
0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65,
0x74, 0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x72, 0x65, 0x73, 0x74, 0x69, 0x63, 0x75,
0x69, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x33,
0x6f, 0x6e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x56, 0x0a, 0x18, 0x6f, 0x70, 0x65, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x73, 0x6e, 0x61, 0x70,
0x73, 0x68, 0x6f, 0x74, 0x18, 0x65, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x76, 0x31, 0x2e,
0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x6e,
0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x48, 0x00, 0x52, 0x16, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
0x42, 0x04, 0x0a, 0x02, 0x6f, 0x70, 0x22, 0x69, 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65,
0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04,
0x74, 0x79, 0x70, 0x65, 0x12, 0x2b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x70, 0x65,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x22, 0x4b, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x61,
0x63, 0x6b, 0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x73, 0x74, 0x61,
0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x76, 0x31, 0x2e, 0x42,
0x61, 0x63, 0x6b, 0x75, 0x70, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x45, 0x6e, 0x74,
0x72, 0x79, 0x52, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x48,
0x0a, 0x16, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78,
0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x2e, 0x0a, 0x08, 0x73, 0x6e, 0x61, 0x70,
0x73, 0x68, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e,
0x52, 0x65, 0x73, 0x74, 0x69, 0x63, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x08,
0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x2a, 0x4d, 0x0a, 0x12, 0x4f, 0x70, 0x65, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x11,
0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10,
0x00, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54,
0x45, 0x44, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x50,
0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x76, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54,
0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x12,
0x0a, 0x0e, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47,
0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e, 0x50,
0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x41,
0x54, 0x55, 0x53, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x03, 0x12, 0x10, 0x0a,
0x0c, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x42,
0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61,
0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x72, 0x65, 0x73, 0x74, 0x69,
0x63, 0x75, 0x69, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62,
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -471,7 +543,7 @@ func file_v1_operations_proto_rawDescGZIP() []byte {
}
var file_v1_operations_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
var file_v1_operations_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
var file_v1_operations_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
var file_v1_operations_proto_goTypes = []interface{}{
(OperationEventType)(0), // 0: v1.OperationEventType
(OperationStatus)(0), // 1: v1.OperationStatus
@@ -479,20 +551,24 @@ var file_v1_operations_proto_goTypes = []interface{}{
(*Operation)(nil), // 3: v1.Operation
(*OperationEvent)(nil), // 4: v1.OperationEvent
(*OperationBackup)(nil), // 5: v1.OperationBackup
(*BackupProgressEntry)(nil), // 6: v1.BackupProgressEntry
(*OperationIndexSnapshot)(nil), // 6: v1.OperationIndexSnapshot
(*BackupProgressEntry)(nil), // 7: v1.BackupProgressEntry
(*ResticSnapshot)(nil), // 8: v1.ResticSnapshot
}
var file_v1_operations_proto_depIdxs = []int32{
3, // 0: v1.OperationList.operations:type_name -> v1.Operation
1, // 1: v1.Operation.status:type_name -> v1.OperationStatus
5, // 2: v1.Operation.operation_backup:type_name -> v1.OperationBackup
0, // 3: v1.OperationEvent.type:type_name -> v1.OperationEventType
3, // 4: v1.OperationEvent.operation:type_name -> v1.Operation
6, // 5: v1.OperationBackup.last_status:type_name -> v1.BackupProgressEntry
6, // [6:6] is the sub-list for method output_type
6, // [6:6] is the sub-list for method input_type
6, // [6:6] is the sub-list for extension type_name
6, // [6:6] is the sub-list for extension extendee
0, // [0:6] is the sub-list for field type_name
6, // 3: v1.Operation.operation_index_snapshot:type_name -> v1.OperationIndexSnapshot
0, // 4: v1.OperationEvent.type:type_name -> v1.OperationEventType
3, // 5: v1.OperationEvent.operation:type_name -> v1.Operation
7, // 6: v1.OperationBackup.last_status:type_name -> v1.BackupProgressEntry
8, // 7: v1.OperationIndexSnapshot.snapshot:type_name -> v1.ResticSnapshot
8, // [8:8] is the sub-list for method output_type
8, // [8:8] is the sub-list for method input_type
8, // [8:8] is the sub-list for extension type_name
8, // [8:8] is the sub-list for extension extendee
0, // [0:8] is the sub-list for field type_name
}
func init() { file_v1_operations_proto_init() }
@@ -550,9 +626,22 @@ func file_v1_operations_proto_init() {
return nil
}
}
file_v1_operations_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*OperationIndexSnapshot); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
file_v1_operations_proto_msgTypes[1].OneofWrappers = []interface{}{
(*Operation_OperationBackup)(nil),
(*Operation_OperationIndexSnapshot)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
@@ -560,7 +649,7 @@ func file_v1_operations_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_v1_operations_proto_rawDesc,
NumEnums: 2,
NumMessages: 4,
NumMessages: 5,
NumExtensions: 0,
NumServices: 0,
},

2
internal/api/server.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/garethgeorge/resticui/gen/go/types"
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"github.com/garethgeorge/resticui/internal/config"
"github.com/garethgeorge/resticui/internal/oplog"
"github.com/garethgeorge/resticui/internal/database/oplog"
"github.com/garethgeorge/resticui/internal/orchestrator"
"github.com/garethgeorge/resticui/pkg/restic"
"go.uber.org/zap"

55
internal/database/indexutil/indexutil.go Normal file
View File

@@ -0,0 +1,55 @@
package indexutil
import (
"bytes"
"github.com/garethgeorge/resticui/internal/database/serializationutil"
bolt "go.etcd.io/bbolt"
)
func IndexByteValue(b *bolt.Bucket, value []byte, recordId int64) error {
key := serializationutil.BytesToKey(value)
key = append(key, serializationutil.Itob(recordId)...)
return b.Put(key, []byte{})
}
func IndexSearchByteValue(b *bolt.Bucket, value []byte) *IndexSearchIterator {
return newSearchIterator(b, serializationutil.BytesToKey(value))
}
func IndexSearchIntValue(b *bolt.Bucket, value int64) *IndexSearchIterator {
return newSearchIterator(b, serializationutil.Itob(value))
}
type IndexSearchIterator struct {
c *bolt.Cursor
k []byte
prefix []byte
}
func newSearchIterator(b *bolt.Bucket, prefix []byte) *IndexSearchIterator {
c := b.Cursor()
k, _ := c.Seek(prefix)
return &IndexSearchIterator{
c: c,
k: k,
prefix: prefix,
}
}
func (i *IndexSearchIterator) Next() (int64, bool) {
if i.k == nil || !bytes.HasPrefix(i.k, i.prefix) {
return 0, false
}
id := serializationutil.Btoi(i.k[len(i.prefix):])
i.k, _ = i.c.Next()
return id, true
}
func (i *IndexSearchIterator) ToSlice() []int64 {
var ids []int64
for id, ok := i.Next(); ok; id, ok = i.Next() {
ids = append(ids, id)
}
return ids
}

64
internal/oplog/oplog.go → internal/database/oplog/oplog.go
View File

@@ -10,6 +10,8 @@ import (
"time"
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"github.com/garethgeorge/resticui/internal/database/indexutil"
"github.com/garethgeorge/resticui/internal/database/serializationutil"
bolt "go.etcd.io/bbolt"
"google.golang.org/protobuf/proto"
)
@@ -28,10 +30,8 @@ var (
OpLogBucket = []byte("oplog.log") // oplog stores the operations themselves
RepoIndexBucket = []byte("oplog.repo_idx") // repo_index tracks IDs of operations affecting a given repo
PlanIndexBucket = []byte("oplog.plan_idx") // plan_index tracks IDs of operations affecting a given plan
SnapshotIdBucket = []byte("oplog.snapshot_id") // index by snapshot ID.
)
// OpLog represents a log of operations performed.
// Operations are indexed by repo and plan.
type OpLog struct {
@@ -90,30 +90,26 @@ func (o *OpLog) Add(op *v1.Operation) error {
return fmt.Errorf("error marshalling operation: %w", err)
}
if err := b.Put(itob(op.Id), bytes); err != nil {
if err := b.Put(serializationutil.Itob(op.Id), bytes); err != nil {
return fmt.Errorf("error putting operation into bucket: %w", err)
}
if op.RepoId != "" {
if err := o.addOpToIndexBucket(tx, RepoIndexBucket, op.RepoId, op.Id); err != nil {
if err := indexutil.IndexByteValue(tx.Bucket(RepoIndexBucket), []byte(op.RepoId), op.Id); err != nil {
return fmt.Errorf("error adding operation to repo index: %w", err)
}
}
if op.PlanId != "" {
if err := o.addOpToIndexBucket(tx, PlanIndexBucket, op.PlanId, op.Id); err != nil {
return fmt.Errorf("error adding operation to plan index: %w", err)
if err := indexutil.IndexByteValue(tx.Bucket(PlanIndexBucket), []byte(op.PlanId), op.Id); err != nil {
return fmt.Errorf("error adding operation to repo index: %w", err)
}
}
return nil
})
if err != nil {
o.subscribersMu.RLock()
defer o.subscribersMu.RUnlock()
for _, sub := range o.subscribers {
(*sub)(EventTypeOpCreated, op)
}
if err == nil {
o.notifyHelper(EventTypeOpCreated, op)
}
return err
}
@@ -126,7 +122,7 @@ func (o *OpLog) Update(op *v1.Operation) error {
err := o.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket(OpLogBucket)
if b.Get(itob(op.Id)) == nil {
if b.Get(serializationutil.Itob(op.Id)) == nil {
return fmt.Errorf("operation with ID %d does not exist", op.Id)
}
@@ -135,25 +131,28 @@ func (o *OpLog) Update(op *v1.Operation) error {
return fmt.Errorf("error marshalling operation: %w", err)
}
if err := b.Put(itob(op.Id), bytes); err != nil {
if err := b.Put(serializationutil.Itob(op.Id), bytes); err != nil {
return fmt.Errorf("error putting operation into bucket: %w", err)
}
return nil
})
if err != nil {
o.subscribersMu.RLock()
defer o.subscribersMu.RUnlock()
for _, sub := range o.subscribers {
(*sub)(EventTypeOpUpdated, op)
}
if err == nil {
o.notifyHelper(EventTypeOpUpdated, op)
}
return err
}
func (o *OpLog) notifyHelper(eventType EventType, op *v1.Operation) {
o.subscribersMu.RLock()
defer o.subscribersMu.RUnlock()
for _, sub := range o.subscribers {
(*sub)(eventType, op)
}
}
func (o *OpLog) getHelper(b *bolt.Bucket, id int64) (*v1.Operation, error) {
bytes := b.Get(itob(id))
bytes := b.Get(serializationutil.Itob(id))
if bytes == nil {
return nil, fmt.Errorf("operation with ID %d does not exist", id)
}
@@ -181,10 +180,7 @@ func (o *OpLog) Get(id int64) (*v1.Operation, error) {
func (o *OpLog) GetByRepo(repoId string, filter Filter) ([]*v1.Operation, error) {
var ops []*v1.Operation
if err := o.db.View(func(tx *bolt.Tx) error {
ids, err := o.readOpsFromIndexBucket(tx, RepoIndexBucket, repoId)
if err != nil {
return err
}
ids := indexutil.IndexSearchByteValue(tx.Bucket(RepoIndexBucket), []byte(repoId)).ToSlice()
b := tx.Bucket(OpLogBucket)
for _, id := range ids {
@@ -205,10 +201,7 @@ func (o *OpLog) GetByRepo(repoId string, filter Filter) ([]*v1.Operation, error)
func (o *OpLog) GetByPlan(planId string, filter Filter) ([]*v1.Operation, error) {
var ops []*v1.Operation
if err := o.db.View(func(tx *bolt.Tx) error {
ids, err := o.readOpsFromIndexBucket(tx, PlanIndexBucket, planId)
if err != nil {
return err
}
ids := indexutil.IndexSearchByteValue(tx.Bucket(PlanIndexBucket), []byte(planId)).ToSlice()
b := tx.Bucket(OpLogBucket)
for _, id := range ids {
@@ -249,9 +242,8 @@ func (o *OpLog) addOpToIndexBucket(tx *bolt.Tx, bucket []byte, indexId string, o
b := tx.Bucket(bucket)
var key []byte
key = append(key, itob(int64(len(indexId)))...)
key = append(key, []byte(indexId)...)
key = append(key, itob(opId)...)
key = append(key, serializationutil.Stob(indexId)...)
key = append(key, serializationutil.Itob(opId)...)
if err := b.Put(key, []byte{}); err != nil {
return fmt.Errorf("error adding operation to repo index: %w", err)
}
@@ -264,9 +256,9 @@ func (o *OpLog) readOpsFromIndexBucket(tx *bolt.Tx, bucket []byte, indexId strin
var ops []int64
c := b.Cursor()
prefix := stob(indexId)
prefix := serializationutil.Stob(indexId)
for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = c.Next() {
ops = append(ops, btoi(k[len(prefix):]))
ops = append(ops, serializationutil.Btoi(k[len(prefix):]))
}
return ops, nil

1
internal/oplog/oplog_test.go → internal/database/oplog/oplog_test.go
View File

@@ -141,6 +141,7 @@ func TestListOperation(t *testing.T) {
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
var ops []*v1.Operation

36
internal/database/serializationutil/serializationutil.go Normal file
View File

@@ -0,0 +1,36 @@
package serializationutil
import "encoding/binary"
func Itob(v int64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(v))
return b
}
func Btoi(b []byte) int64 {
return int64(binary.BigEndian.Uint64(b))
}
func Stob(v string) []byte {
var b []byte
b = append(b, Itob(int64(len(v)))...)
b = append(b, []byte(v)...)
return b
}
func Btos(b []byte) (string, int64) {
length := Btoi(b[:8])
return string(b[8:8+length]), 8+length
}
func BytesToKey(b []byte) []byte {
var key []byte
key = append(key, Itob(int64(len(b)))...)
key = append(key, b...)
return key
}
func NormalizeSnapshotId(id string) string {
return id[:8]
}

10
internal/oplog/serializationutil_test.go → internal/database/serializationutil/serializationutil_test.go
View File

@@ -1,12 +1,12 @@
package oplog
package serializationutil
import "testing"
func TestItoa(t *testing.T) {
nums := []int64{0, 1, 2, 3, 4, 1 << 32, int64(1) << 62}
for _, num := range nums {
b := itob(num)
if btoi(b) != num {
b := Itob(num)
if Btoi(b) != num {
t.Errorf("itob/btoi failed for %d", num)
}
}
@@ -15,8 +15,8 @@ func TestItoa(t *testing.T) {
func TestStob(t *testing.T) {
strs := []string{"", "a", "ab", "abc", "abcd", "abcde", "abcdef"}
for _, str := range strs {
b := stob(str)
if val, _ := btos(b); val != str {
b := Stob(str)
if val, _ := Btos(b); val != str {
t.Errorf("stob/btos failed for %s", str)
}
}

29
internal/oplog/serializationutil.go
View File

@@ -1,29 +0,0 @@
package oplog
import "encoding/binary"
func itob(v int64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(v))
return b
}
func btoi(b []byte) int64 {
return int64(binary.BigEndian.Uint64(b))
}
func stob(v string) []byte {
var b []byte
b = append(b, itob(int64(len(v)))...)
b = append(b, []byte(v)...)
return b
}
func btos(b []byte) (string, int64) {
length := btoi(b[:8])
return string(b[8:8+length]), 8+length
}
func normalizeSnapshotId(id string) string {
return id[:8]
}

2
internal/orchestrator/orchestrator.go
View File

@@ -9,7 +9,7 @@ import (
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"github.com/garethgeorge/resticui/internal/config"
"github.com/garethgeorge/resticui/internal/oplog"
"github.com/garethgeorge/resticui/internal/database/oplog"
"github.com/garethgeorge/resticui/pkg/restic"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"

2
internal/orchestrator/tasks.go
View File

@@ -6,7 +6,7 @@ import (
"time"
v1 "github.com/garethgeorge/resticui/gen/go/v1"
"github.com/garethgeorge/resticui/internal/oplog"
"github.com/garethgeorge/resticui/internal/database/oplog"
"github.com/garethgeorge/resticui/pkg/restic"
"github.com/gitploy-io/cronexpr"
"github.com/hashicorp/go-multierror"

6
proto/v1/operations.proto
View File

@@ -21,6 +21,7 @@ message Operation {
oneof op {
OperationBackup operation_backup = 100;
OperationIndexSnapshot operation_index_snapshot = 101;
}
}
@@ -48,3 +49,8 @@ enum OperationStatus {
message OperationBackup {
BackupProgressEntry last_status = 3;
}
// OperationIndexSnapshot tracks that a snapshot was detected by resticui.
message OperationIndexSnapshot {
ResticSnapshot snapshot = 2;
}