From aeb831868bfe54efbb92ce8a2adf284a7274d619 Mon Sep 17 00:00:00 2001 From: garethgeorge Date: Fri, 10 Nov 2023 18:32:29 -0800 Subject: [PATCH] restic library --- cmd/resticui/resticui.go | 13 +-- gen/go/v1/config.pb.go | 115 ++++--------------- gen/ts/v1/config.pb.ts | 7 +- go.mod | 11 +- go.sum | 10 ++ internal/config/config.go | 42 ------- internal/config/validate.go | 68 ++++++++++++ internal/config/yamlstore.go | 18 ++- internal/eventlog/rotatinglogdir.go | 0 internal/restic/error.go | 40 +++++++ internal/restic/outputs.go | 106 ++++++++++++++++++ internal/restic/restic.go | 165 ++++++++++++++++++++++++++++ internal/restic/restic_test.go | 84 ++++++++++++++ internal/test/helpers/testdata.go | 21 ++++ proto/v1/config.proto | 7 +- 15 files changed, 543 insertions(+), 164 deletions(-) create mode 100644 internal/config/validate.go delete mode 100644 internal/eventlog/rotatinglogdir.go create mode 100644 internal/restic/error.go create mode 100644 internal/restic/outputs.go create mode 100644 internal/restic/restic_test.go create mode 100644 internal/test/helpers/testdata.go diff --git a/cmd/resticui/resticui.go b/cmd/resticui/resticui.go index b518ae36..3f83f2b6 100644 --- a/cmd/resticui/resticui.go +++ b/cmd/resticui/resticui.go @@ -27,11 +27,15 @@ func main() { mux := http.NewServeMux() mux.Handle("/", http.FileServer(http.FS(static.FS))) + server := &http.Server{ + Addr: ":9090", + Handler: mux, + } + // Serve the API wg.Add(1) go func() { defer wg.Done() - defer cancel() err := api.ServeAPI(ctx, mux) if err != nil { zap.S().Fatal("Error serving API", zap.Error(err)) @@ -39,11 +43,6 @@ func main() { cancel() // cancel the context when the API server exits (e.g. on fatal error) }() - server := &http.Server{ - Addr: ":9090", - Handler: mux, - } - // Serve the HTTP gateway wg.Add(1) go func() { @@ -66,7 +65,7 @@ func main() { func init() { zap.ReplaceGlobals(zap.Must(zap.NewProduction())) if os.Getenv("DEBUG") != "" { - zap.ReplaceGlobals(zap.Must(zap.NewDevelopment())) + zap.ReplaceGlobals(zap.Must(zap.NewDevelopmentConfig().Build())) } } diff --git a/gen/go/v1/config.pb.go b/gen/go/v1/config.pb.go index caabf8b3..185b180f 100644 --- a/gen/go/v1/config.pb.go +++ b/gen/go/v1/config.pb.go @@ -96,10 +96,10 @@ type Repo struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - Uri string `protobuf:"bytes,2,opt,name=uri,proto3" json:"uri,omitempty"` - Password string `protobuf:"bytes,3,opt,name=password,proto3" json:"password,omitempty"` - Env []*EnvVar `protobuf:"bytes,4,rep,name=env,proto3" json:"env,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Uri string `protobuf:"bytes,2,opt,name=uri,proto3" json:"uri,omitempty"` + Password string `protobuf:"bytes,3,opt,name=password,proto3" json:"password,omitempty"` + Env []string `protobuf:"bytes,4,rep,name=env,proto3" json:"env,omitempty"` } func (x *Repo) Reset() { @@ -155,7 +155,7 @@ func (x *Repo) GetPassword() string { return "" } -func (x *Repo) GetEnv() []*EnvVar { +func (x *Repo) GetEnv() []string { if x != nil { return x.Env } @@ -233,61 +233,6 @@ func (x *Plan) GetPaths() []string { return nil } -type EnvVar struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` -} - -func (x *EnvVar) Reset() { - *x = EnvVar{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_config_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EnvVar) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EnvVar) ProtoMessage() {} - -func (x *EnvVar) ProtoReflect() protoreflect.Message { - mi := &file_v1_config_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EnvVar.ProtoReflect.Descriptor instead. -func (*EnvVar) Descriptor() ([]byte, []int) { - return file_v1_config_proto_rawDescGZIP(), []int{3} -} - -func (x *EnvVar) GetName() string { - if x != nil { - return x.Name - } - return "" -} - -func (x *EnvVar) GetValue() string { - if x != nil { - return x.Value - } - return "" -} - var File_v1_config_proto protoreflect.FileDescriptor var file_v1_config_proto_rawDesc = []byte{ @@ -300,22 +245,18 @@ var file_v1_config_proto_rawDesc = []byte{ 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x52, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x1e, 0x0a, 0x05, 0x70, 0x6c, 0x61, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6c, 0x61, 0x6e, 0x52, 0x05, 0x70, 0x6c, - 0x61, 0x6e, 0x73, 0x22, 0x62, 0x0a, 0x04, 0x52, 0x65, 0x70, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x61, 0x6e, 0x73, 0x22, 0x56, 0x0a, 0x04, 0x52, 0x65, 0x70, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x69, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x69, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x1c, 0x0a, 0x03, 0x65, 0x6e, 0x76, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x56, - 0x61, 0x72, 0x52, 0x03, 0x65, 0x6e, 0x76, 0x22, 0x5e, 0x0a, 0x04, 0x50, 0x6c, 0x61, 0x6e, 0x12, - 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, - 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, - 0x65, 0x70, 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, - 0x68, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x22, 0x32, 0x0a, 0x06, 0x45, 0x6e, 0x76, 0x56, 0x61, - 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x2e, 0x5a, 0x2c, 0x67, + 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x76, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x65, 0x6e, 0x76, 0x22, 0x5e, 0x0a, 0x04, 0x50, + 0x6c, 0x61, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, + 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, + 0x5f, 0x70, 0x61, 0x74, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x72, 0x65, 0x73, 0x74, 0x69, 0x63, 0x75, 0x69, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, @@ -334,22 +275,20 @@ func file_v1_config_proto_rawDescGZIP() []byte { return file_v1_config_proto_rawDescData } -var file_v1_config_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_v1_config_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_v1_config_proto_goTypes = []interface{}{ (*Config)(nil), // 0: v1.Config (*Repo)(nil), // 1: v1.Repo (*Plan)(nil), // 2: v1.Plan - (*EnvVar)(nil), // 3: v1.EnvVar } var file_v1_config_proto_depIdxs = []int32{ 1, // 0: v1.Config.repos:type_name -> v1.Repo 2, // 1: v1.Config.plans:type_name -> v1.Plan - 3, // 2: v1.Repo.env:type_name -> v1.EnvVar - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_v1_config_proto_init() } @@ -394,18 +333,6 @@ func file_v1_config_proto_init() { return nil } } - file_v1_config_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EnvVar); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } } type x struct{} out := protoimpl.TypeBuilder{ @@ -413,7 +340,7 @@ func file_v1_config_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_v1_config_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 3, NumExtensions: 0, NumServices: 0, }, diff --git a/gen/ts/v1/config.pb.ts b/gen/ts/v1/config.pb.ts index eb4f369d..1bbadee2 100644 --- a/gen/ts/v1/config.pb.ts +++ b/gen/ts/v1/config.pb.ts @@ -14,7 +14,7 @@ export type Repo = { id?: string uri?: string password?: string - env?: EnvVar[] + env?: string[] } export type Plan = { @@ -22,9 +22,4 @@ export type Plan = { repo?: string repoPath?: string paths?: string[] -} - -export type EnvVar = { - name?: string - value?: string } \ No newline at end of file diff --git a/go.mod b/go.mod index e239b56e..f2f60fcd 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,11 @@ go 1.21.3 require ( github.com/google/renameio v1.0.1 - github.com/grpc-ecosystem/grpc-gateway v1.16.0 - github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 github.com/hashicorp/go-multierror v1.1.1 go.uber.org/zap v1.26.0 - google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b + golang.org/x/sync v0.5.0 + google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 @@ -17,12 +17,13 @@ require ( require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/vine-io/vine v1.6.16 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect + google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect ) diff --git a/go.sum b/go.sum index 695dd745..65d95da8 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2 h1:dygLcbEBA+t/P7ck6a8AkXv6juQ4cK0RHBoh32jxhHM= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.2/go.mod h1:Ap9RLCIJVtgQg1/BBgVEfypOAySvvlcpcVQkSzJCH4Y= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= @@ -69,6 +71,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -77,6 +81,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -107,8 +113,12 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 h1:I6WNifs6pF9tNdSob2W24JtyxIYjzFB9qDlpUC76q+U= google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405/go.mod h1:3WDQMjmJk36UQhjQ89emUzb1mdaHcPeeAh4SCBKznB4= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/internal/config/config.go b/internal/config/config.go index f3e44100..feec4510 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,14 +1,12 @@ package config import ( - "errors" "flag" "fmt" "os" "path" v1 "github.com/garethgeorge/resticui/gen/go/v1" - multierror "github.com/hashicorp/go-multierror" ) var configDirFlag = flag.String("config_dir", "", "The directory to store the config file") @@ -30,46 +28,6 @@ func NewDefaultConfig() *v1.Config { } } -func ValidateConfig(c *v1.Config) error { - if c.LogDir == "" { - return errors.New("log_dir is required") - } - - if c.Repos == nil { - return errors.New("repos is required") - } - - if c.Plans == nil { - return errors.New("plans is required") - } - - var error error - - repos := make(map[string]*v1.Repo) - for _, repo := range c.Repos { - if repo.GetId() == "" { - error = multierror.Append(error, fmt.Errorf("repo name is required")) - } - repos[repo.GetId()] = repo - } - - for _, plan := range c.Plans { - if plan.Paths == nil || len(plan.Paths) == 0 { - error = multierror.Append(error, fmt.Errorf("plan %s: path is required", plan.GetId())) - } - - if plan.Repo == "" { - error = multierror.Append(error,fmt.Errorf("plan %s: repo is required", plan.GetId())) - } - - if _, ok := repos[plan.Repo]; !ok { - error = multierror.Append(error, fmt.Errorf("plan %s: repo %s not found", plan.GetId(), plan.Repo)) - } - } - - return error -} - func configDir(override string) string { if override != "" { return override diff --git a/internal/config/validate.go b/internal/config/validate.go new file mode 100644 index 00000000..c12eb455 --- /dev/null +++ b/internal/config/validate.go @@ -0,0 +1,68 @@ +package config + +import ( + "errors" + "fmt" + "strings" + + v1 "github.com/garethgeorge/resticui/gen/go/v1" + "github.com/hashicorp/go-multierror" +) + +func validateConfig(c *v1.Config) error { + if c.LogDir == "" { + return errors.New("log_dir is required") + } + + var err error + repos := make(map[string]*v1.Repo) + if c.Repos != nil { + for _, repo := range c.Repos { + if e := validateRepo(repo); e != nil { + err = multierror.Append(e, fmt.Errorf("repo %s: %w", repo.GetId(), err)) + } + repos[repo.GetId()] = repo + } + } + + if c.Plans != nil { + for _, plan := range c.Plans { + if plan.Paths == nil || len(plan.Paths) == 0 { + err = multierror.Append(err, fmt.Errorf("plan %s: path is required", plan.GetId())) + } + + if plan.Repo == "" { + err = multierror.Append(err,fmt.Errorf("plan %s: repo is required", plan.GetId())) + } + + if _, ok := repos[plan.Repo]; !ok { + err = multierror.Append(err, fmt.Errorf("plan %s: repo %s not found", plan.GetId(), plan.Repo)) + } + } + } + + return err +} + +func validateRepo(repo *v1.Repo) error { + var err error + if repo.GetId() == "" { + err = multierror.Append(err, errors.New("id is required")) + } + + if repo.GetUri() == "" { + err = multierror.Append(err, errors.New("uri is required")) + } + + if repo.GetPassword() == "" { + err = multierror.Append(err, errors.New("password is required")) + } + + for _, env := range repo.GetEnv() { + if !strings.Contains(env, "=") { + err = multierror.Append(err, fmt.Errorf("invalid env var %s, must take format KEY=VALUE", env)) + } + } + + return err +} \ No newline at end of file diff --git a/internal/config/yamlstore.go b/internal/config/yamlstore.go index 9a45ac83..830f36c9 100644 --- a/internal/config/yamlstore.go +++ b/internal/config/yamlstore.go @@ -24,9 +24,9 @@ var _ ConfigStore = &YamlFileStore{} func (f *YamlFileStore) Get() (*v1.Config, error) { f.mu.Lock() - defer f.mu.Unlock() if f.config != nil { + f.mu.Unlock() return f.config, nil } @@ -41,17 +41,23 @@ func (f *YamlFileStore) Get() (*v1.Config, error) { return nil, fmt.Errorf("failed to read config file: %w", err) } + defer f.mu.Unlock() + data, err = yamlToJson(data) if err != nil { return nil, fmt.Errorf("failed to parse YAML config: %w", err) } var config v1.Config - err = protojson.Unmarshal(data, &config) - if err != nil { + + if err = protojson.Unmarshal(data, &config); err != nil { return nil, fmt.Errorf("failed to unmarshal config: %w", err) } + if err := validateConfig(&config); err != nil { + return nil, fmt.Errorf("invalid config: %w", err) + } + f.config = &config return f.config, nil } @@ -60,6 +66,10 @@ func (f *YamlFileStore) Update(config *v1.Config) error { f.mu.Lock() defer f.mu.Unlock() + if err := validateConfig(config); err != nil { + return fmt.Errorf("invalid config: %w", err) + } + data, err := protojson.Marshal(config) if err != nil { return fmt.Errorf("failed to marshal config: %w", err) @@ -79,11 +89,11 @@ func (f *YamlFileStore) Update(config *v1.Config) error { if err != nil { return fmt.Errorf("failed to write config file: %w", err) } + f.config = config return nil } - func jsonToYaml(data []byte) ([]byte, error) { var config interface{} err := json.Unmarshal(data, &config) diff --git a/internal/eventlog/rotatinglogdir.go b/internal/eventlog/rotatinglogdir.go deleted file mode 100644 index e69de29b..00000000 diff --git a/internal/restic/error.go b/internal/restic/error.go new file mode 100644 index 00000000..4b497d1e --- /dev/null +++ b/internal/restic/error.go @@ -0,0 +1,40 @@ +package restic + +import ( + "fmt" + "os/exec" +) + +type CmdError struct { + Command string + Err error + Output string +} + +func (e *CmdError) Error() string { + m := fmt.Sprintf("command %s failed: %s", e.Command, e.Err.Error()) + if e.Output != "" { + m += "\nDetails: \n" + e.Output + } + return m +} + +func (e *CmdError) Unwrap() error { + return e.Err +} + +// NewCmdError creates a new error indicating that running a command failed. +func NewCmdError(cmd *exec.Cmd, output []byte, err error) *CmdError { + cerr := &CmdError{ + Command: cmd.String(), + Err: err, + } + + if len(output) > 0 { + if len(output) > 1000 { + output = output[:1000] + } + cerr.Output = string(output) + } + return cerr +} \ No newline at end of file diff --git a/internal/restic/outputs.go b/internal/restic/outputs.go new file mode 100644 index 00000000..e21866e1 --- /dev/null +++ b/internal/restic/outputs.go @@ -0,0 +1,106 @@ +package restic + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "os/exec" + "slices" + + "go.uber.org/zap" +) + +type Snapshot struct { + Time string `json:"time"` + Tree string `json:"tree"` + Paths []string `json:"paths"` + Hostname string `json:"hostname"` + Username string `json:"username"` + Id string `json:"id"` + ShortId string `json:"short_id"` +} + +type BackupEvent struct { + // Common fields + MessageType string `json:"message_type"` // "summary" or "status" + + // Summary fields + FilesNew int `json:"files_new"` + FilesChanged int `json:"files_changed"` + FilesUnmodified int `json:"files_unmodified"` + DirsNew int `json:"dirs_new"` + DirsChanged int `json:"dirs_changed"` + DirsUnmodified int `json:"dirs_unmodified"` + DataBlobs int `json:"data_blobs"` + TreeBlobs int `json:"tree_blobs"` + DataAdded int `json:"data_added"` + TotalFilesProcessed int `json:"total_files_processed"` + TotalBytesProcessed int `json:"total_bytes_processed"` + TotalDuration float64 `json:"total_duration"` + SnapshotId string `json:"snapshot_id"` + + // Status fields + PercentDone float64 `json:"percent_done"` + TotalFiles int `json:"total_files"` + FilesDone int `json:"files_done"` + TotalBytes int `json:"total_bytes"` + BytesDone int `json:"bytes_done"` + + // Error fields + Error string `json:"error"` +} + +// readBackupEvents returns the summary event or an error if the command failed. +func readBackupEvents(cmd *exec.Cmd, output io.Reader, callback func(event *BackupEvent)) (*BackupEvent, error) { + scanner := bufio.NewScanner(output) + scanner.Split(bufio.ScanLines) + + // first event is handled specially to detect non-JSON output and fast-path out. + if scanner.Scan() { + var event BackupEvent + + if err := json.Unmarshal(scanner.Bytes(), &event); err != nil { + var bytes = slices.Clone(scanner.Bytes()) + for scanner.Scan() { + bytes = append(bytes, scanner.Bytes()...) + } + + jsonErr := fmt.Errorf("command output was not JSON: %w", err) + + if err := cmd.Wait(); err != nil { + return nil, NewCmdError(cmd, bytes, errors.Join( + fmt.Errorf("command failed: %w", err), + fmt.Errorf("command output was not JSON: %w", err), + )) + } + + return nil, NewCmdError(cmd, bytes, jsonErr) + } + } + + // remaining events are parsed as JSON + var summary *BackupEvent + + for scanner.Scan() { + var event *BackupEvent + if err := json.Unmarshal(scanner.Bytes(), &event); err != nil { + return nil, fmt.Errorf("failed to parse JSON: %w", err) + } + + callback(event) + + if event.MessageType == "summary" { + summary = event + } + } + + if err := scanner.Err(); err != nil { + return summary, fmt.Errorf("scanner encountered error: %w", err) + } + + zap.L().Debug("finished reading events", zap.String("command", cmd.String())) + + return summary, nil +} diff --git a/internal/restic/restic.go b/internal/restic/restic.go index e69de29b..61157937 100644 --- a/internal/restic/restic.go +++ b/internal/restic/restic.go @@ -0,0 +1,165 @@ +package restic + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + + v1 "github.com/garethgeorge/resticui/gen/go/v1" + "golang.org/x/sync/errgroup" +) + +type Repo struct { + cmd string + repo *v1.Repo + flags []string + env []string + initialized bool +} + +func NewRepo(repo *v1.Repo, opts ...RepoOption) *Repo { + var opt RepoOpts + for _, o := range opts { + o(&opt) + } + + return &Repo{ + cmd: "restic", // TODO: configurable binary path + repo: repo, + flags: opt.flags, + env: opt.env, + } +} + +func (r *Repo) buildEnv() []string { + env := []string{ + "RESTIC_REPOSITORY=" + r.repo.GetUri(), + "RESTIC_PASSWORD=" + r.repo.GetPassword(), + } + env = append(env, r.repo.GetEnv()...) + env = append(env, r.env...) + return env +} + +// init initializes the repo, the command will be cancelled with the context. +func (r *Repo) init(ctx context.Context) error { + if r.initialized { + return nil + } + + var args = []string{"init", "--json"} + args = append(args, r.flags...) + + cmd := exec.CommandContext(ctx, r.cmd, args...) + cmd.Env = append(cmd.Env, r.buildEnv()...) + + if output, err := cmd.CombinedOutput(); err != nil { + return NewCmdError(cmd, output, err) + } + + r.initialized = true + return nil +} + +func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupEvent), opts ...BackupOption) (*BackupEvent, error) { + if err := r.init(ctx); err != nil { + return nil, fmt.Errorf("failed to initialize repo: %w", err) + } + + opt := &BackupOpts{} + for _, o := range opts { + o(opt) + } + + args := []string{"backup", "--json", "--exclude-caches"} + args = append(args, r.flags...) + args = append(args, opt.paths...) + + for _, e := range opt.excludes { + args = append(args, "--exclude", e) + } + + reader, writer := io.Pipe() + + cmd := exec.CommandContext(ctx, r.cmd, args...) + cmd.Env = append(cmd.Env, r.buildEnv()...) + cmd.Stderr = writer + cmd.Stdout = writer + + if err := cmd.Start(); err != nil { + return nil, NewCmdError(cmd, nil, err) + } + + var summary *BackupEvent + var errgroup errgroup.Group + + errgroup.Go(func() error { + var err error + summary, err = readBackupEvents(cmd, reader, progressCallback) + if err != nil { + return fmt.Errorf("processing command output: %w", err) + } + return nil + }) + + errgroup.Go(func() error { + defer writer.Close() + if err := cmd.Wait(); err != nil { + return NewCmdError(cmd, nil, err) + } + return nil + }) + + if err := errgroup.Wait(); err != nil { + return nil, err + } + + return summary, nil +} + +type RepoOpts struct { + env []string // global env overrides + flags []string // global flags +} + +type RepoOption func(opts *RepoOpts) + +// WithHostEnv copies values from the host environment into the restic environment. +func WithRepoHostEnv() RepoOption { + return func(opts *RepoOpts) { + opts.env = append(opts.env, "HOME=" + os.Getenv("HOME"), "XDG_CACHE_HOME=" + os.Getenv("XDG_CACHE_HOME")) + } +} + +func WithRepoEnv(env ...string) RepoOption { + return func(opts *RepoOpts) { + opts.env = append(opts.env, env...) + } +} + +func WithRepoFlags(flags ...string) RepoOption { + return func(opts *RepoOpts) { + opts.flags = append(opts.flags, flags...) + } +} + +type BackupOpts struct { + paths []string + excludes []string +} + +type BackupOption func(opts *BackupOpts) + +func WithBackupPaths(paths ...string) BackupOption { + return func(opts *BackupOpts) { + opts.paths = append(opts.paths, paths...) + } +} + +func WithBackupExcludes(excludes ...string) BackupOption { + return func(opts *BackupOpts) { + opts.excludes = append(opts.excludes, excludes...) + } +} diff --git a/internal/restic/restic_test.go b/internal/restic/restic_test.go new file mode 100644 index 00000000..5636720d --- /dev/null +++ b/internal/restic/restic_test.go @@ -0,0 +1,84 @@ +package restic + +import ( + "context" + "testing" + + v1 "github.com/garethgeorge/resticui/gen/go/v1" + test "github.com/garethgeorge/resticui/internal/test/helpers" +) + +func TestResticInit(t *testing.T) { + repo := t.TempDir() + + r := NewRepo(&v1.Repo{ + Id: "test", + Uri: repo, + Password: "test", + }, WithRepoFlags("--no-cache")) + + r.init(context.Background()) +} + +func TestResticBackup(t *testing.T) { + repo := t.TempDir() + + // create a new repo with cache disabled for testing + r := NewRepo(&v1.Repo{ + Id: "test", + Uri: repo, + Password: "test", + }, WithRepoFlags("--no-cache")) + + r.init(context.Background()) + + testData := test.CreateTestData(t) + testData2 := test.CreateTestData(t) + + var tests = []struct { + name string + opts []BackupOption + files int // expected files at the end of the backup + }{ + { + name: "no options", + opts: []BackupOption{WithBackupPaths(testData)}, + files: 100, + }, + { + name: "with two paths", + opts:[]BackupOption{WithBackupPaths(testData), WithBackupPaths(testData2)}, + files: 200, + }, + { + name: "with exclude", + opts: []BackupOption{WithBackupPaths(testData), WithBackupExcludes("file1*")}, + files: 90, + }, + { + name: "with exclude pattern", + opts: []BackupOption{WithBackupPaths(testData), WithBackupExcludes("file*")}, + files: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + summary, err := r.Backup(context.Background(), func(event *BackupEvent) { + t.Logf("backup event: %v", event) + }, tc.opts...) + if err != nil { + t.Errorf("failed to backup: %v", err) + } + + if summary == nil { + t.Fatalf("wanted summary, got: nil") + } + + if summary.TotalFilesProcessed != tc.files { + t.Errorf("wanted %d files, got: %d", tc.files, summary.TotalFilesProcessed) + } + }) + } +} + diff --git a/internal/test/helpers/testdata.go b/internal/test/helpers/testdata.go new file mode 100644 index 00000000..dfc79d0a --- /dev/null +++ b/internal/test/helpers/testdata.go @@ -0,0 +1,21 @@ +package test + +import ( + "fmt" + "os" + "path" + "testing" +) + +func CreateTestData(t *testing.T) string { + t.Helper() + dir := t.TempDir() + + for i := 0; i < 100; i++ { + err := os.WriteFile(path.Join(dir, fmt.Sprintf("file%2d", i)), []byte(fmt.Sprintf("test data %d", i)), 0644) + if err != nil { + t.Fatalf("failed to create test data: %v", err) + } + } + return dir +} \ No newline at end of file diff --git a/proto/v1/config.proto b/proto/v1/config.proto index a2d4c125..c32615c2 100644 --- a/proto/v1/config.proto +++ b/proto/v1/config.proto @@ -15,7 +15,7 @@ message Repo { string id = 1 [json_name="id"]; string uri = 2 [json_name="uri"]; string password = 3 [json_name="password"]; - repeated EnvVar env = 4 [json_name="env"]; + repeated string env = 4 [json_name="env"]; } message Plan { @@ -24,8 +24,3 @@ message Plan { string repo_path = 3 [json_name="repo_path"]; // subpath of the repo to backup to repeated string paths = 4 [json_name="paths"]; } - -message EnvVar { - string name = 1 [json_name="name"]; - string value = 2 [json_name="value"]; -} \ No newline at end of file