lee/backrest
1
0
Fork 0
mirror of https://github.com/garethgeorge/backrest.git synced 2026-08-28 03:46:29 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: add repo stats to restic package

Browse Source
This commit is contained in:
Gareth George
2023-12-24 08:36:43 +00:00
parent 2ab4a94f44
commit 086f9cd92f
11 changed files with 348 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backrest.go
+1 -1
View File
@@ -73,7 +73,7 @@ func main() {
// Create and serve the HTTP gateway
apiServer := api.NewServer(
configStore,
orchestrator, // TODO: eliminate default config
orchestrator,
oplog,
)
gen/go/v1/restic.pb.go
+113 -6
View File
@@ -582,6 +582,85 @@ func (x *RestoreProgressEntry) GetPercentDone() float64 {
return 0
}
type RepoStats struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
TotalSize int64 `protobuf:"varint,1,opt,name=total_size,json=totalSize,proto3" json:"total_size,omitempty"`
TotalUncompressedSize int64 `protobuf:"varint,2,opt,name=total_uncompressed_size,json=totalUncompressedSize,proto3" json:"total_uncompressed_size,omitempty"`
CompressionRatio float64 `protobuf:"fixed64,3,opt,name=compression_ratio,json=compressionRatio,proto3" json:"compression_ratio,omitempty"`
TotalBlobCount int64 `protobuf:"varint,5,opt,name=total_blob_count,json=totalBlobCount,proto3" json:"total_blob_count,omitempty"`
SnapshotCount int64 `protobuf:"varint,6,opt,name=snapshot_count,json=snapshotCount,proto3" json:"snapshot_count,omitempty"`
}
func (x *RepoStats) Reset() {
*x = RepoStats{}
if protoimpl.UnsafeEnabled {
mi := &file_v1_restic_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *RepoStats) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*RepoStats) ProtoMessage() {}
func (x *RepoStats) ProtoReflect() protoreflect.Message {
mi := &file_v1_restic_proto_msgTypes[6]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use RepoStats.ProtoReflect.Descriptor instead.
func (*RepoStats) Descriptor() ([]byte, []int) {
return file_v1_restic_proto_rawDescGZIP(), []int{6}
}
func (x *RepoStats) GetTotalSize() int64 {
if x != nil {
return x.TotalSize
}
return 0
}
func (x *RepoStats) GetTotalUncompressedSize() int64 {
if x != nil {
return x.TotalUncompressedSize
}
return 0
}
func (x *RepoStats) GetCompressionRatio() float64 {
if x != nil {
return x.CompressionRatio
}
return 0
}
func (x *RepoStats) GetTotalBlobCount() int64 {
if x != nil {
return x.TotalBlobCount
}
return 0
}
func (x *RepoStats) GetSnapshotCount() int64 {
if x != nil {
return x.SnapshotCount
}
return 0
}
var File_v1_restic_proto protoreflect.FileDescriptor
var file_v1_restic_proto_rawDesc = []byte{
@@ -676,10 +755,25 @@ var file_v1_restic_proto_rawDesc = []byte{
0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74,
0x5f, 0x64, 0x6f, 0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x70, 0x65, 0x72,
0x63, 0x65, 0x6e, 0x74, 0x44, 0x6f, 0x6e, 0x65, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68,
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f,
0x72, 0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, 0x67, 0x65, 0x6e,
0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x63, 0x65, 0x6e, 0x74, 0x44, 0x6f, 0x6e, 0x65, 0x22, 0xe0, 0x01, 0x0a, 0x09, 0x52, 0x65, 0x70,
0x6f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f,
0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x6f, 0x74, 0x61,
0x6c, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x36, 0x0a, 0x17, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75,
0x6e, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x5f, 0x73, 0x69, 0x7a, 0x65,
0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x15, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x63,
0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2b, 0x0a,
0x11, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65,
0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x74, 0x69, 0x6f, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x6f,
0x74, 0x61, 0x6c, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05,
0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x42, 0x6c, 0x6f, 0x62, 0x43,
0x6f, 0x75, 0x6e, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x73, 0x6e,
0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x42, 0x2c, 0x5a, 0x2a, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68,
0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f,
0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x33,
}
var (
@@ -694,7 +788,7 @@ func file_v1_restic_proto_rawDescGZIP() []byte {
return file_v1_restic_proto_rawDescData
}
var file_v1_restic_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
var file_v1_restic_proto_msgTypes = make([]protoimpl.MessageInfo, 7)
var file_v1_restic_proto_goTypes = []interface{}{
(*ResticSnapshot)(nil), // 0: v1.ResticSnapshot
(*ResticSnapshotList)(nil), // 1: v1.ResticSnapshotList
@@ -702,6 +796,7 @@ var file_v1_restic_proto_goTypes = []interface{}{
(*BackupProgressStatusEntry)(nil), // 3: v1.BackupProgressStatusEntry
(*BackupProgressSummary)(nil), // 4: v1.BackupProgressSummary
(*RestoreProgressEntry)(nil), // 5: v1.RestoreProgressEntry
(*RepoStats)(nil), // 6: v1.RepoStats
}
var file_v1_restic_proto_depIdxs = []int32{
0, // 0: v1.ResticSnapshotList.snapshots:type_name -> v1.ResticSnapshot
@@ -792,6 +887,18 @@ func file_v1_restic_proto_init() {
return nil
}
}
file_v1_restic_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*RepoStats); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
file_v1_restic_proto_msgTypes[2].OneofWrappers = []interface{}{
(*BackupProgressEntry_Status)(nil),
@@ -803,7 +910,7 @@ func file_v1_restic_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_v1_restic_proto_rawDesc,
NumEnums: 0,
NumMessages: 6,
NumMessages: 7,
NumExtensions: 0,
NumServices: 0,
},
internal/cache/cache.go
Vendored
+34
View File
@@ -0,0 +1,34 @@
package cache
import (
"fmt"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"go.etcd.io/bbolt"
)
type Cache struct {
db *bbolt.DB
}
func NewCache(databasePath string) (*Cache, error) {
db, err := bbolt.Open(databasePath, 0600, nil)
if err != nil {
return nil, fmt.Errorf("open database: %w", err)
}
return &Cache{db: db}, nil
}
func (c *Cache) Close() error {
return c.db.Close()
}
func (c *Cache) SetRepoStats(repo *v1.Repo, stats *v1.RepoStats) error {
return c.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("repo_stats"))
if err != nil {
return fmt.Errorf("create bucket: %w", err)
}
return b.Put([]byte(repo.Id), protoutil.MustMarshal(stats))
})
}
internal/oplog/oplog.go
+6 -20
View File
@@ -6,7 +6,6 @@ import (
"os"
"path"
"sync"
"sync/atomic"
"time"
v1 "github.com/garethgeorge/backrest/gen/go/v1"
@@ -30,12 +29,11 @@ const (
var ErrNotExist = errors.New("operation does not exist")
var (
SystemBucket = []byte("oplog.system") // system stores metadata
OpLogBucket = []byte("oplog.log") // oplog stores existant operations.
OpLogSoftDeleteBucket = []byte("oplog.log_soft_delete") // oplog_soft_delete stores soft deleted operations
RepoIndexBucket = []byte("oplog.repo_idx") // repo_index tracks IDs of operations affecting a given repo
PlanIndexBucket = []byte("oplog.plan_idx") // plan_index tracks IDs of operations affecting a given plan
SnapshotIndexBucket = []byte("oplog.snapshot_idx") // snapshot_index tracks IDs of operations affecting a given snapshot
SystemBucket = []byte("oplog.system") // system stores metadata
OpLogBucket = []byte("oplog.log") // oplog stores existant operations.
RepoIndexBucket = []byte("oplog.repo_idx") // repo_index tracks IDs of operations affecting a given repo
PlanIndexBucket = []byte("oplog.plan_idx") // plan_index tracks IDs of operations affecting a given plan
SnapshotIndexBucket = []byte("oplog.snapshot_idx") // snapshot_index tracks IDs of operations affecting a given snapshot
)
// OpLog represents a log of operations performed.
@@ -45,7 +43,6 @@ type OpLog struct {
subscribersMu sync.RWMutex
subscribers []*func(*v1.Operation, *v1.Operation)
nextId atomic.Int64
}
func NewOpLog(databasePath string) (*OpLog, error) {
@@ -61,12 +58,11 @@ func NewOpLog(databasePath string) (*OpLog, error) {
o := &OpLog{
db: db,
}
o.nextId.Store(1)
if err := db.Update(func(tx *bolt.Tx) error {
// Create the buckets if they don't exist
for _, bucket := range [][]byte{
SystemBucket, OpLogBucket, OpLogSoftDeleteBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket,
SystemBucket, OpLogBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket,
} {
if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
return fmt.Errorf("creating bucket %s: %s", string(bucket), err)
@@ -194,17 +190,7 @@ func (o *OpLog) Delete(ids ...int64) error {
return fmt.Errorf("deleting operation %v: %w", id, err)
}
removedOps = append(removedOps, removed)
b := tx.Bucket(OpLogSoftDeleteBucket)
val, err := proto.Marshal(removed)
if err != nil {
return fmt.Errorf("marshalling operation %v: %w", id, err)
}
if err := b.Put(serializationutil.Itob(id), val); err != nil {
return fmt.Errorf("putting operation %v into soft delete bucket: %w", id, err)
}
}
return nil
})
if err == nil {
internal/protoutil/conversion.go
+10
View File
@@ -107,3 +107,13 @@ func RestoreProgressEntryToProto(p *restic.RestoreProgressEntry) *v1.RestoreProg
PercentDone: p.PercentDone,
}
}
func RepoStatsToProto(s *restic.RepoStats) *v1.RepoStats {
return &v1.RepoStats{
TotalSize: int64(s.TotalSize),
TotalUncompressedSize: int64(s.TotalUncompressedSize),
CompressionRatio: s.CompressionRatio,
TotalBlobCount: int64(s.TotalBlobCount),
SnapshotCount: int64(s.SnapshotsCount),
}
}
internal/protoutil/conversion_test.go
+42
View File
@@ -116,3 +116,45 @@ func TestBackupProgressEntryToProto(t *testing.T) {
})
}
}
func TestRepoStatsToProto(t *testing.T) {
cases := []struct {
name string
stats *restic.RepoStats
want *v1.RepoStats
}{
{
name: "no stats",
stats: &restic.RepoStats{},
want: &v1.RepoStats{},
},
{
name: "with stats",
stats: &restic.RepoStats{
TotalSize: 1,
TotalUncompressedSize: 2,
CompressionRatio: 3,
TotalBlobCount: 5,
CompressionProgress: 6,
CompressionSpaceSaving: 7,
SnapshotsCount: 8,
},
want: &v1.RepoStats{
TotalSize: 1,
TotalUncompressedSize: 2,
CompressionRatio: 3,
TotalBlobCount: 5,
SnapshotCount: 8,
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := RepoStatsToProto(c.stats)
if !proto.Equal(got, c.want) {
t.Errorf("wanted: %+v, got: %+v", c.want, got)
}
})
}
}
pkg/restic/outputs.go
+10
View File
@@ -304,3 +304,13 @@ func ValidateSnapshotId(id string) error {
}
return nil
}
type RepoStats struct {
TotalSize int64 `json:"total_size"`
TotalUncompressedSize int64 `json:"total_uncompressed_size"`
CompressionRatio float64 `json:"compression_ratio"`
CompressionProgress int64 `json:"compression_progress"`
CompressionSpaceSaving float64 `json:"compression_space_saving"`
TotalBlobCount int64 `json:"total_blob_count"`
SnapshotsCount int64 `json:"snapshots_count"`
}
pkg/restic/restic.go
+24
View File
@@ -347,6 +347,30 @@ func (r *Repo) Unlock(ctx context.Context, opts ...GenericOption) error {
return nil
}
func (r *Repo) Stats(ctx context.Context, opts ...GenericOption) (*RepoStats, error) {
opt := resolveOpts(opts)
args := []string{"stats", "--json", "--mode=raw-data"}
args = append(args, r.extraArgs...)
args = append(args, opt.extraArgs...)
cmd := exec.CommandContext(ctx, r.cmd, args...)
cmd.Env = append(cmd.Env, r.buildEnv()...)
cmd.Env = append(cmd.Env, opt.extraEnv...)
output, err := cmd.CombinedOutput()
if err != nil {
return nil, newCmdError(cmd, string(output), err)
}
var stats RepoStats
if err := json.Unmarshal(output, &stats); err != nil {
return nil, newCmdError(cmd, string(output), fmt.Errorf("command output is not valid JSON: %w", err))
}
return &stats, nil
}
type RetentionPolicy struct {
KeepLastN int // keep the last n snapshots.
KeepHourly int // keep the last n hourly snapshots.
pkg/restic/restic_test.go
+39
View File
@@ -362,3 +362,42 @@ func TestResticRestore(t *testing.T) {
t.Errorf("wanted 101 files to be restored, got: %d", summary.TotalFiles)
}
}
func TestResticStats(t *testing.T) {
t.Parallel()
repo := t.TempDir()
r := NewRepo(helpers.ResticBinary(t), &v1.Repo{
Id: "test",
Uri: repo,
Password: "test",
}, WithFlags("--no-cache"))
if err := r.Init(context.Background()); err != nil {
t.Fatalf("failed to init repo: %v", err)
}
testData := helpers.CreateTestData(t)
_, err := r.Backup(context.Background(), nil, WithBackupPaths(testData))
if err != nil {
t.Fatalf("failed to backup and create new snapshot: %v", err)
}
// restore all files
stats, err := r.Stats(context.Background())
if err != nil {
t.Fatalf("failed to get stats: %v", err)
}
if stats.SnapshotsCount != 1 {
t.Errorf("wanted 1 snapshot, got: %d", stats.SnapshotsCount)
}
if stats.TotalSize == 0 {
t.Errorf("wanted non-zero total size, got: %d", stats.TotalSize)
}
if stats.TotalUncompressedSize == 0 {
t.Errorf("wanted non-zero total uncompressed size, got: %d", stats.TotalUncompressedSize)
}
if stats.TotalBlobCount == 0 {
t.Errorf("wanted non-zero total blob count, got: %d", stats.TotalBlobCount)
}
}
proto/v1/restic.proto
+8
View File
@@ -65,4 +65,12 @@ message RestoreProgressEntry {
int64 total_files = 5;
int64 files_restored = 6;
double percent_done = 7; // 0.0 - 1.0
}
message RepoStats {
int64 total_size = 1;
int64 total_uncompressed_size = 2;
double compression_ratio = 3;
int64 total_blob_count = 5;
int64 snapshot_count = 6;
}
webui/gen/ts/v1/restic_pb.ts
+61
View File
@@ -443,3 +443,64 @@ export class RestoreProgressEntry extends Message<RestoreProgressEntry> {
}
}
/**
* @generated from message v1.RepoStats
*/
export class RepoStats extends Message<RepoStats> {
/**
* @generated from field: int64 total_size = 1;
*/
totalSize = protoInt64.zero;
/**
* @generated from field: int64 total_uncompressed_size = 2;
*/
totalUncompressedSize = protoInt64.zero;
/**
* @generated from field: double compression_ratio = 3;
*/
compressionRatio = 0;
/**
* @generated from field: int64 total_blob_count = 5;
*/
totalBlobCount = protoInt64.zero;
/**
* @generated from field: int64 snapshot_count = 6;
*/
snapshotCount = protoInt64.zero;
constructor(data?: PartialMessage<RepoStats>) {
super();
proto3.util.initPartial(data, this);
}
static readonly runtime: typeof proto3 = proto3;
static readonly typeName = "v1.RepoStats";
static readonly fields: FieldList = proto3.util.newFieldList(() => [
{ no: 1, name: "total_size", kind: "scalar", T: 3 /* ScalarType.INT64 */ },
{ no: 2, name: "total_uncompressed_size", kind: "scalar", T: 3 /* ScalarType.INT64 */ },
{ no: 3, name: "compression_ratio", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ },
{ no: 5, name: "total_blob_count", kind: "scalar", T: 3 /* ScalarType.INT64 */ },
{ no: 6, name: "snapshot_count", kind: "scalar", T: 3 /* ScalarType.INT64 */ },
]);
static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): RepoStats {
return new RepoStats().fromBinary(bytes, options);
}
static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): RepoStats {
return new RepoStats().fromJson(jsonValue, options);
}
static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): RepoStats {
return new RepoStats().fromJsonString(jsonString, options);
}
static equals(a: RepoStats | PlainMessage<RepoStats> | undefined, b: RepoStats | PlainMessage<RepoStats> | undefined): boolean {
return proto3.util.equals(RepoStats, a, b);
}
}