Revert "chore: use storm for indexing on top boltdb"

This reverts commit 90e7b0c593.

internal/api/server.go

@@ -10,7 +10,7 @@ import (
 	"github.com/garethgeorge/resticui/gen/go/types"
 	v1 "github.com/garethgeorge/resticui/gen/go/v1"
 	"github.com/garethgeorge/resticui/internal/config"
-	"github.com/garethgeorge/resticui/internal/oplog"
+	"github.com/garethgeorge/resticui/internal/database/oplog"
 	"github.com/garethgeorge/resticui/internal/orchestrator"
 	"github.com/garethgeorge/resticui/pkg/restic"
 	"go.uber.org/zap"

internal/database/indexutil/indexutil.go

@@ -0,0 +1,57 @@
+package indexutil
+
+import (
+	"bytes"
+
+	"github.com/garethgeorge/resticui/internal/database/serializationutil"
+	bolt "go.etcd.io/bbolt"
+)
+
+// IndexByteValue indexes a value and recordId tuple creating multimap from value to lists of associated recordIds.
+func IndexByteValue(b *bolt.Bucket, value []byte, recordId int64) error {
+	key := serializationutil.BytesToKey(value)
+	key = append(key, serializationutil.Itob(recordId)...)
+	return b.Put(key, []byte{})
+}
+
+// IndexSearchByteValue searches the index given a value and returns an iterator over the associated recordIds.
+func IndexSearchByteValue(b *bolt.Bucket, value []byte) *IndexSearchIterator {
+	return newSearchIterator(b, serializationutil.BytesToKey(value))
+}
+
+type IndexSearchIterator struct {
+	c *bolt.Cursor
+	k []byte
+	prefix []byte
+}
+
+func newSearchIterator(b *bolt.Bucket, prefix []byte) *IndexSearchIterator {
+	c := b.Cursor()
+	k, _ := c.Seek(prefix)
+	return &IndexSearchIterator{
+		c: c,
+		k: k,
+		prefix: prefix,
+	}
+}
+
+func (i *IndexSearchIterator) Next() (int64, bool) {
+	if i.k == nil || !bytes.HasPrefix(i.k, i.prefix) {
+		return 0, false
+	}
+	id, err := serializationutil.Btoi(i.k[len(i.prefix):])
+	if err != nil {
+		// this sholud never happen, if it does it indicates database corruption.
+		return 0, false
+	}
+	i.k, _ = i.c.Next()
+	return id, true
+}
+
+func (i *IndexSearchIterator) ToSlice() []int64 {
+	var ids []int64
+	for id, ok := i.Next(); ok; id, ok = i.Next() {
+		ids = append(ids, id)
+	}
+	return ids
+}

internal/database/indexutil/indexutil_test.go

@@ -0,0 +1,45 @@
+package indexutil
+
+import (
+	"fmt"
+	"testing"
+
+	"go.etcd.io/bbolt"
+)
+
+func TestIndexing(t *testing.T) {
+	db, err := bbolt.Open(t.TempDir() + "/test.boltdb", 0600, nil)
+	if err != nil {
+		t.Fatalf("error opening database: %s", err)
+	}
+
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		b, err := tx.CreateBucket([]byte("test"))
+		if err != nil {
+			return fmt.Errorf("error creating bucket: %s", err)
+		}
+		for id := 0; id < 100; id += 1 {
+			if err := IndexByteValue(b, []byte("document"), int64(id)); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("db.Update error: %v", err)
+	}
+	
+	if err := db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte("test"))
+		ids := IndexSearchByteValue(b, []byte("document")).ToSlice()
+		if len(ids) != 100 {
+			t.Errorf("want 100 ids, got %d", len(ids))
+		}
+		ids = IndexSearchByteValue(b, []byte("other")).ToSlice()
+		if len(ids) != 0 {
+			t.Errorf("want 0 ids, got %d", len(ids))
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("db.View error: %v", err)
+	}
+}

internal/database/oplog/oplog.go

@@ -0,0 +1,386 @@
+package oplog
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	v1 "github.com/garethgeorge/resticui/gen/go/v1"
+	"github.com/garethgeorge/resticui/internal/database/indexutil"
+	"github.com/garethgeorge/resticui/internal/database/serializationutil"
+	bolt "go.etcd.io/bbolt"
+	"go.uber.org/zap"
+	"google.golang.org/protobuf/proto"
+)
+
+type EventType int
+
+const (
+	EventTypeUnknown   = EventType(iota)
+	EventTypeOpCreated = EventType(iota)
+	EventTypeOpUpdated = EventType(iota)
+)
+
+var (
+	SystemBucket              = []byte("oplog.system")            // system stores metadata
+	OpLogBucket               = []byte("oplog.log")               // oplog stores the operations themselves
+	RepoIndexBucket           = []byte("oplog.repo_idx")          // repo_index tracks IDs of operations affecting a given repo
+	PlanIndexBucket           = []byte("oplog.plan_idx")          // plan_index tracks IDs of operations affecting a given plan
+	IndexedSnapshotsSetBucket = []byte("oplog.indexed_snapshots") // indexed_snapshots is a set of snapshot IDs that have been indexed
+)
+
+// OpLog represents a log of operations performed.
+// Operations are indexed by repo and plan.
+type OpLog struct {
+	db *bolt.DB
+
+	subscribersMu sync.RWMutex
+	subscribers   []*func(EventType, *v1.Operation)
+}
+
+func NewOpLog(databasePath string) (*OpLog, error) {
+	if err := os.MkdirAll(path.Dir(databasePath), 0700); err != nil {
+		return nil, fmt.Errorf("error creating database directory: %s", err)
+	}
+
+	db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return nil, fmt.Errorf("error opening database: %s", err)
+	}
+
+	if err := db.Update(func(tx *bolt.Tx) error {
+		// Create the buckets if they don't exist
+		for _, bucket := range [][]byte{
+			SystemBucket, OpLogBucket, RepoIndexBucket, PlanIndexBucket, IndexedSnapshotsSetBucket,
+		} {
+			if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
+				return fmt.Errorf("creating bucket %s: %s", string(bucket), err)
+			}
+		}
+
+		// Validate the operation log on startup.
+		sysBucket := tx.Bucket(SystemBucket)
+		opLogBucket := tx.Bucket(OpLogBucket)
+		c := opLogBucket.Cursor()
+		if lastValidated := sysBucket.Get([]byte("last_validated")); lastValidated != nil {
+			c.Seek(lastValidated)
+		}
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			op := &v1.Operation{}
+			if err := proto.Unmarshal(v, op); err != nil {
+				zap.L().Error("error unmarshalling operation, there may be corruption in the oplog", zap.Error(err))
+				continue
+			}
+			if op.Status == v1.OperationStatus_STATUS_INPROGRESS {
+				op.Status = v1.OperationStatus_STATUS_ERROR
+				op.DisplayMessage = "Operation timeout."
+				bytes, err := proto.Marshal(op)
+				if err != nil {
+					return fmt.Errorf("marshalling operation: %w", err)
+				}
+				if err := opLogBucket.Put(k, bytes); err != nil {
+					return fmt.Errorf("putting operation into bucket: %w", err)
+				}
+			}
+		}
+		if lastValidated, _ := c.Last(); lastValidated != nil {
+			if err := sysBucket.Put([]byte("last_validated"), lastValidated); err != nil {
+				return fmt.Errorf("checkpointing last_validated key: %w", err)
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return &OpLog{db: db}, nil
+}
+
+func (o *OpLog) Close() error {
+	return o.db.Close()
+}
+
+// Add adds a generic operation to the operation log.
+func (o *OpLog) Add(op *v1.Operation) error {
+	if op.Id != 0 {
+		return errors.New("operation already has an ID, OpLog.Add is expected to set the ID")
+	}
+
+	err := o.db.Update(func(tx *bolt.Tx) error {
+		err := o.addOperationHelper(tx, op)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err == nil {
+		o.notifyHelper(EventTypeOpCreated, op)
+	}
+	return err
+}
+
+func (o *OpLog) BulkAdd(ops []*v1.Operation) error {
+	err := o.db.Update(func(tx *bolt.Tx) error {
+		for _, op := range ops {
+			if err := o.addOperationHelper(tx, op); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err == nil {
+		for _, op := range ops {
+			o.notifyHelper(EventTypeOpCreated, op)
+		}
+	}
+	return err
+}
+
+func (o *OpLog) addOperationHelper(tx *bolt.Tx, op *v1.Operation) error {
+	b := tx.Bucket(OpLogBucket)
+
+	id, err := b.NextSequence()
+	if err != nil {
+		return fmt.Errorf("error getting next sequence: %w", err)
+	}
+
+	op.Id = int64(id)
+
+	bytes, err := proto.Marshal(op)
+	if err != nil {
+		return fmt.Errorf("error marshalling operation: %w", err)
+	}
+
+	if err := b.Put(serializationutil.Itob(op.Id), bytes); err != nil {
+		return fmt.Errorf("error putting operation into bucket: %w", err)
+	}
+
+	// Update always universal indices
+	if op.RepoId != "" {
+		if err := indexutil.IndexByteValue(tx.Bucket(RepoIndexBucket), []byte(op.RepoId), op.Id); err != nil {
+			return fmt.Errorf("error adding operation to repo index: %w", err)
+		}
+	}
+	if op.PlanId != "" {
+		if err := indexutil.IndexByteValue(tx.Bucket(PlanIndexBucket), []byte(op.PlanId), op.Id); err != nil {
+			return fmt.Errorf("error adding operation to repo index: %w", err)
+		}
+	}
+
+	// Update operation type dependent indices.
+	switch wrappedOp := op.Op.(type) {
+	case *v1.Operation_OperationBackup:
+		// Nothing extra to be done.
+	case *v1.Operation_OperationIndexSnapshot:
+		if wrappedOp.OperationIndexSnapshot == nil || wrappedOp.OperationIndexSnapshot.Snapshot == nil {
+			return errors.New("op.OperationIndexSnapshot or op.OperationIndexSnapshot.Snapshot is nil")
+		}
+		snapshotId := serializationutil.NormalizeSnapshotId(wrappedOp.OperationIndexSnapshot.Snapshot.Id)
+		key := serializationutil.BytesToKey([]byte(snapshotId))
+		if err := tx.Bucket(IndexedSnapshotsSetBucket).Put(key, serializationutil.Itob(op.Id)); err != nil {
+			return fmt.Errorf("error adding OperationIndexSnapshot to indexed snapshots set: %w", err)
+		}
+	default:
+		return fmt.Errorf("unknown operation type: %T", wrappedOp)
+	}
+
+	return nil
+}
+
+func (o *OpLog) HasIndexedSnapshot(snapshotId string) (int64, error) {
+	var id int64
+	if err := o.db.View(func(tx *bolt.Tx) error {
+		snapshotId := serializationutil.NormalizeSnapshotId(snapshotId)
+		key := serializationutil.BytesToKey([]byte(snapshotId))
+		idBytes := tx.Bucket(IndexedSnapshotsSetBucket).Get(key)
+		if idBytes == nil {
+			id = -1
+		} else {
+			var err error
+			id, err = serializationutil.Btoi(idBytes)
+			if err != nil {
+				return fmt.Errorf("database corrupt, couldn't convert ID bytes to int: %w", err)
+			}
+		}
+		return nil
+	}); err != nil {
+		return 0, err
+	}
+	return id, nil
+}
+
+func (o *OpLog) Update(op *v1.Operation) error {
+	if op.Id == 0 {
+		return errors.New("operation does not have an ID, OpLog.Update expects operation with an ID")
+	}
+
+	err := o.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(OpLogBucket)
+
+		if b.Get(serializationutil.Itob(op.Id)) == nil {
+			return fmt.Errorf("operation with ID %d does not exist", op.Id)
+		}
+
+		bytes, err := proto.Marshal(op)
+		if err != nil {
+			return fmt.Errorf("error marshalling operation: %w", err)
+		}
+
+		if err := b.Put(serializationutil.Itob(op.Id), bytes); err != nil {
+			return fmt.Errorf("error putting operation into bucket: %w", err)
+		}
+
+		return nil
+	})
+	if err == nil {
+		o.notifyHelper(EventTypeOpUpdated, op)
+	}
+	return err
+}
+
+func (o *OpLog) notifyHelper(eventType EventType, op *v1.Operation) {
+	o.subscribersMu.RLock()
+	defer o.subscribersMu.RUnlock()
+	for _, sub := range o.subscribers {
+		(*sub)(eventType, op)
+	}
+}
+
+func (o *OpLog) getHelper(b *bolt.Bucket, id int64) (*v1.Operation, error) {
+	bytes := b.Get(serializationutil.Itob(id))
+	if bytes == nil {
+		return nil, fmt.Errorf("operation with ID %d does not exist", id)
+	}
+
+	var op v1.Operation
+	if err := proto.Unmarshal(bytes, &op); err != nil {
+		return nil, fmt.Errorf("error unmarshalling operation: %w", err)
+	}
+
+	return &op, nil
+}
+
+func (o *OpLog) Get(id int64) (*v1.Operation, error) {
+	var op *v1.Operation
+	if err := o.db.View(func(tx *bolt.Tx) error {
+		var err error
+		op, err = o.getHelper(tx.Bucket(OpLogBucket), id)
+		return err
+	}); err != nil {
+		return nil, err
+	}
+	return op, nil
+}
+
+func (o *OpLog) GetByRepo(repoId string, filter Filter) ([]*v1.Operation, error) {
+	var ops []*v1.Operation
+	if err := o.db.View(func(tx *bolt.Tx) error {
+		ids := indexutil.IndexSearchByteValue(tx.Bucket(RepoIndexBucket), []byte(repoId)).ToSlice()
+		ids = filter(ids)
+
+		b := tx.Bucket(OpLogBucket)
+		for _, id := range ids {
+			op, err := o.getHelper(b, id)
+			if err != nil {
+				return err
+			}
+			ops = append(ops, op)
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return ops, nil
+}
+
+func (o *OpLog) GetByPlan(planId string, filter Filter) ([]*v1.Operation, error) {
+	var ops []*v1.Operation
+	if err := o.db.View(func(tx *bolt.Tx) error {
+		ids := indexutil.IndexSearchByteValue(tx.Bucket(PlanIndexBucket), []byte(planId)).ToSlice()
+		ids = filter(ids)
+
+		b := tx.Bucket(OpLogBucket)
+		for _, id := range ids {
+			op, err := o.getHelper(b, id)
+			if err != nil {
+				return err
+			}
+			ops = append(ops, op)
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return ops, nil
+}
+
+func (o *OpLog) GetAll(filter Filter) ([]*v1.Operation, error) {
+	var ops []*v1.Operation
+	if err := o.db.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket(OpLogBucket).Cursor()
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			op := &v1.Operation{}
+			if err := proto.Unmarshal(v, op); err != nil {
+				return fmt.Errorf("error unmarshalling operation: %w", err)
+			}
+			ops = append(ops, op)
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return ops, nil
+}
+
+func (o *OpLog) Subscribe(callback *func(EventType, *v1.Operation)) {
+	o.subscribersMu.Lock()
+	defer o.subscribersMu.Unlock()
+	o.subscribers = append(o.subscribers, callback)
+}
+
+func (o *OpLog) Unsubscribe(callback *func(EventType, *v1.Operation)) {
+	o.subscribersMu.Lock()
+	defer o.subscribersMu.Unlock()
+	subs := o.subscribers
+	for i, c := range subs {
+		if c == callback {
+			subs[i] = subs[len(subs)-1]
+			o.subscribers = subs[:len(o.subscribers)-1]
+		}
+	}
+}
+
+type Filter func([]int64) []int64
+
+func FilterKeepAll() Filter {
+	return func(ids []int64) []int64 {
+		return ids
+	}
+}
+
+func FilterLastN(n int64) Filter {
+	return func(ids []int64) []int64 {
+		if len(ids) > int(n) {
+			ids = ids[len(ids)-int(n):]
+		}
+		return ids
+	}
+}
+
+func FilterLimitOffset(limit, offset int64) Filter {
+	return func(ids []int64) []int64 {
+		if len(ids) > int(offset) {
+			ids = ids[offset:]
+		}
+		if len(ids) > int(limit) {
+			ids = ids[:limit]
+		}
+		return ids
+	}
+}

internal/database/oplog/oplog_test.go

@@ -7,56 +7,6 @@ import (
 	v1 "github.com/garethgeorge/resticui/gen/go/v1"
 )
 
-type MyStruct struct {
-	ID    string `storm:"id"`
-	Field string `storm:"index"`
-}
-
-func TestDatabase(t *testing.T) {
-	log, err := NewOpLog(t.TempDir() + "/test.boltdb")
-	if err != nil {
-		t.Fatalf("error creating oplog: %s", err)
-	}
-
-	t.Run("can store struct", func(t *testing.T) {
-		log.sdb.From("struct").Save(&MyStruct{ID: "1", Field: "test"})
-
-		var s MyStruct
-		if err := log.sdb.From("struct").One("Field", "test", &s); err != nil {
-			t.Fatalf("error getting struct: %s", err)
-		}
-		t.Logf("Got struct: %+v", s)
-		if s.ID != "1" {
-			t.Errorf("want ID 1, got %s", s.ID)
-		}
-		if s.Field != "test" {
-			t.Errorf("want field test, got %s", s.Field)
-		}
-	})
-
-	t.Run("can store proto", func(t *testing.T) {
-		log.sdb.From("proto").Save(&v1.Operation{Id: 1, PlanId: "foo", RepoId: "bar"})
-
-		var op v1.Operation
-		if err := log.sdb.From("proto").One("PlanId", "foo", &op); err != nil {
-			t.Fatalf("error getting operation: %s", err)
-		}
-		t.Logf("Got operation: %+v", op)
-		if op.Id != 1 {
-			t.Errorf("want ID 1, got %d", op.Id)
-		}
-
-		var ops []v1.Operation
-		if err := log.sdb.From("proto").Find("RepoId", "bar", &ops); err != nil {
-			t.Fatalf("error getting operations: %s", err)
-		}
-		t.Logf("Got operations: %+v", ops)
-		if len(ops) != 1 {
-			t.Errorf("want 1 operation, got %d", len(ops))
-		}
-	})
-}
-
 func TestCreate(t *testing.T) {
 	// t.Parallel()
 	log, err := NewOpLog(t.TempDir() + "/test.boltdb")
@@ -82,10 +32,18 @@ func TestAddOperation(t *testing.T) {
 		op      *v1.Operation
 		wantErr bool
 	}{
+		{
+			name: "no operation",
+			op: &v1.Operation{
+				Id: 0,
+			},
+			wantErr: true,
+		},
 		{
 			name: "basic backup operation",
 			op: &v1.Operation{
 				Id: 0,
+				Op: &v1.Operation_OperationBackup{},
 			},
 			wantErr: false,
 		},
@@ -103,10 +61,21 @@ func TestAddOperation(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "basic snapshot operation with no snapshot",
+			op: &v1.Operation{
+				Id: 0,
+				Op: &v1.Operation_OperationIndexSnapshot{
+					OperationIndexSnapshot: &v1.OperationIndexSnapshot{},
+				},
+			},
+			wantErr: true,
+		},
 		{
 			name: "operation with ID",
 			op: &v1.Operation{
 				Id: 1,
+				Op: &v1.Operation_OperationBackup{},
 			},
 			wantErr: true,
 		},
@@ -115,6 +84,7 @@ func TestAddOperation(t *testing.T) {
 			op: &v1.Operation{
 				Id:     0,
 				RepoId: "testrepo",
+				Op:     &v1.Operation_OperationBackup{},
 			},
 		},
 		{
@@ -122,6 +92,7 @@ func TestAddOperation(t *testing.T) {
 			op: &v1.Operation{
 				Id:     0,
 				PlanId: "testplan",
+				Op:     &v1.Operation_OperationBackup{},
 			},
 		},
 	}
@@ -135,20 +106,13 @@ func TestAddOperation(t *testing.T) {
 				if tc.op.Id == 0 {
 					t.Errorf("Add() did not set op ID")
 				}
-
-				got, err := log.Get(tc.op.Id)
-				if err != nil {
-					t.Errorf("Get() error = %v", err)
-				}
-				if got.Id != tc.op.Id {
-					t.Errorf("Get() got = %+v, want %+v", got, tc.op)
-				}
 			}
 		})
 	}
 }
 
 func TestListOperation(t *testing.T) {
+	// t.Parallel()
 	log, err := NewOpLog(t.TempDir() + "/test.boltdb")
 	if err != nil {
 		t.Fatalf("error creating oplog: %s", err)
@@ -161,16 +125,19 @@ func TestListOperation(t *testing.T) {
 			PlanId:         "plan1",
 			RepoId:         "repo1",
 			DisplayMessage: "op1",
+			Op:             &v1.Operation_OperationBackup{},
 		},
 		{
 			PlanId:         "plan1",
 			RepoId:         "repo2",
 			DisplayMessage: "op2",
+			Op:             &v1.Operation_OperationBackup{},
 		},
 		{
 			PlanId:         "plan2",
 			RepoId:         "repo2",
 			DisplayMessage: "op3",
+			Op:             &v1.Operation_OperationBackup{},
 		},
 	}
 
@@ -216,7 +183,7 @@ func TestListOperation(t *testing.T) {
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
+			// t.Parallel()
 			var ops []*v1.Operation
 			var err error
 			if tc.byPlan {
@@ -238,7 +205,7 @@ func TestListOperation(t *testing.T) {
 }
 
 func TestBigIO(t *testing.T) {
-	t.Skip()
+	// t.Parallel()
 	log, err := NewOpLog(t.TempDir() + "/test.boltdb")
 	if err != nil {
 		t.Fatalf("error creating oplog: %s", err)
@@ -281,9 +248,8 @@ func TestIndexSnapshot(t *testing.T) {
 	t.Cleanup(func() { log.Close() })
 
 	op := &v1.Operation{
-		PlanId:     "plan1",
-		RepoId:     "repo1",
-		SnapshotId: NormalizeSnapshotId("abcdefghijklmnop"),
+		PlanId: "plan1",
+		RepoId: "repo1",
 		Op: &v1.Operation_OperationIndexSnapshot{
 			OperationIndexSnapshot: &v1.OperationIndexSnapshot{
 				Snapshot: &v1.ResticSnapshot{
@@ -296,30 +262,27 @@ func TestIndexSnapshot(t *testing.T) {
 		t.Fatalf("error adding operation: %s", err)
 	}
 
-	ops, err := log.GetBySnapshotId("abcdefgh")
+	id, err := log.HasIndexedSnapshot("abcdefgh")
 	if err != nil {
 		t.Fatalf("error checking for snapshot: %s", err)
 	}
-	if len(ops) != 1 {
-		t.Fatalf("want 1 operation, got %d", len(ops))
-	}
-	if ops[0].Id != op.Id {
-		t.Fatalf("want id %d, got %d", op.Id, ops[0].Id)
+	if id != op.Id {
+		t.Fatalf("want id %d, got %d", op.Id, id)
 	}
 
-	ops, err = log.GetBySnapshotId("notfound")
+	id, err = log.HasIndexedSnapshot("notfound")
 	if err != nil {
 		t.Fatalf("error checking for snapshot: %s", err)
 	}
-	if len(ops) != 0 {
-		t.Fatalf("want 0 operations, got %d", len(ops))
+	if id != -1 {
+		t.Fatalf("want id -1, got %d", id)
 	}
 }
 
 func collectMessages(ops []*v1.Operation) []string {
 	var messages []string
-	for idx, _ := range ops {
-		messages = append(messages, ops[idx].DisplayMessage)
+	for _, op := range ops {
+		messages = append(messages, op.DisplayMessage)
 	}
 	return messages
 }

internal/database/serializationutil/serializationutil.go

@@ -0,0 +1,53 @@
+package serializationutil
+
+import (
+	"encoding/binary"
+	"errors"
+)
+
+var ErrInvalidLength = errors.New("invalid length")
+
+func Itob(v int64) []byte {
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, uint64(v))
+	return b
+}
+
+func Btoi(b []byte) (int64, error) {
+	if len(b) != 8 {
+		return 0, ErrInvalidLength
+	}
+	return int64(binary.BigEndian.Uint64(b)), nil
+}
+
+func Stob(v string) []byte {
+	var b []byte
+	b = append(b, Itob(int64(len(v)))...)
+	b = append(b, []byte(v)...)
+	return b
+}
+
+func Btos(b []byte) (string, int64, error) {
+	if len(b) < 8 {
+		return "", 0, ErrInvalidLength
+	}
+	length, _ := Btoi(b[:8])
+	if int64(len(b)) < 8+length {
+		return "", 0, ErrInvalidLength
+	}
+	return string(b[8:8+length]), 8+length, nil
+}
+
+func BytesToKey(b []byte) []byte {
+	var key []byte
+	key = append(key, Itob(int64(len(b)))...)
+	key = append(key, b...)
+	return key
+}
+
+func NormalizeSnapshotId(id string) string {
+	if len(id) < 8 {
+		return id
+	}
+	return id[:8]
+}

internal/database/serializationutil/serializationutil_test.go

@@ -0,0 +1,23 @@
+package serializationutil
+
+import "testing"
+
+func TestItoa(t *testing.T) {
+	nums := []int64{0, 1, 2, 3, 4, 1 << 32, int64(1) << 62}
+	for _, num := range nums {
+		b := Itob(num)
+		if v, _ := Btoi(b); v != num {
+			t.Errorf("itob/btoi failed for %d", num)
+		}
+	}
+}
+
+func TestStob(t *testing.T) {
+	strs := []string{"", "a", "ab", "abc", "abcd", "abcde", "abcdef"}
+	for _, str := range strs {
+		b := Stob(str)
+		if val, _, _ := Btos(b); val != str {
+			t.Errorf("stob/btos failed for %s", str)
+		}
+	}
+}

internal/oplog/oplog.go

@@ -1,244 +0,0 @@
-package oplog
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"path"
-	"sync"
-	"time"
-
-	storm "github.com/asdine/storm/v3"
-	"github.com/asdine/storm/v3/codec/protobuf"
-	"github.com/asdine/storm/v3/index"
-	v1 "github.com/garethgeorge/resticui/gen/go/v1"
-	bolt "go.etcd.io/bbolt"
-	"go.uber.org/zap"
-)
-
-type EventType int
-
-const (
-	EventTypeUnknown   = EventType(iota)
-	EventTypeOpCreated = EventType(iota)
-	EventTypeOpUpdated = EventType(iota)
-)
-
-const (
-	OpLogSchemaVersion = 1
-)
-
-var (
-	OpLogSysBucket   = "oplog.sys"
-	OpLogBucket      = "oplog.log"
-	schemaVersionKey = "schema_version"
-)
-
-// OpLog represents a log of operations performed.
-// Operations are indexed by repo and plan.
-type OpLog struct {
-	db  *bolt.DB
-	sdb *storm.DB
-
-	subscribersMu sync.RWMutex
-	subscribers   []*func(EventType, *v1.Operation)
-}
-
-func NewOpLog(databasePath string) (*OpLog, error) {
-	if err := os.MkdirAll(path.Dir(databasePath), 0700); err != nil {
-		return nil, fmt.Errorf("creating database directory: %s", err)
-	}
-
-	db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
-	if err != nil {
-		return nil, fmt.Errorf("opening database: %s", err)
-	}
-
-	sdb, err := storm.Open("", storm.UseDB(db), storm.Codec(protobuf.Codec))
-	if err != nil {
-		return nil, fmt.Errorf("opening storm: %s", err)
-	}
-
-	var schemaVersion int
-	if err := sdb.Get(OpLogSysBucket, schemaVersionKey, &schemaVersion); err != nil {
-		if errors.Is(err, storm.ErrNotFound) {
-			schemaVersion = -1
-		} else {
-			return nil, fmt.Errorf("getting schema version: %s", err)
-		}
-	}
-	if schemaVersion != OpLogSchemaVersion {
-		zap.L().Info("oplog schema version mismatch, reindexing")
-		sdb.From(OpLogBucket).ReIndex(&v1.Operation{})
-		if err := sdb.Set(OpLogSysBucket, schemaVersionKey, OpLogSchemaVersion); err != nil {
-			return nil, fmt.Errorf("setting schema version: %s", err)
-		}
-	}
-
-	return &OpLog{
-		db:  db,
-		sdb: sdb,
-	}, nil
-}
-
-func (o *OpLog) Close() error {
-	return o.db.Close()
-}
-
-// Add adds a generic operation to the operation log.
-func (o *OpLog) Add(op *v1.Operation) error {
-	if op.Id != 0 {
-		return errors.New("operation already has an ID, OpLog.Add is expected to set the ID")
-	}
-	op.SnapshotId = NormalizeSnapshotId(op.SnapshotId)
-
-	if err := o.sdb.From(OpLogBucket).Save(op); err != nil {
-		return fmt.Errorf("saving operation: %w", err)
-	}
-
-	o.notifyHelper(EventTypeOpCreated, op)
-	return nil
-}
-
-func (o *OpLog) BulkAdd(ops []*v1.Operation) error {
-	tx, err := o.sdb.From(OpLogBucket).Begin(true)
-	if err != nil {
-		return fmt.Errorf("beginning transaction: %w", err)
-	}
-	defer tx.Rollback()
-
-	for _, op := range ops {
-		if op.Id != 0 {
-			return errors.New("operation already has an ID, OpLog.BulkAdd is expected to set the ID")
-		}
-
-		if err := tx.Save(op); err != nil {
-			return fmt.Errorf("saving operation: %w", err)
-		}
-	}
-
-	if err := tx.Commit(); err != nil {
-		return fmt.Errorf("committing transaction: %w", err)
-	}
-
-	for _, op := range ops {
-		o.notifyHelper(EventTypeOpCreated, op)
-	}
-
-	return nil
-}
-
-func (o *OpLog) Update(op *v1.Operation) error {
-	if err := o.sdb.From(OpLogBucket).Update(op); err != nil {
-		return fmt.Errorf("updating operation %v: %w", op.Id, err)
-	}
-	o.notifyHelper(EventTypeOpUpdated, op)
-	return nil
-}
-
-func (o *OpLog) notifyHelper(eventType EventType, op *v1.Operation) {
-	o.subscribersMu.RLock()
-	defer o.subscribersMu.RUnlock()
-	for _, sub := range o.subscribers {
-		(*sub)(eventType, op)
-	}
-}
-
-func (o *OpLog) Get(id int64) (*v1.Operation, error) {
-	var op v1.Operation
-	if err := o.sdb.From(OpLogBucket).One("Id", id, &op); err != nil {
-		return nil, fmt.Errorf("getting operation: %w", err)
-	}
-	return &op, nil
-}
-
-func (o *OpLog) GetByRepo(repoId string, filter Filter) ([]*v1.Operation, error) {
-	var ops []v1.Operation
-	if err := o.sdb.From(OpLogBucket).Find("RepoId", repoId, &ops); err != nil {
-		if errors.Is(err, storm.ErrNotFound) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("getting operations by repo %q: %w", repoId, err)
-	}
-	return opsToRefs(ops), nil
-}
-
-func (o *OpLog) GetByPlan(planId string, filter Filter) ([]*v1.Operation, error) {
-	var ops []v1.Operation
-	if err := o.sdb.From(OpLogBucket).Find("PlanId", planId, &ops, filter); err != nil {
-		if errors.Is(err, storm.ErrNotFound) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("getting operations by plan %q: %w", planId, err)
-	}
-	return opsToRefs(ops), nil
-}
-
-func (o *OpLog) GetBySnapshotId(snapshotId string) ([]*v1.Operation, error) {
-	snapshotId = NormalizeSnapshotId(snapshotId)
-	var ops []v1.Operation
-	if err := o.sdb.From(OpLogBucket).Find("SnapshotId", snapshotId, &ops); err != nil {
-		if errors.Is(err, storm.ErrNotFound) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("getting operations by snapshot %q: %w", snapshotId, err)
-	}
-	return opsToRefs(ops), nil
-}
-func (o *OpLog) GetAll(filter Filter) ([]*v1.Operation, error) {
-	ops := []v1.Operation{}
-	if err := o.sdb.From(OpLogBucket).All(ops); err != nil {
-		return nil, fmt.Errorf("getting all operations: %w", err)
-	}
-	return opsToRefs(ops), nil
-}
-
-func (o *OpLog) Subscribe(callback *func(EventType, *v1.Operation)) {
-	o.subscribersMu.Lock()
-	defer o.subscribersMu.Unlock()
-	o.subscribers = append(o.subscribers, callback)
-}
-
-func (o *OpLog) Unsubscribe(callback *func(EventType, *v1.Operation)) {
-	o.subscribersMu.Lock()
-	defer o.subscribersMu.Unlock()
-	subs := o.subscribers
-	for i, c := range subs {
-		if c == callback {
-			subs[i] = subs[len(subs)-1]
-			o.subscribers = subs[:len(o.subscribers)-1]
-		}
-	}
-}
-
-type Filter func(*index.Options)
-
-func FilterKeepAll() Filter {
-	return func(*index.Options) {}
-}
-
-func FilterLastN(n int64) Filter {
-	return storm.Limit(int(n))
-}
-
-func FilterLimitOffset(limit, offset int64) Filter {
-	return func(opts *index.Options) {
-		storm.Skip(int(offset))(opts)
-		storm.Limit(int(limit))(opts)
-	}
-}
-
-func NormalizeSnapshotId(id string) string {
-	if len(id) <= 8 {
-		return id
-	}
-	return id[:8]
-}
-
-func opsToRefs(ops []v1.Operation) []*v1.Operation {
-	refs := make([]*v1.Operation, len(ops))
-	for i := range ops {
-		refs[i] = &ops[i]
-	}
-	return refs
-}

internal/orchestrator/orchestrator.go

@@ -9,7 +9,7 @@ import (
 
 	v1 "github.com/garethgeorge/resticui/gen/go/v1"
 	"github.com/garethgeorge/resticui/internal/config"
-	"github.com/garethgeorge/resticui/internal/oplog"
+	"github.com/garethgeorge/resticui/internal/database/oplog"
 	"github.com/garethgeorge/resticui/pkg/restic"
 	"go.uber.org/zap"
 	"google.golang.org/protobuf/proto"
@@ -21,13 +21,14 @@ var ErrPlanNotFound = errors.New("plan not found")
 
 // Orchestrator is responsible for managing repos and backups.
 type Orchestrator struct {
-	mu       sync.Mutex
-	config   *v1.Config
-	oplog    *oplog.OpLog
+	mu sync.Mutex
+	config *v1.Config
+	oplog *oplog.OpLog
 	repoPool *resticRepoPool
 
+	
 	configUpdates chan *v1.Config // configUpdates chan makes config changes available to Run()
-	externTasks   chan Task       // externTasks is a channel that externally added tasks can be added to, they will be consumed by Run()
+	externTasks chan Task // externTasks is a channel that externally added tasks can be added to, they will be consumed by Run()
 }
 
 func NewOrchestrator(configProvider config.ConfigStore, oplog *oplog.OpLog) (*Orchestrator, error) {
@@ -37,9 +38,9 @@ func NewOrchestrator(configProvider config.ConfigStore, oplog *oplog.OpLog) (*Or
 	}
 
 	return &Orchestrator{
-		config:      cfg,
-		oplog:       oplog,
-		repoPool:    newResticRepoPool(&config.MemoryStore{Config: cfg}),
+		config: cfg,
+		oplog: oplog,
+		repoPool: newResticRepoPool(&config.MemoryStore{Config: cfg}),
 		externTasks: make(chan Task, 2),
 	}, nil
 }
@@ -50,7 +51,7 @@ func (o *Orchestrator) ApplyConfig(cfg *v1.Config) error {
 	o.config = cfg
 
 	zap.L().Debug("Applying config to orchestrator", zap.Any("config", cfg))
-
+	
 	// Update the config provided to the repo pool.
 	if err := o.repoPool.configProvider.Update(cfg); err != nil {
 		return fmt.Errorf("failed to update repo pool config: %w", err)
@@ -69,7 +70,7 @@ func (o *Orchestrator) GetRepo(repoId string) (repo *RepoOrchestrator, err error
 	defer o.mu.Unlock()
 
 	r, err := o.repoPool.GetRepo(repoId)
-	if err != nil {
+	if  err != nil {
 		return nil, fmt.Errorf("failed to get repo %q: %w", repoId, err)
 	}
 	return r, nil
@@ -118,7 +119,7 @@ func (o *Orchestrator) Run(mainCtx context.Context) error {
 func (o *Orchestrator) runVersion(mainCtx context.Context, config *v1.Config) bool {
 	var lock sync.Mutex
 	ctx, cancel := context.WithCancel(mainCtx)
-
+	
 	var wg sync.WaitGroup
 
 	var execTask func(t Task)
@@ -148,7 +149,7 @@ func (o *Orchestrator) runVersion(mainCtx context.Context, config *v1.Config) bo
 				lock.Lock()
 				defer lock.Unlock()
 				zap.L().Info("running task", zap.String("task", t.Name()))
-
+				
 				// Task execution runs with mainCtx meaning config changes do not interrupt it, but cancelling the orchestration loop will.
 				if err := t.Run(mainCtx); err != nil {
 					zap.L().Error("task failed", zap.String("task", t.Name()), zap.Error(err))
@@ -158,7 +159,7 @@ func (o *Orchestrator) runVersion(mainCtx context.Context, config *v1.Config) bo
 
 				if ctx.Err() != nil {
 					zap.L().Debug("not attempting to reschedule task, orchestrator context is cancelled.", zap.String("task", t.Name()))
-					return
+					return 
 				}
 
 				execTask(t)
@@ -201,14 +202,15 @@ func (o *Orchestrator) EnqueueTask(t Task) {
 
 // resticRepoPool caches restic repos.
 type resticRepoPool struct {
-	mu             sync.Mutex
-	repos          map[string]*RepoOrchestrator
+	mu sync.Mutex
+	repos map[string]*RepoOrchestrator
 	configProvider config.ConfigStore
 }
 
+
 func newResticRepoPool(configProvider config.ConfigStore) *resticRepoPool {
 	return &resticRepoPool{
-		repos:          make(map[string]*RepoOrchestrator),
+		repos: make(map[string]*RepoOrchestrator),
 		configProvider: configProvider,
 	}
 }
@@ -242,7 +244,7 @@ func (rp *resticRepoPool) GetRepo(repoId string) (repo *RepoOrchestrator, err er
 	if ok && proto.Equal(repo.repoConfig, repoProto) {
 		return repo, nil
 	}
-	delete(rp.repos, repoId)
+	delete(rp.repos, repoId);
 
 	var opts []restic.GenericOption
 	opts = append(opts, restic.WithPropagatedEnvVars(restic.EnvToPropagate...))

internal/orchestrator/tasks.go

@@ -6,15 +6,16 @@ import (
 	"time"
 
 	v1 "github.com/garethgeorge/resticui/gen/go/v1"
-	"github.com/garethgeorge/resticui/internal/oplog"
+	"github.com/garethgeorge/resticui/internal/database/oplog"
 	"github.com/garethgeorge/resticui/pkg/restic"
 	"github.com/gitploy-io/cronexpr"
 	"github.com/hashicorp/go-multierror"
 	"go.uber.org/zap"
 )
 
+
 type Task interface {
-	Name() string                  // huamn readable name for this task.
+	Name() string // huamn readable name for this task.
 	Next(now time.Time) *time.Time // when this task would like to be run.
 	Run(ctx context.Context) error // run the task.
 }
@@ -22,8 +23,8 @@ type Task interface {
 // BackupTask is a scheduled backup operation.
 type ScheduledBackupTask struct {
 	orchestrator *Orchestrator // owning orchestrator
-	plan         *v1.Plan
-	schedule     *cronexpr.Schedule
+	plan *v1.Plan
+	schedule *cronexpr.Schedule 
 }
 
 var _ Task = &ScheduledBackupTask{}
@@ -36,8 +37,8 @@ func NewScheduledBackupTask(orchestrator *Orchestrator, plan *v1.Plan) (*Schedul
 
 	return &ScheduledBackupTask{
 		orchestrator: orchestrator,
-		plan:         plan,
-		schedule:     sched,
+		plan: plan,
+		schedule: sched,
 	}, nil
 }
 
@@ -57,15 +58,15 @@ func (t *ScheduledBackupTask) Run(ctx context.Context) error {
 // OnetimeBackupTask is a single backup operation.
 type OnetimeBackupTask struct {
 	orchestrator *Orchestrator
-	plan         *v1.Plan
-	time         *time.Time
+	plan *v1.Plan
+	time *time.Time
 }
 
 func NewOneofBackupTask(orchestrator *Orchestrator, plan *v1.Plan, at time.Time) *OnetimeBackupTask {
 	return &OnetimeBackupTask{
 		orchestrator: orchestrator,
-		plan:         plan,
-		time:         &at,
+		plan: plan,
+		time: &at,
 	}
 }
 
@@ -74,9 +75,9 @@ func (t *OnetimeBackupTask) Name() string {
 }
 
 func (t *OnetimeBackupTask) Next(now time.Time) *time.Time {
-	ret := t.time
+	ret := t.time 
 	t.time = nil
-	return ret
+	return ret 
 }
 
 func (t *OnetimeBackupTask) Run(ctx context.Context) error {
@@ -90,11 +91,11 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan
 	}
 
 	op := &v1.Operation{
-		PlanId:          plan.Id,
-		RepoId:          plan.Repo,
+		PlanId: plan.Id,
+		RepoId: plan.Repo,
 		UnixTimeStartMs: curTimeMillis(),
-		Status:          v1.OperationStatus_STATUS_INPROGRESS,
-		Op:              backupOp,
+		Status: v1.OperationStatus_STATUS_INPROGRESS,
+		Op: backupOp,
 	}
 
 	startTime := time.Now()
@@ -108,7 +109,7 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan
 
 		lastSent := time.Now() // debounce progress updates, these can endup being very frequent.
 		summary, err := repo.Backup(ctx, plan, func(entry *restic.BackupProgressEntry) {
-			if time.Since(lastSent) < 200*time.Millisecond {
+			if time.Since(lastSent) < 200 * time.Millisecond {
 				return
 			}
 			lastSent = time.Now()
@@ -123,9 +124,10 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan
 			return fmt.Errorf("repo.Backup for repo %q: %w", plan.Repo, err)
 		}
 
-		// Set final properties (i.e. SnapshotId and LastStatus)
-		op.SnapshotId = oplog.NormalizeSnapshotId(summary.SnapshotId)
 		backupOp.OperationBackup.LastStatus = summary.ToProto()
+		if err := orchestrator.oplog.Update(op); err != nil {
+			return fmt.Errorf("update oplog with summary for backup: %v", err)
+		}
 
 		zap.L().Info("backup complete", zap.String("plan", plan.Id), zap.Duration("duration", time.Since(startTime)))
 		return nil
@@ -158,32 +160,22 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, plan
 	opTime := curTimeMillis()
 	var indexOps []*v1.Operation
 	for _, snapshot := range snapshots {
-		ops, err := orchestrator.oplog.GetBySnapshotId(snapshot.Id)
+		opid, err := orchestrator.oplog.HasIndexedSnapshot(snapshot.Id)
 		if err != nil {
 			return fmt.Errorf("HasIndexSnapshot for snapshot %q: %w", snapshot.Id, err)
 		}
 
-		// check if an indexing operation exists for this snapshot
-		indexOpFound := false
-		for _, op := range ops {
-			if _, ok := op.Op.(*v1.Operation_OperationIndexSnapshot); ok {
-				indexOpFound = true
-				break
-			}
-		}
-		if indexOpFound {
+		if opid >= 0 {
 			alreadyIndexed += 1
 			continue
 		}
-
-		// otherwise add one.
+		
 		indexOps = append(indexOps, &v1.Operation{
-			RepoId:          plan.Repo,
-			PlanId:          plan.Id,
-			SnapshotId:      oplog.NormalizeSnapshotId(snapshot.Id),
+			RepoId: plan.Repo,
+			PlanId: plan.Id,
 			UnixTimeStartMs: opTime,
-			UnixTimeEndMs:   opTime,
-			Status:          v1.OperationStatus_STATUS_SUCCESS,
+			UnixTimeEndMs: opTime,
+			Status: v1.OperationStatus_STATUS_SUCCESS,
 			Op: &v1.Operation_OperationIndexSnapshot{
 				OperationIndexSnapshot: &v1.OperationIndexSnapshot{
 					Snapshot: snapshot.ToProto(),
@@ -196,11 +188,11 @@ func indexSnapshotsHelper(ctx context.Context, orchestrator *Orchestrator, plan
 		return fmt.Errorf("BulkAdd snapshot operations: %w", err)
 	}
 
-	zap.L().Debug("Indexed snapshots",
-		zap.String("plan", plan.Id),
-		zap.Duration("duration", time.Since(startTime)),
-		zap.Int("alreadyIndexed", alreadyIndexed),
-		zap.Int("newlyAdded", len(snapshots)-alreadyIndexed),
+	zap.L().Debug("Indexed snapshots", 
+		zap.String("plan", plan.Id), 
+		zap.Duration("duration", time.Since(startTime)), 
+		zap.Int("alreadyIndexed", alreadyIndexed), 
+		zap.Int("newlyAdded", len(snapshots) - alreadyIndexed),
 	)
 
 	return err
@@ -217,7 +209,7 @@ func WithOperation(oplog *oplog.OpLog, op *v1.Operation, do func() error) error
 	}
 	err := do()
 	if err != nil {
-		op.Status = v1.OperationStatus_STATUS_ERROR
+		op.Status = v1.OperationStatus_STATUS_ERROR 
 		op.DisplayMessage = err.Error()
 	}
 	op.UnixTimeEndMs = curTimeMillis()
@@ -232,5 +224,5 @@ func WithOperation(oplog *oplog.OpLog, op *v1.Operation, do func() error) error
 
 func curTimeMillis() int64 {
 	t := time.Now()
-	return t.Unix()*1000 + int64(t.Nanosecond()/1000000)
-}
+	return t.Unix() * 1000 + int64(t.Nanosecond() / 1000000)
+}