Revert "chore: use storm for indexing on top boltdb"

This reverts commit 90e7b0c593.

internal/database/indexutil/indexutil.go

@@ -0,0 +1,57 @@
+package indexutil
+
+import (
+	"bytes"
+
+	"github.com/garethgeorge/resticui/internal/database/serializationutil"
+	bolt "go.etcd.io/bbolt"
+)
+
+// IndexByteValue indexes a value and recordId tuple creating multimap from value to lists of associated recordIds.
+func IndexByteValue(b *bolt.Bucket, value []byte, recordId int64) error {
+	key := serializationutil.BytesToKey(value)
+	key = append(key, serializationutil.Itob(recordId)...)
+	return b.Put(key, []byte{})
+}
+
+// IndexSearchByteValue searches the index given a value and returns an iterator over the associated recordIds.
+func IndexSearchByteValue(b *bolt.Bucket, value []byte) *IndexSearchIterator {
+	return newSearchIterator(b, serializationutil.BytesToKey(value))
+}
+
+type IndexSearchIterator struct {
+	c *bolt.Cursor
+	k []byte
+	prefix []byte
+}
+
+func newSearchIterator(b *bolt.Bucket, prefix []byte) *IndexSearchIterator {
+	c := b.Cursor()
+	k, _ := c.Seek(prefix)
+	return &IndexSearchIterator{
+		c: c,
+		k: k,
+		prefix: prefix,
+	}
+}
+
+func (i *IndexSearchIterator) Next() (int64, bool) {
+	if i.k == nil || !bytes.HasPrefix(i.k, i.prefix) {
+		return 0, false
+	}
+	id, err := serializationutil.Btoi(i.k[len(i.prefix):])
+	if err != nil {
+		// this sholud never happen, if it does it indicates database corruption.
+		return 0, false
+	}
+	i.k, _ = i.c.Next()
+	return id, true
+}
+
+func (i *IndexSearchIterator) ToSlice() []int64 {
+	var ids []int64
+	for id, ok := i.Next(); ok; id, ok = i.Next() {
+		ids = append(ids, id)
+	}
+	return ids
+}

internal/database/indexutil/indexutil_test.go

@@ -0,0 +1,45 @@
+package indexutil
+
+import (
+	"fmt"
+	"testing"
+
+	"go.etcd.io/bbolt"
+)
+
+func TestIndexing(t *testing.T) {
+	db, err := bbolt.Open(t.TempDir() + "/test.boltdb", 0600, nil)
+	if err != nil {
+		t.Fatalf("error opening database: %s", err)
+	}
+
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		b, err := tx.CreateBucket([]byte("test"))
+		if err != nil {
+			return fmt.Errorf("error creating bucket: %s", err)
+		}
+		for id := 0; id < 100; id += 1 {
+			if err := IndexByteValue(b, []byte("document"), int64(id)); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("db.Update error: %v", err)
+	}
+	
+	if err := db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte("test"))
+		ids := IndexSearchByteValue(b, []byte("document")).ToSlice()
+		if len(ids) != 100 {
+			t.Errorf("want 100 ids, got %d", len(ids))
+		}
+		ids = IndexSearchByteValue(b, []byte("other")).ToSlice()
+		if len(ids) != 0 {
+			t.Errorf("want 0 ids, got %d", len(ids))
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("db.View error: %v", err)
+	}
+}

internal/database/oplog/oplog.go

@@ -0,0 +1,386 @@
+package oplog
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	v1 "github.com/garethgeorge/resticui/gen/go/v1"
+	"github.com/garethgeorge/resticui/internal/database/indexutil"
+	"github.com/garethgeorge/resticui/internal/database/serializationutil"
+	bolt "go.etcd.io/bbolt"
+	"go.uber.org/zap"
+	"google.golang.org/protobuf/proto"
+)
+
+type EventType int
+
+const (
+	EventTypeUnknown   = EventType(iota)
+	EventTypeOpCreated = EventType(iota)
+	EventTypeOpUpdated = EventType(iota)
+)
+
+var (
+	SystemBucket              = []byte("oplog.system")            // system stores metadata
+	OpLogBucket               = []byte("oplog.log")               // oplog stores the operations themselves
+	RepoIndexBucket           = []byte("oplog.repo_idx")          // repo_index tracks IDs of operations affecting a given repo
+	PlanIndexBucket           = []byte("oplog.plan_idx")          // plan_index tracks IDs of operations affecting a given plan
+	IndexedSnapshotsSetBucket = []byte("oplog.indexed_snapshots") // indexed_snapshots is a set of snapshot IDs that have been indexed
+)
+
+// OpLog represents a log of operations performed.
+// Operations are indexed by repo and plan.
+type OpLog struct {
+	db *bolt.DB
+
+	subscribersMu sync.RWMutex
+	subscribers   []*func(EventType, *v1.Operation)
+}
+
+func NewOpLog(databasePath string) (*OpLog, error) {
+	if err := os.MkdirAll(path.Dir(databasePath), 0700); err != nil {
+		return nil, fmt.Errorf("error creating database directory: %s", err)
+	}
+
+	db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return nil, fmt.Errorf("error opening database: %s", err)
+	}
+
+	if err := db.Update(func(tx *bolt.Tx) error {
+		// Create the buckets if they don't exist
+		for _, bucket := range [][]byte{
+			SystemBucket, OpLogBucket, RepoIndexBucket, PlanIndexBucket, IndexedSnapshotsSetBucket,
+		} {
+			if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
+				return fmt.Errorf("creating bucket %s: %s", string(bucket), err)
+			}
+		}
+
+		// Validate the operation log on startup.
+		sysBucket := tx.Bucket(SystemBucket)
+		opLogBucket := tx.Bucket(OpLogBucket)
+		c := opLogBucket.Cursor()
+		if lastValidated := sysBucket.Get([]byte("last_validated")); lastValidated != nil {
+			c.Seek(lastValidated)
+		}
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			op := &v1.Operation{}
+			if err := proto.Unmarshal(v, op); err != nil {
+				zap.L().Error("error unmarshalling operation, there may be corruption in the oplog", zap.Error(err))
+				continue
+			}
+			if op.Status == v1.OperationStatus_STATUS_INPROGRESS {
+				op.Status = v1.OperationStatus_STATUS_ERROR
+				op.DisplayMessage = "Operation timeout."
+				bytes, err := proto.Marshal(op)
+				if err != nil {
+					return fmt.Errorf("marshalling operation: %w", err)
+				}
+				if err := opLogBucket.Put(k, bytes); err != nil {
+					return fmt.Errorf("putting operation into bucket: %w", err)
+				}
+			}
+		}
+		if lastValidated, _ := c.Last(); lastValidated != nil {
+			if err := sysBucket.Put([]byte("last_validated"), lastValidated); err != nil {
+				return fmt.Errorf("checkpointing last_validated key: %w", err)
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return &OpLog{db: db}, nil
+}
+
+func (o *OpLog) Close() error {
+	return o.db.Close()
+}
+
+// Add adds a generic operation to the operation log.
+func (o *OpLog) Add(op *v1.Operation) error {
+	if op.Id != 0 {
+		return errors.New("operation already has an ID, OpLog.Add is expected to set the ID")
+	}
+
+	err := o.db.Update(func(tx *bolt.Tx) error {
+		err := o.addOperationHelper(tx, op)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err == nil {
+		o.notifyHelper(EventTypeOpCreated, op)
+	}
+	return err
+}
+
+func (o *OpLog) BulkAdd(ops []*v1.Operation) error {
+	err := o.db.Update(func(tx *bolt.Tx) error {
+		for _, op := range ops {
+			if err := o.addOperationHelper(tx, op); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err == nil {
+		for _, op := range ops {
+			o.notifyHelper(EventTypeOpCreated, op)
+		}
+	}
+	return err
+}
+
+func (o *OpLog) addOperationHelper(tx *bolt.Tx, op *v1.Operation) error {
+	b := tx.Bucket(OpLogBucket)
+
+	id, err := b.NextSequence()
+	if err != nil {
+		return fmt.Errorf("error getting next sequence: %w", err)
+	}
+
+	op.Id = int64(id)
+
+	bytes, err := proto.Marshal(op)
+	if err != nil {
+		return fmt.Errorf("error marshalling operation: %w", err)
+	}
+
+	if err := b.Put(serializationutil.Itob(op.Id), bytes); err != nil {
+		return fmt.Errorf("error putting operation into bucket: %w", err)
+	}
+
+	// Update always universal indices
+	if op.RepoId != "" {
+		if err := indexutil.IndexByteValue(tx.Bucket(RepoIndexBucket), []byte(op.RepoId), op.Id); err != nil {
+			return fmt.Errorf("error adding operation to repo index: %w", err)
+		}
+	}
+	if op.PlanId != "" {
+		if err := indexutil.IndexByteValue(tx.Bucket(PlanIndexBucket), []byte(op.PlanId), op.Id); err != nil {
+			return fmt.Errorf("error adding operation to repo index: %w", err)
+		}
+	}
+
+	// Update operation type dependent indices.
+	switch wrappedOp := op.Op.(type) {
+	case *v1.Operation_OperationBackup:
+		// Nothing extra to be done.
+	case *v1.Operation_OperationIndexSnapshot:
+		if wrappedOp.OperationIndexSnapshot == nil || wrappedOp.OperationIndexSnapshot.Snapshot == nil {
+			return errors.New("op.OperationIndexSnapshot or op.OperationIndexSnapshot.Snapshot is nil")
+		}
+		snapshotId := serializationutil.NormalizeSnapshotId(wrappedOp.OperationIndexSnapshot.Snapshot.Id)
+		key := serializationutil.BytesToKey([]byte(snapshotId))
+		if err := tx.Bucket(IndexedSnapshotsSetBucket).Put(key, serializationutil.Itob(op.Id)); err != nil {
+			return fmt.Errorf("error adding OperationIndexSnapshot to indexed snapshots set: %w", err)
+		}
+	default:
+		return fmt.Errorf("unknown operation type: %T", wrappedOp)
+	}
+
+	return nil
+}
+
+func (o *OpLog) HasIndexedSnapshot(snapshotId string) (int64, error) {
+	var id int64
+	if err := o.db.View(func(tx *bolt.Tx) error {
+		snapshotId := serializationutil.NormalizeSnapshotId(snapshotId)
+		key := serializationutil.BytesToKey([]byte(snapshotId))
+		idBytes := tx.Bucket(IndexedSnapshotsSetBucket).Get(key)
+		if idBytes == nil {
+			id = -1
+		} else {
+			var err error
+			id, err = serializationutil.Btoi(idBytes)
+			if err != nil {
+				return fmt.Errorf("database corrupt, couldn't convert ID bytes to int: %w", err)
+			}
+		}
+		return nil
+	}); err != nil {
+		return 0, err
+	}
+	return id, nil
+}
+
+func (o *OpLog) Update(op *v1.Operation) error {
+	if op.Id == 0 {
+		return errors.New("operation does not have an ID, OpLog.Update expects operation with an ID")
+	}
+
+	err := o.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(OpLogBucket)
+
+		if b.Get(serializationutil.Itob(op.Id)) == nil {
+			return fmt.Errorf("operation with ID %d does not exist", op.Id)
+		}
+
+		bytes, err := proto.Marshal(op)
+		if err != nil {
+			return fmt.Errorf("error marshalling operation: %w", err)
+		}
+
+		if err := b.Put(serializationutil.Itob(op.Id), bytes); err != nil {
+			return fmt.Errorf("error putting operation into bucket: %w", err)
+		}
+
+		return nil
+	})
+	if err == nil {
+		o.notifyHelper(EventTypeOpUpdated, op)
+	}
+	return err
+}
+
+func (o *OpLog) notifyHelper(eventType EventType, op *v1.Operation) {
+	o.subscribersMu.RLock()
+	defer o.subscribersMu.RUnlock()
+	for _, sub := range o.subscribers {
+		(*sub)(eventType, op)
+	}
+}
+
+func (o *OpLog) getHelper(b *bolt.Bucket, id int64) (*v1.Operation, error) {
+	bytes := b.Get(serializationutil.Itob(id))
+	if bytes == nil {
+		return nil, fmt.Errorf("operation with ID %d does not exist", id)
+	}
+
+	var op v1.Operation
+	if err := proto.Unmarshal(bytes, &op); err != nil {
+		return nil, fmt.Errorf("error unmarshalling operation: %w", err)
+	}
+
+	return &op, nil
+}
+
+func (o *OpLog) Get(id int64) (*v1.Operation, error) {
+	var op *v1.Operation
+	if err := o.db.View(func(tx *bolt.Tx) error {
+		var err error
+		op, err = o.getHelper(tx.Bucket(OpLogBucket), id)
+		return err
+	}); err != nil {
+		return nil, err
+	}
+	return op, nil
+}
+
+func (o *OpLog) GetByRepo(repoId string, filter Filter) ([]*v1.Operation, error) {
+	var ops []*v1.Operation
+	if err := o.db.View(func(tx *bolt.Tx) error {
+		ids := indexutil.IndexSearchByteValue(tx.Bucket(RepoIndexBucket), []byte(repoId)).ToSlice()
+		ids = filter(ids)
+
+		b := tx.Bucket(OpLogBucket)
+		for _, id := range ids {
+			op, err := o.getHelper(b, id)
+			if err != nil {
+				return err
+			}
+			ops = append(ops, op)
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return ops, nil
+}
+
+func (o *OpLog) GetByPlan(planId string, filter Filter) ([]*v1.Operation, error) {
+	var ops []*v1.Operation
+	if err := o.db.View(func(tx *bolt.Tx) error {
+		ids := indexutil.IndexSearchByteValue(tx.Bucket(PlanIndexBucket), []byte(planId)).ToSlice()
+		ids = filter(ids)
+
+		b := tx.Bucket(OpLogBucket)
+		for _, id := range ids {
+			op, err := o.getHelper(b, id)
+			if err != nil {
+				return err
+			}
+			ops = append(ops, op)
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return ops, nil
+}
+
+func (o *OpLog) GetAll(filter Filter) ([]*v1.Operation, error) {
+	var ops []*v1.Operation
+	if err := o.db.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket(OpLogBucket).Cursor()
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			op := &v1.Operation{}
+			if err := proto.Unmarshal(v, op); err != nil {
+				return fmt.Errorf("error unmarshalling operation: %w", err)
+			}
+			ops = append(ops, op)
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return ops, nil
+}
+
+func (o *OpLog) Subscribe(callback *func(EventType, *v1.Operation)) {
+	o.subscribersMu.Lock()
+	defer o.subscribersMu.Unlock()
+	o.subscribers = append(o.subscribers, callback)
+}
+
+func (o *OpLog) Unsubscribe(callback *func(EventType, *v1.Operation)) {
+	o.subscribersMu.Lock()
+	defer o.subscribersMu.Unlock()
+	subs := o.subscribers
+	for i, c := range subs {
+		if c == callback {
+			subs[i] = subs[len(subs)-1]
+			o.subscribers = subs[:len(o.subscribers)-1]
+		}
+	}
+}
+
+type Filter func([]int64) []int64
+
+func FilterKeepAll() Filter {
+	return func(ids []int64) []int64 {
+		return ids
+	}
+}
+
+func FilterLastN(n int64) Filter {
+	return func(ids []int64) []int64 {
+		if len(ids) > int(n) {
+			ids = ids[len(ids)-int(n):]
+		}
+		return ids
+	}
+}
+
+func FilterLimitOffset(limit, offset int64) Filter {
+	return func(ids []int64) []int64 {
+		if len(ids) > int(offset) {
+			ids = ids[offset:]
+		}
+		if len(ids) > int(limit) {
+			ids = ids[:limit]
+		}
+		return ids
+	}
+}

internal/database/oplog/oplog_test.go

@@ -0,0 +1,288 @@
+package oplog
+
+import (
+	"slices"
+	"testing"
+
+	v1 "github.com/garethgeorge/resticui/gen/go/v1"
+)
+
+func TestCreate(t *testing.T) {
+	// t.Parallel()
+	log, err := NewOpLog(t.TempDir() + "/test.boltdb")
+	t.Cleanup(func() { log.Close() })
+	if err != nil {
+		t.Fatalf("error creating oplog: %s", err)
+	}
+	if err := log.Close(); err != nil {
+		t.Fatalf("error closing oplog: %s", err)
+	}
+}
+
+func TestAddOperation(t *testing.T) {
+	// t.Parallel()
+	log, err := NewOpLog(t.TempDir() + "/test.boltdb")
+	if err != nil {
+		t.Fatalf("error creating oplog: %s", err)
+	}
+	t.Cleanup(func() { log.Close() })
+
+	var tests = []struct {
+		name    string
+		op      *v1.Operation
+		wantErr bool
+	}{
+		{
+			name: "no operation",
+			op: &v1.Operation{
+				Id: 0,
+			},
+			wantErr: true,
+		},
+		{
+			name: "basic backup operation",
+			op: &v1.Operation{
+				Id: 0,
+				Op: &v1.Operation_OperationBackup{},
+			},
+			wantErr: false,
+		},
+		{
+			name: "basic snapshot operation",
+			op: &v1.Operation{
+				Id: 0,
+				Op: &v1.Operation_OperationIndexSnapshot{
+					OperationIndexSnapshot: &v1.OperationIndexSnapshot{
+						Snapshot: &v1.ResticSnapshot{
+							Id: "test",
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "basic snapshot operation with no snapshot",
+			op: &v1.Operation{
+				Id: 0,
+				Op: &v1.Operation_OperationIndexSnapshot{
+					OperationIndexSnapshot: &v1.OperationIndexSnapshot{},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "operation with ID",
+			op: &v1.Operation{
+				Id: 1,
+				Op: &v1.Operation_OperationBackup{},
+			},
+			wantErr: true,
+		},
+		{
+			name: "operation with repo",
+			op: &v1.Operation{
+				Id:     0,
+				RepoId: "testrepo",
+				Op:     &v1.Operation_OperationBackup{},
+			},
+		},
+		{
+			name: "operation with plan",
+			op: &v1.Operation{
+				Id:     0,
+				PlanId: "testplan",
+				Op:     &v1.Operation_OperationBackup{},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if err := log.Add(tc.op); (err != nil) != tc.wantErr {
+				t.Errorf("Add() error = %v, wantErr %v", err, tc.wantErr)
+			}
+			if !tc.wantErr {
+				if tc.op.Id == 0 {
+					t.Errorf("Add() did not set op ID")
+				}
+			}
+		})
+	}
+}
+
+func TestListOperation(t *testing.T) {
+	// t.Parallel()
+	log, err := NewOpLog(t.TempDir() + "/test.boltdb")
+	if err != nil {
+		t.Fatalf("error creating oplog: %s", err)
+	}
+	t.Cleanup(func() { log.Close() })
+
+	// these should get assigned IDs 1-3 respectively by the oplog
+	ops := []*v1.Operation{
+		{
+			PlanId:         "plan1",
+			RepoId:         "repo1",
+			DisplayMessage: "op1",
+			Op:             &v1.Operation_OperationBackup{},
+		},
+		{
+			PlanId:         "plan1",
+			RepoId:         "repo2",
+			DisplayMessage: "op2",
+			Op:             &v1.Operation_OperationBackup{},
+		},
+		{
+			PlanId:         "plan2",
+			RepoId:         "repo2",
+			DisplayMessage: "op3",
+			Op:             &v1.Operation_OperationBackup{},
+		},
+	}
+
+	for _, op := range ops {
+		if err := log.Add(op); err != nil {
+			t.Fatalf("error adding operation: %s", err)
+		}
+	}
+
+	tests := []struct {
+		name     string
+		byPlan   bool
+		byRepo   bool
+		id       string
+		expected []string
+	}{
+		{
+			name:     "list plan1",
+			byPlan:   true,
+			id:       "plan1",
+			expected: []string{"op1", "op2"},
+		},
+		{
+			name:     "list plan2",
+			byPlan:   true,
+			id:       "plan2",
+			expected: []string{"op3"},
+		},
+		{
+			name:     "list repo1",
+			byRepo:   true,
+			id:       "repo1",
+			expected: []string{"op1"},
+		},
+		{
+			name:     "list repo2",
+			byRepo:   true,
+			id:       "repo2",
+			expected: []string{"op2", "op3"},
+		},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			// t.Parallel()
+			var ops []*v1.Operation
+			var err error
+			if tc.byPlan {
+				ops, err = log.GetByPlan(tc.id, FilterKeepAll())
+			} else if tc.byRepo {
+				ops, err = log.GetByRepo(tc.id, FilterKeepAll())
+			} else {
+				t.Fatalf("must specify byPlan or byRepo")
+			}
+			if err != nil {
+				t.Fatalf("error listing operations: %s", err)
+			}
+			got := collectMessages(ops)
+			if slices.Compare(got, tc.expected) != 0 {
+				t.Errorf("want operations: %v, got unexpected operations: %v", tc.expected, got)
+			}
+		})
+	}
+}
+
+func TestBigIO(t *testing.T) {
+	// t.Parallel()
+	log, err := NewOpLog(t.TempDir() + "/test.boltdb")
+	if err != nil {
+		t.Fatalf("error creating oplog: %s", err)
+	}
+	t.Cleanup(func() { log.Close() })
+
+	for i := 0; i < 100; i++ {
+		if err := log.Add(&v1.Operation{
+			PlanId: "plan1",
+			RepoId: "repo1",
+			Op:     &v1.Operation_OperationBackup{},
+		}); err != nil {
+			t.Fatalf("error adding operation: %s", err)
+		}
+	}
+
+	ops, err := log.GetByPlan("plan1", FilterKeepAll())
+	if err != nil {
+		t.Fatalf("error listing operations: %s", err)
+	}
+	if len(ops) != 100 {
+		t.Errorf("want 100 operations, got %d", len(ops))
+	}
+
+	ops, err = log.GetByRepo("repo1", FilterKeepAll())
+	if err != nil {
+		t.Fatalf("error listing operations: %s", err)
+	}
+	if len(ops) != 100 {
+		t.Errorf("want 100 operations, got %d", len(ops))
+	}
+}
+
+func TestIndexSnapshot(t *testing.T) {
+	t.Parallel()
+	log, err := NewOpLog(t.TempDir() + "/test.boltdb")
+	if err != nil {
+		t.Fatalf("error creating oplog: %s", err)
+	}
+	t.Cleanup(func() { log.Close() })
+
+	op := &v1.Operation{
+		PlanId: "plan1",
+		RepoId: "repo1",
+		Op: &v1.Operation_OperationIndexSnapshot{
+			OperationIndexSnapshot: &v1.OperationIndexSnapshot{
+				Snapshot: &v1.ResticSnapshot{
+					Id: "abcdefghijklmnop",
+				},
+			},
+		},
+	}
+	if err := log.Add(op); err != nil {
+		t.Fatalf("error adding operation: %s", err)
+	}
+
+	id, err := log.HasIndexedSnapshot("abcdefgh")
+	if err != nil {
+		t.Fatalf("error checking for snapshot: %s", err)
+	}
+	if id != op.Id {
+		t.Fatalf("want id %d, got %d", op.Id, id)
+	}
+
+	id, err = log.HasIndexedSnapshot("notfound")
+	if err != nil {
+		t.Fatalf("error checking for snapshot: %s", err)
+	}
+	if id != -1 {
+		t.Fatalf("want id -1, got %d", id)
+	}
+}
+
+func collectMessages(ops []*v1.Operation) []string {
+	var messages []string
+	for _, op := range ops {
+		messages = append(messages, op.DisplayMessage)
+	}
+	return messages
+}

internal/database/serializationutil/serializationutil.go

@@ -0,0 +1,53 @@
+package serializationutil
+
+import (
+	"encoding/binary"
+	"errors"
+)
+
+var ErrInvalidLength = errors.New("invalid length")
+
+func Itob(v int64) []byte {
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, uint64(v))
+	return b
+}
+
+func Btoi(b []byte) (int64, error) {
+	if len(b) != 8 {
+		return 0, ErrInvalidLength
+	}
+	return int64(binary.BigEndian.Uint64(b)), nil
+}
+
+func Stob(v string) []byte {
+	var b []byte
+	b = append(b, Itob(int64(len(v)))...)
+	b = append(b, []byte(v)...)
+	return b
+}
+
+func Btos(b []byte) (string, int64, error) {
+	if len(b) < 8 {
+		return "", 0, ErrInvalidLength
+	}
+	length, _ := Btoi(b[:8])
+	if int64(len(b)) < 8+length {
+		return "", 0, ErrInvalidLength
+	}
+	return string(b[8:8+length]), 8+length, nil
+}
+
+func BytesToKey(b []byte) []byte {
+	var key []byte
+	key = append(key, Itob(int64(len(b)))...)
+	key = append(key, b...)
+	return key
+}
+
+func NormalizeSnapshotId(id string) string {
+	if len(id) < 8 {
+		return id
+	}
+	return id[:8]
+}

internal/database/serializationutil/serializationutil_test.go

@@ -0,0 +1,23 @@
+package serializationutil
+
+import "testing"
+
+func TestItoa(t *testing.T) {
+	nums := []int64{0, 1, 2, 3, 4, 1 << 32, int64(1) << 62}
+	for _, num := range nums {
+		b := Itob(num)
+		if v, _ := Btoi(b); v != num {
+			t.Errorf("itob/btoi failed for %d", num)
+		}
+	}
+}
+
+func TestStob(t *testing.T) {
+	strs := []string{"", "a", "ab", "abc", "abcd", "abcde", "abcdef"}
+	for _, str := range strs {
+		b := Stob(str)
+		if val, _, _ := Btos(b); val != str {
+			t.Errorf("stob/btos failed for %s", str)
+		}
+	}
+}