chore: add new utilities and scripts for manual validations

cmd/devtools/oplogexport/.gitignore

@@ -0,0 +1 @@
+oplog.*

cmd/devtools/oplogexport/main.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"flag"
+	"os"
+	"path"
+
+	v1 "github.com/garethgeorge/backrest/gen/go/v1"
+	"github.com/garethgeorge/backrest/internal/env"
+	"github.com/garethgeorge/backrest/internal/oplog"
+	"github.com/garethgeorge/backrest/internal/oplog/bboltstore"
+	"go.etcd.io/bbolt"
+	"go.uber.org/zap"
+	"google.golang.org/protobuf/encoding/prototext"
+)
+
+var (
+	outpath = flag.String("export-oplog-path", "", "path to export the oplog as a compressed textproto e.g. .textproto.gz")
+)
+
+func main() {
+	flag.Parse()
+
+	if *outpath == "" {
+		flag.Usage()
+		return
+	}
+
+	oplogFile := path.Join(env.DataDir(), "oplog.boltdb")
+	opstore, err := bboltstore.NewBboltStore(oplogFile)
+	if err != nil {
+		if !errors.Is(err, bbolt.ErrTimeout) {
+			zap.S().Fatalf("timeout while waiting to open database, is the database open elsewhere?")
+		}
+		zap.S().Warnf("operation log may be corrupted, if errors recur delete the file %q and restart. Your backups stored in your repos are safe.", oplogFile)
+		zap.S().Fatalf("error creating oplog : %v", err)
+	}
+	defer opstore.Close()
+
+	output := &v1.OperationList{}
+
+	log := oplog.NewOpLog(opstore)
+	log.Query(oplog.Query{}, func(op *v1.Operation) error {
+		output.Operations = append(output.Operations, op)
+		return nil
+	})
+
+	bytes, err := prototext.MarshalOptions{Multiline: true}.Marshal(output)
+	if err != nil {
+		zap.S().Fatalf("error marshalling operations: %v", err)
+	}
+
+	bytes, err = compress(bytes)
+	if err != nil {
+		zap.S().Fatalf("error compressing operations: %v", err)
+	}
+
+	if err := os.WriteFile(*outpath, bytes, 0644); err != nil {
+		zap.S().Fatalf("error writing to file: %v", err)
+	}
+}
+
+func compress(data []byte) ([]byte, error) {
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+
+	if _, err := zw.Write(data); err != nil {
+		return nil, err
+	}
+
+	if err := zw.Close(); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}

cmd/devtools/oplogimport/main.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"flag"
+	"log"
+	"os"
+	"path"
+
+	v1 "github.com/garethgeorge/backrest/gen/go/v1"
+	"github.com/garethgeorge/backrest/internal/env"
+	"github.com/garethgeorge/backrest/internal/oplog/bboltstore"
+	"go.uber.org/zap"
+	"google.golang.org/protobuf/encoding/prototext"
+)
+
+var (
+	outpath = flag.String("import-oplog-path", "", "path to import the oplog from compressed textproto e.g. .textproto.gz")
+)
+
+func main() {
+	flag.Parse()
+
+	if *outpath == "" {
+		flag.Usage()
+		return
+	}
+
+	// create a reader from the file
+	f, err := os.Open(*outpath)
+	if err != nil {
+		log.Fatalf("error opening file: %v", err)
+	}
+	defer f.Close()
+
+	cr, err := gzip.NewReader(f)
+	if err != nil {
+		log.Fatalf("error creating gzip reader: %v", err)
+	}
+	defer cr.Close()
+
+	// read into a buffer
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(cr); err != nil {
+		log.Printf("error reading from gzip reader: %v", err)
+	}
+
+	log.Printf("importing operations from %q", *outpath)
+
+	output := &v1.OperationList{}
+	if err := prototext.Unmarshal(buf.Bytes(), output); err != nil {
+		log.Fatalf("error unmarshalling operations: %v", err)
+	}
+
+	zap.S().Infof("importing %d operations", len(output.Operations))
+
+	oplogFile := path.Join(env.DataDir(), "oplog.boltdb")
+	opstore, err := bboltstore.NewBboltStore(oplogFile)
+	if err != nil {
+		log.Fatalf("error creating oplog : %v", err)
+	}
+	defer opstore.Close()
+
+	for _, op := range output.Operations {
+		if err := opstore.Add(op); err != nil {
+			log.Printf("error adding operation to oplog: %v", err)
+		}
+	}
+}

scripts/testing/run-fresh.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+BASEDIR=$(dirname "$0")
+TEMPDIR=$(mktemp -d)
+
+function cleanup {
+  echo "Removing temp dir: $TEMPDIR"
+  rm -rf $TEMPDIR
+}
+
+trap cleanup EXIT
+
+echo "Temp dir: $TEMPDIR"
+
+go run $BASEDIR/../../cmd/backrest --config-file=$TEMPDIR/config.json --data-dir=$TEMPDIR/data

scripts/testing/run-in-dir.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+BASEDIR=$(dirname "$0")
+RUNDIR=$1
+
+if [ -z "$RUNDIR" ]; then
+  echo "Usage: $0 <run-dir>"
+  exit 1
+fi
+
+go run $BASEDIR/../../cmd/backrest --config-file=$RUNDIR/config.json --data-dir=$RUNDIR
+