feat: use sqlite logstore (#514)

internal/orchestrator/tasks/taskcollectgarbage.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	v1 "github.com/garethgeorge/backrest/gen/go/v1"
+	"github.com/garethgeorge/backrest/internal/logstore"
 	"github.com/garethgeorge/backrest/internal/oplog"
 	"go.uber.org/zap"
 )
@@ -30,14 +31,16 @@ func gcAgeForOperation(op *v1.Operation) time.Duration {
 type CollectGarbageTask struct {
 	BaseTask
 	firstRun bool
+	logstore *logstore.LogStore
 }
 
-func NewCollectGarbageTask() *CollectGarbageTask {
+func NewCollectGarbageTask(logstore *logstore.LogStore) *CollectGarbageTask {
 	return &CollectGarbageTask{
 		BaseTask: BaseTask{
 			TaskType: "collect_garbage",
 			TaskName: "collect garbage",
 		},
+		logstore: logstore,
 	}
 }
 
@@ -82,9 +85,12 @@ func (t *CollectGarbageTask) gcOperations(log *oplog.OpLog) error {
 		return fmt.Errorf("identifying forgotten snapshots: %w", err)
 	}
 
+	validIDs := make(map[int64]struct{})
 	forgetIDs := []int64{}
 	curTime := curTimeMillis()
 	if err := log.Query(oplog.SelectAll, func(op *v1.Operation) error {
+		validIDs[op.Id] = struct{}{}
+
 		forgot, ok := snapshotForgottenForFlow[op.FlowId]
 		if !ok {
 			// no snapshot associated with this flow; check if it's old enough to be gc'd
@@ -103,9 +109,33 @@ func (t *CollectGarbageTask) gcOperations(log *oplog.OpLog) error {
 
 	if err := log.Delete(forgetIDs...); err != nil {
 		return fmt.Errorf("removing gc eligible operations: %w", err)
+	} else if len(forgetIDs) > 0 {
+		for _, id := range forgetIDs {
+			delete(validIDs, id)
+		}
 	}
 
 	zap.L().Info("collecting garbage",
 		zap.Any("operations_removed", len(forgetIDs)))
+
+	// cleaning up logstore
+	toDelete := []string{}
+	if err := t.logstore.SelectAll(func(id string, parentID int64) {
+		if parentID == 0 {
+			return
+		}
+		if _, ok := validIDs[parentID]; !ok {
+			toDelete = append(toDelete, id)
+		}
+	}); err != nil {
+		return fmt.Errorf("selecting all logstore entries: %w", err)
+	}
+	for _, id := range toDelete {
+		if err := t.logstore.Delete(id); err != nil {
+			zap.L().Error("deleting logstore entry", zap.String("id", id), zap.Error(err))
+		}
+	}
+	zap.L().Info("collecting garbage logs", zap.Any("logs_removed", len(toDelete)))
+
 	return nil
 }