fix: sqlitestore creates backups periodically and prior to running database migrations or schema migrations

internal/oplog/sqlitestore/sqlitestore.go

@@ -9,11 +9,14 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 	"sync/atomic"
 	"testing"
+	"time"
 
 	v1 "github.com/garethgeorge/backrest/gen/go/v1"
+	"github.com/garethgeorge/backrest/internal/config/migrations"
 	"github.com/garethgeorge/backrest/internal/ioutil"
 	"github.com/garethgeorge/backrest/internal/kvstore"
 	"github.com/garethgeorge/backrest/internal/oplog"
@@ -93,6 +96,9 @@ func NewSqliteStore(db string) (*SqliteStore, error) {
 	} else if !locked {
 		return nil, ErrLocked
 	}
+	if err := store.backup(db, 3, false); err != nil {
+		return nil, fmt.Errorf("backup sqlite db: %v", err)
+	}
 	if err := store.init(); err != nil {
 		return nil, err
 	}
@@ -149,6 +155,63 @@ func (m *SqliteStore) init() error {
 	return nil
 }
 
+// backup creates a backup of the database using VACUUM INTO.
+// keepCount specifies how many old backups to keep (older ones are deleted).
+// force skips the time check and creates a backup even if the latest is recent.
+func (m *SqliteStore) backup(to string, keepCount int, force bool) error {
+	dir := filepath.Dir(to)
+	base := filepath.Base(to)
+	pattern := fmt.Sprintf("%s-*.backup", base)
+	matches, err := filepath.Glob(filepath.Join(dir, pattern))
+	if err != nil {
+		return fmt.Errorf("glob for old backups: %v", err)
+	}
+	sort.Strings(matches)
+
+	// Create a suffix indicating the schema, this way we can always create a new backup
+	// if the schema of the last backup doesn't match the current schema implying we're about to run a migration.
+	backupSuffix := fmt.Sprintf("s%02dm%02d.backup", sqlSchemaVersion, migrations.CurrentVersion)
+
+	// Skip creating a new backup if the latest is less than an hour old OR if the schema suffix doesn't match
+	if !force && len(matches) > 0 {
+		latestBackup := matches[len(matches)-1]
+		info, err := os.Stat(latestBackup)
+		if err != nil {
+			return fmt.Errorf("stat latest backup %q: %w", latestBackup, err)
+		}
+		if strings.HasSuffix(latestBackup, backupSuffix) && time.Since(info.ModTime()) < 7*24*time.Hour {
+			// Don't create a new backup more than once a week if the last one matches the schema.
+			return nil
+		}
+	}
+
+	// Create the backup using VACUUM INTO
+	backupPath := fmt.Sprintf("%s-%s-%s", to, time.Now().Format("20060102.150405.000"), backupSuffix)
+	_, err = m.dbpool.ExecContext(context.Background(), "VACUUM INTO ?", backupPath)
+	if err != nil {
+		return fmt.Errorf("backup sqlite db: %v", err)
+	}
+
+	// Refresh the list of backups after creating the new one
+	matches, err = filepath.Glob(filepath.Join(dir, pattern))
+	if err != nil {
+		return fmt.Errorf("glob for backups after creation: %v", err)
+	}
+	sort.Strings(matches)
+
+	// Delete old backups, keeping only the specified number
+	if len(matches) > keepCount {
+		toDelete := matches[:len(matches)-keepCount]
+		for _, f := range toDelete {
+			if err := os.Remove(f); err != nil {
+				return fmt.Errorf("delete old backup %q: %w", f, err)
+			}
+		}
+	}
+
+	return nil
+}
+
 func (m *SqliteStore) GetHighestOpIDAndModno(q oplog.Query) (int64, int64, error) {
 	var highestID sql.NullInt64
 	var highestModno sql.NullInt64

internal/oplog/sqlitestore/sqlitestore_test.go

@@ -0,0 +1,86 @@
+package sqlitestore
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// TestBackup_NewDatabaseCreatesValidBackup verifies that a new database creates
+// a backup during initialization and that the backup is valid and can be read.
+func TestBackup_NewDatabaseCreatesValidBackup(t *testing.T) {
+	tempDir := t.TempDir()
+	dbPath := filepath.Join(tempDir, "test.db")
+
+	store, err := NewSqliteStore(dbPath)
+	if err != nil {
+		t.Fatalf("failed to create store: %v", err)
+	}
+	defer store.Close()
+
+	// NewSqliteStore should have created an initial backup
+	matches, err := filepath.Glob(filepath.Join(tempDir, "test.db-*.bak"))
+	if err != nil {
+		t.Fatalf("failed to glob for backups: %v", err)
+	}
+
+	if len(matches) != 1 {
+		t.Fatalf("expected 1 backup file from NewSqliteStore, got %d", len(matches))
+	}
+
+	// Verify the backup file exists and is not empty
+	info, err := os.Stat(matches[0])
+	if err != nil {
+		t.Fatalf("failed to stat backup file: %v", err)
+	}
+
+	if info.Size() == 0 {
+		t.Fatal("backup file is empty")
+	}
+
+	// Try to open the backup as a SQLite database to verify it's valid
+	backupStore, err := NewSqliteStore(matches[0])
+	if err != nil {
+		t.Fatalf("backup file is not a valid SQLite database: %v", err)
+	}
+	defer backupStore.Close()
+}
+
+// TestBackup_CleansUpOldBackups verifies that after running backup multiple times,
+// it correctly cleans up old backups and keeps only the specified number.
+func TestBackup_CleansUpOldBackups(t *testing.T) {
+	tempDir := t.TempDir()
+	dbPath := filepath.Join(tempDir, "test.db")
+
+	store, err := NewSqliteStore(dbPath)
+	if err != nil {
+		t.Fatalf("failed to create store: %v", err)
+	}
+	defer store.Close()
+
+	// Clear the initial backup created during NewSqliteStore
+	matches, _ := filepath.Glob(filepath.Join(tempDir, "test.db-*.bak"))
+	for _, match := range matches {
+		os.Remove(match)
+	}
+
+	// Create 10 backups with force=true, keeping only 3
+	for i := 0; i < 10; i++ {
+		time.Sleep(10 * time.Millisecond) // Ensure different timestamps
+		err = store.backup(dbPath, 3, true)
+		if err != nil {
+			t.Fatalf("failed to run backup %d: %v", i, err)
+		}
+	}
+
+	// Should have only 3 backups remaining
+	matches, err = filepath.Glob(filepath.Join(tempDir, "test.db-*.backup"))
+	if err != nil {
+		t.Fatalf("failed to glob for backups: %v", err)
+	}
+
+	if len(matches) != 3 {
+		t.Errorf("expected 3 backups (keepCount=3), got %d", len(matches))
+	}
+}