diff --git a/internal/oplog/sqlitestore/sqlitestore.go b/internal/oplog/sqlitestore/sqlitestore.go index e69fe524..5a4b9cc6 100644 --- a/internal/oplog/sqlitestore/sqlitestore.go +++ b/internal/oplog/sqlitestore/sqlitestore.go @@ -9,11 +9,14 @@ import ( "os" "path/filepath" "runtime" + "sort" "strings" "sync/atomic" "testing" + "time" v1 "github.com/garethgeorge/backrest/gen/go/v1" + "github.com/garethgeorge/backrest/internal/config/migrations" "github.com/garethgeorge/backrest/internal/ioutil" "github.com/garethgeorge/backrest/internal/kvstore" "github.com/garethgeorge/backrest/internal/oplog" @@ -93,6 +96,9 @@ func NewSqliteStore(db string) (*SqliteStore, error) { } else if !locked { return nil, ErrLocked } + if err := store.backup(db, 3, false); err != nil { + return nil, fmt.Errorf("backup sqlite db: %v", err) + } if err := store.init(); err != nil { return nil, err } @@ -149,6 +155,63 @@ func (m *SqliteStore) init() error { return nil } +// backup creates a backup of the database using VACUUM INTO. +// keepCount specifies how many old backups to keep (older ones are deleted). +// force skips the time check and creates a backup even if the latest is recent. +func (m *SqliteStore) backup(to string, keepCount int, force bool) error { + dir := filepath.Dir(to) + base := filepath.Base(to) + pattern := fmt.Sprintf("%s-*.backup", base) + matches, err := filepath.Glob(filepath.Join(dir, pattern)) + if err != nil { + return fmt.Errorf("glob for old backups: %v", err) + } + sort.Strings(matches) + + // Create a suffix indicating the schema, this way we can always create a new backup + // if the schema of the last backup doesn't match the current schema implying we're about to run a migration. + backupSuffix := fmt.Sprintf("s%02dm%02d.backup", sqlSchemaVersion, migrations.CurrentVersion) + + // Skip creating a new backup if the latest is less than an hour old OR if the schema suffix doesn't match + if !force && len(matches) > 0 { + latestBackup := matches[len(matches)-1] + info, err := os.Stat(latestBackup) + if err != nil { + return fmt.Errorf("stat latest backup %q: %w", latestBackup, err) + } + if strings.HasSuffix(latestBackup, backupSuffix) && time.Since(info.ModTime()) < 7*24*time.Hour { + // Don't create a new backup more than once a week if the last one matches the schema. + return nil + } + } + + // Create the backup using VACUUM INTO + backupPath := fmt.Sprintf("%s-%s-%s", to, time.Now().Format("20060102.150405.000"), backupSuffix) + _, err = m.dbpool.ExecContext(context.Background(), "VACUUM INTO ?", backupPath) + if err != nil { + return fmt.Errorf("backup sqlite db: %v", err) + } + + // Refresh the list of backups after creating the new one + matches, err = filepath.Glob(filepath.Join(dir, pattern)) + if err != nil { + return fmt.Errorf("glob for backups after creation: %v", err) + } + sort.Strings(matches) + + // Delete old backups, keeping only the specified number + if len(matches) > keepCount { + toDelete := matches[:len(matches)-keepCount] + for _, f := range toDelete { + if err := os.Remove(f); err != nil { + return fmt.Errorf("delete old backup %q: %w", f, err) + } + } + } + + return nil +} + func (m *SqliteStore) GetHighestOpIDAndModno(q oplog.Query) (int64, int64, error) { var highestID sql.NullInt64 var highestModno sql.NullInt64 diff --git a/internal/oplog/sqlitestore/sqlitestore_test.go b/internal/oplog/sqlitestore/sqlitestore_test.go new file mode 100644 index 00000000..ae4b173a --- /dev/null +++ b/internal/oplog/sqlitestore/sqlitestore_test.go @@ -0,0 +1,86 @@ +package sqlitestore + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// TestBackup_NewDatabaseCreatesValidBackup verifies that a new database creates +// a backup during initialization and that the backup is valid and can be read. +func TestBackup_NewDatabaseCreatesValidBackup(t *testing.T) { + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "test.db") + + store, err := NewSqliteStore(dbPath) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + defer store.Close() + + // NewSqliteStore should have created an initial backup + matches, err := filepath.Glob(filepath.Join(tempDir, "test.db-*.bak")) + if err != nil { + t.Fatalf("failed to glob for backups: %v", err) + } + + if len(matches) != 1 { + t.Fatalf("expected 1 backup file from NewSqliteStore, got %d", len(matches)) + } + + // Verify the backup file exists and is not empty + info, err := os.Stat(matches[0]) + if err != nil { + t.Fatalf("failed to stat backup file: %v", err) + } + + if info.Size() == 0 { + t.Fatal("backup file is empty") + } + + // Try to open the backup as a SQLite database to verify it's valid + backupStore, err := NewSqliteStore(matches[0]) + if err != nil { + t.Fatalf("backup file is not a valid SQLite database: %v", err) + } + defer backupStore.Close() +} + +// TestBackup_CleansUpOldBackups verifies that after running backup multiple times, +// it correctly cleans up old backups and keeps only the specified number. +func TestBackup_CleansUpOldBackups(t *testing.T) { + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "test.db") + + store, err := NewSqliteStore(dbPath) + if err != nil { + t.Fatalf("failed to create store: %v", err) + } + defer store.Close() + + // Clear the initial backup created during NewSqliteStore + matches, _ := filepath.Glob(filepath.Join(tempDir, "test.db-*.bak")) + for _, match := range matches { + os.Remove(match) + } + + // Create 10 backups with force=true, keeping only 3 + for i := 0; i < 10; i++ { + time.Sleep(10 * time.Millisecond) // Ensure different timestamps + err = store.backup(dbPath, 3, true) + if err != nil { + t.Fatalf("failed to run backup %d: %v", i, err) + } + } + + // Should have only 3 backups remaining + matches, err = filepath.Glob(filepath.Join(tempDir, "test.db-*.backup")) + if err != nil { + t.Fatalf("failed to glob for backups: %v", err) + } + + if len(matches) != 3 { + t.Errorf("expected 3 backups (keepCount=3), got %d", len(matches)) + } +}