fix: restic package properly handles 'verbose_status' and 'exit_error' status types

internal/orchestrator/tasks/taskbackup.go

@@ -170,7 +170,7 @@ func (t *BackupTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunne
 				return
 			}
 			backupOp.OperationBackup.Errors = append(backupOp.OperationBackup.Errors, backupError)
-		} else if entry.MessageType != "summary" {
+		} else if entry.MessageType != "summary" && entry.MessageType != "verbose_status" {
 			zap.S().Warnf("unexpected message type %q in backup progress entry", entry.MessageType)
 		}
 

pkg/restic/outputs.go

@@ -84,12 +84,12 @@ func (s *Snapshot) Validate() error {
 
 type BackupProgressEntry struct {
 	// Common fields
-	MessageType string `json:"message_type"` // "summary" or "status" or "error"
+	MessageType string `json:"message_type"` // "summary" or "status" or "error" or "verbose_status" or "exit_error"
 
 	// Error fields
 	Error  any    `json:"error"`
 	During string `json:"during"`
-	Item   string `json:"item"`
+	Item   string `json:"item"` // also present for verbose_status for some actions
 
 	// Summary fields
 	FilesNew            int64   `json:"files_new"`
@@ -113,9 +113,27 @@ type BackupProgressEntry struct {
 	TotalBytes   int64    `json:"total_bytes"`
 	BytesDone    int64    `json:"bytes_done"`
 	CurrentFiles []string `json:"current_files"`
+
+	// Verbose status fields
+	Action string `json:"action"`
+
+	// Exit error fields
+	ExitError string `json:"exit_error"`
+	Message   string `json:"message"`
+}
+
+var validBackupMessageTypes = map[string]struct{}{
+	"summary":        {},
+	"status":         {},
+	"verbose_status": {},
+	"error":          {}, // Errors are not fatal, they are just logged.
 }
 
 func (b *BackupProgressEntry) Validate() error {
+	if _, ok := validBackupMessageTypes[b.MessageType]; !ok {
+		return fmt.Errorf("invalid message type: %v", b.MessageType)
+	}
+
 	if b.MessageType == "summary" && b.SnapshotId != "" {
 		if err := ValidateSnapshotId(b.SnapshotId); err != nil {
 			return err
@@ -125,8 +143,11 @@ func (b *BackupProgressEntry) Validate() error {
 	return nil
 }
 
-func (b *BackupProgressEntry) IsError() bool {
-	return b.MessageType == "error"
+func (b *BackupProgressEntry) IsFatalError() error {
+	if b.MessageType == "exit_error" {
+		return errors.New(b.Message)
+	}
+	return nil
 }
 
 func (b *BackupProgressEntry) IsSummary() bool {
@@ -134,24 +155,40 @@ func (b *BackupProgressEntry) IsSummary() bool {
 }
 
 type RestoreProgressEntry struct {
-	MessageType    string  `json:"message_type"` // "summary" or "status"
+	MessageType    string  `json:"message_type"` // "summary" or "status" or "verbose_status" or "error" or "exit_error"
 	SecondsElapsed float64 `json:"seconds_elapsed"`
 	TotalBytes     int64   `json:"total_bytes"`
 	BytesRestored  int64   `json:"bytes_restored"`
 	TotalFiles     int64   `json:"total_files"`
 	FilesRestored  int64   `json:"files_restored"`
 	PercentDone    float64 `json:"percent_done"`
+
+	// Verbose status fields
+	Action string `json:"action"`
+
+	// Exit error fields
+	ExitError string `json:"exit_error"`
+	Message   string `json:"message"`
+}
+
+var validRestoreMessageTypes = map[string]struct{}{
+	"summary":        {},
+	"status":         {},
+	"verbose_status": {},
+	"error":          {}, // Errors are not fatal, they are just logged.
 }
 
 func (e *RestoreProgressEntry) Validate() error {
-	if e.MessageType != "summary" && e.MessageType != "status" {
-		return fmt.Errorf("message_type must be 'summary' or 'status', got %v", e.MessageType)
+	if _, ok := validRestoreMessageTypes[e.MessageType]; !ok {
+		return fmt.Errorf("invalid message type: %v", e.MessageType)
 	}
 	return nil
 }
-
-func (r *RestoreProgressEntry) IsError() bool {
-	return r.MessageType == "error"
+func (e *RestoreProgressEntry) IsFatalError() error {
+	if e.MessageType == "exit_error" {
+		return errors.New(e.Message)
+	}
+	return nil
 }
 
 func (r *RestoreProgressEntry) IsSummary() bool {
@@ -160,7 +197,7 @@ func (r *RestoreProgressEntry) IsSummary() bool {
 
 type ProgressEntryValidator interface {
 	Validate() error
-	IsError() bool
+	IsFatalError() error
 	IsSummary() bool
 }
 
@@ -192,17 +229,16 @@ func processProgressOutput[T ProgressEntryValidator](
 			continue
 		}
 
+		if err := event.IsFatalError(); err != nil {
+			return nullT, newErrorWithOutput(fmt.Errorf("restic died with error: %v", err), nonJSONOutput.String())
+		}
+
 		if err := event.Validate(); err != nil {
 			captureNonJSON.Write(line)
 			captureNonJSON.Write([]byte("\n"))
 			continue
 		}
 
-		if event.IsError() && logger != nil {
-			captureNonJSON.Write(line)
-			captureNonJSON.Write([]byte("\n"))
-		}
-
 		if callback != nil {
 			callback(event)
 		}

pkg/restic/restic.go

@@ -211,6 +211,19 @@ type cmdRunnerWithProgress[T ProgressEntryValidator] struct {
 	failureErr error
 }
 
+// handleExitError processes a command exit error and converts it to an appropriate error type
+func (cr *cmdRunnerWithProgress[T]) handleExitError(err error) error {
+	var exitErr *exec.ExitError
+	if errors.As(err, &exitErr) {
+		if exitErr.ExitCode() == 3 {
+			return ErrPartialBackup
+		} else {
+			return fmt.Errorf("exit code %d: %w", exitErr.ExitCode(), cr.failureErr)
+		}
+	}
+	return err
+}
+
 func (cr *cmdRunnerWithProgress[T]) Run(ctx context.Context, args []string, opts ...GenericOption) (T, error) {
 	logger := LoggerFromContext(ctx)
 	cmdCtx, cancel := context.WithCancel(ctx)
@@ -237,7 +250,7 @@ func (cr *cmdRunnerWithProgress[T]) Run(ctx context.Context, args []string, opts
 		result, err := processProgressOutput[T](reader, logger, cr.callback)
 		summary = result
 		if err != nil {
-			readErr = fmt.Errorf("processing command output: %w", err)
+			readErr = fmt.Errorf("output processing: %w", err)
 		}
 	}()
 
@@ -247,7 +260,7 @@ func (cr *cmdRunnerWithProgress[T]) Run(ctx context.Context, args []string, opts
 
 	if cmdErr != nil || readErr != nil {
 		if cmdErr != nil {
-			cmdErr = cr.repo.handleExitError(cmdErr, cr.failureErr)
+			cmdErr = cr.handleExitError(cmdErr)
 		}
 		return summary, newCmdError(ctx, cmd, errors.Join(cmdErr, readErr))
 	}
@@ -286,19 +299,6 @@ func (r *Repo) Restore(ctx context.Context, snapshot string, callback func(*Rest
 	return cr.Run(ctx, args, opts...)
 }
 
-// handleExitError processes a command exit error and converts it to an appropriate error type
-func (r *Repo) handleExitError(err error, failureErr error) error {
-	var exitErr *exec.ExitError
-	if errors.As(err, &exitErr) {
-		if exitErr.ExitCode() == 3 {
-			return ErrPartialBackup
-		} else {
-			return fmt.Errorf("exit code %d: %w", exitErr.ExitCode(), failureErr)
-		}
-	}
-	return err
-}
-
 func (r *Repo) Snapshots(ctx context.Context, opts ...GenericOption) ([]*Snapshot, error) {
 	var snapshots []*Snapshot
 	if err := r.executeWithJSONOutput(ctx, []string{"snapshots", "--json"}, &snapshots, opts...); err != nil {

pkg/restic/restic_test.go

@@ -608,6 +608,36 @@ func TestResticCheck(t *testing.T) {
 	}
 }
 
+func TestResticExitError(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS == "windows" {
+		t.Skip("skipping on windows")
+	}
+
+	// Create a directory that isn't writable
+	tmpDir := t.TempDir()
+	os.Chmod(tmpDir, 0000)
+	defer os.Chmod(tmpDir, 0755)
+
+	// Test data
+	testData := helpers.CreateTestData(t)
+
+	repo := t.TempDir()
+	r := NewRepo(helpers.ResticBinary(t), repo, WithFlags("--no-cache"), WithEnv("RESTIC_PASSWORD=test", "TMPDIR="+tmpDir))
+	if err := r.Init(context.Background()); err != nil {
+		t.Fatalf("failed to init repo: %v", err)
+	}
+
+	_, err := r.Backup(context.Background(), []string{testData}, nil)
+	if err == nil {
+		t.Fatal("expected backup to fail, got nil error")
+	}
+	if !strings.Contains(err.Error(), "unable to save snapshot") {
+		t.Errorf("expected error to contain 'unable to save snapshot', got: %v", err)
+	}
+}
+
 func TestJSONCommandResilantToBeginningWarnings(t *testing.T) {
 	t.Parallel()
 	if runtime.GOOS == "windows" {