Recover Lightning decoder failures with a software retry

src/lib/exporter/streamingDecodePipeline.ts

@@ -39,12 +39,70 @@ const STARTUP_MAX_PENDING_FRAMES = 28;
 
 export async function decodeVideoStream(
 	context: StreamingDecodeContext,
+	targetFrameRate: number,
+	trimRegions: TrimRegion[] | undefined,
+	speedRegions: SpeedRegion[] | undefined,
+	onFrame: OnFrameCallback,
+	segmentsOverride?: VideoSegment[],
+): Promise<void> {
+	let emittedFrames = 0;
+	try {
+		await decodeVideoStreamAttempt(
+			context,
+			targetFrameRate,
+			trimRegions,
+			speedRegions,
+			async (...args) => {
+				await onFrame(...args);
+				emittedFrames++;
+			},
+			segmentsOverride,
+		);
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		const codec = context.metadata.codec.toLowerCase();
+		// Only retry failures that completed the decode-error cleanup. Configuration
+		// and renderer/encoder callback failures must propagate without replay.
+		if (
+			context.cancelled ||
+			context.decoder !== null ||
+			codec.includes("av01") ||
+			codec.includes("av1") ||
+			!/^\[VIDEO_DECODE_ENCODING_ERROR\]|^\[VIDEO_DECODER_RESOURCE_EXHAUSTED\]/.test(message)
+		) {
+			throw error;
+		}
+		console.warn(
+			"[StreamingVideoDecoder] Decoder failed; replaying with software decoding",
+			error,
+		);
+		// Replay from a keyframe to rebuild decoder state. Output already handed to
+		// the renderer/encoder must not be submitted again.
+		let replayedFrames = 0;
+		await decodeVideoStreamAttempt(
+			context,
+			targetFrameRate,
+			trimRegions,
+			speedRegions,
+			async (...args) => {
+				if (replayedFrames++ < emittedFrames) return;
+				await onFrame(...args);
+			},
+			segmentsOverride,
+			true,
+		);
+	}
+}
+
+async function decodeVideoStreamAttempt(
+	context: StreamingDecodeContext,
 
 	targetFrameRate: number,
 	trimRegions: TrimRegion[] | undefined,
 	speedRegions: SpeedRegion[] | undefined,
 	onFrame: OnFrameCallback,
 	segmentsOverride?: VideoSegment[],
+	forceSoftwareDecode = false,
 ): Promise<void> {
 	if (!context.demuxer || !context.metadata) {
 		throw new Error("Must call loadMetadata() before decodeAll()");
@@ -52,7 +110,8 @@ export async function decodeVideoStream(
 
 	const decoderConfig = await context.demuxer.getDecoderConfig("video");
 	const codec = context.metadata.codec.toLowerCase();
-	const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1");
+	const shouldPreferSoftwareDecode =
+		forceSoftwareDecode || codec.includes("av01") || codec.includes("av1");
 	const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({
 		duration: context.metadata.duration,
 		streamDuration: context.metadata.streamDuration,
@@ -157,7 +216,7 @@ export async function decodeVideoStream(
 	try {
 		context.decoder.configure(preferredDecoderConfig);
 	} catch (error) {
-		if (!shouldPreferSoftwareDecode) {
+		if (!shouldPreferSoftwareDecode || forceSoftwareDecode) {
 			throw buildVideoDecodeFailure(error, getDecoderFailureContext());
 		}
 		// Fall back to default decoder config if software preference is unsupported.

src/lib/exporter/streamingDecoder.test.ts

@@ -139,18 +139,19 @@ describe("StreamingVideoDecoder decode failures", () => {
 			codedWidth: 1920,
 			codedHeight: 1080,
 		});
-		mockDemuxerRead.mockReturnValue(
-			new ReadableStream({
-				start(controller) {
-					controller.enqueue({
-						type: "key",
-						timestamp: 0,
-						duration: 33_333,
-						byteLength: 4,
-					});
-					controller.close();
-				},
-			}),
+		mockDemuxerRead.mockImplementation(
+			() =>
+				new ReadableStream({
+					start(controller) {
+						controller.enqueue({
+							type: "key",
+							timestamp: 0,
+							duration: 33_333,
+							byteLength: 4,
+						});
+						controller.close();
+					},
+				}),
 		);
 		Object.assign(globalThis, {
 			window: {
@@ -168,7 +169,7 @@ describe("StreamingVideoDecoder decode failures", () => {
 	});
 
 	it("does not emit frozen remaining frames after a decoder error", async () => {
-		const frame = { timestamp: 0, close: vi.fn() } as unknown as VideoFrame;
+		const frames: Array<{ timestamp: number; close: ReturnType<typeof vi.fn> }> = [];
 		class FailingVideoDecoder {
 			state: CodecState = "unconfigured";
 			decodeQueueSize = 0;
@@ -182,7 +183,9 @@ describe("StreamingVideoDecoder decode failures", () => {
 				this.state = "configured";
 			}
 			decode() {
-				this.callbacks.output(frame);
+				const frame = { timestamp: 0, close: vi.fn() };
+				frames.push(frame);
+				this.callbacks.output(frame as unknown as VideoFrame);
 				this.callbacks.error(new DOMException("bad frame", "EncodingError"));
 			}
 			async flush() {}
@@ -200,7 +203,131 @@ describe("StreamingVideoDecoder decode failures", () => {
 			"[VIDEO_DECODE_ENCODING_ERROR]",
 		);
 		expect(onFrame).not.toHaveBeenCalled();
-		expect(frame.close).toHaveBeenCalledTimes(1);
+		expect(frames).toHaveLength(2);
+		expect(mockDemuxerRead).toHaveBeenCalledTimes(2);
+		for (const frame of frames) expect(frame.close).toHaveBeenCalledTimes(1);
+	});
+
+	it.each([
+		"EncodingError",
+		"QuotaExceededError",
+	])("replays %s in software without duplicating exported frames", async (errorName) => {
+		mockDemuxerRead.mockImplementation(
+			() =>
+				new ReadableStream({
+					start(controller) {
+						for (let i = 0; i < 120; i++)
+							controller.enqueue({ timestamp: (i * 1_000_000) / 30 });
+						controller.close();
+					},
+				}),
+		);
+		const configs: VideoDecoderConfig[] = [];
+		const timestamps: number[] = [];
+		let framesBeforeRetry = 0;
+		const closedFrames: ReturnType<typeof vi.fn>[] = [];
+		class RecoveringDecoder {
+			state: CodecState = "unconfigured";
+			decodeQueueSize = 0;
+			software = false;
+			constructor(
+				private callbacks: {
+					output: (frame: VideoFrame) => void;
+					error: (error: DOMException) => void;
+				},
+			) {}
+			configure(config: VideoDecoderConfig) {
+				configs.push(config);
+				this.software = config.hardwareAcceleration === "prefer-software";
+				if (this.software) framesBeforeRetry = timestamps.length;
+				this.state = "configured";
+			}
+			decode(chunk: EncodedVideoChunk) {
+				if (!this.software && chunk.timestamp >= 500_000) {
+					this.state = "closed";
+					this.callbacks.error(new DOMException("driver decode failed", errorName));
+					return;
+				}
+				const close = vi.fn();
+				closedFrames.push(close);
+				this.callbacks.output({
+					timestamp: chunk.timestamp,
+					close,
+				} as unknown as VideoFrame);
+			}
+			async flush() {}
+			close() {
+				this.state = "closed";
+			}
+		}
+		vi.stubGlobal("VideoDecoder", RecoveringDecoder);
+		const decoder = new StreamingVideoDecoder();
+		await decoder.loadMetadata("/tmp/driver-failure.mp4");
+		await decoder.decodeAll(30, undefined, undefined, async (_frame, timestamp) => {
+			timestamps.push(timestamp);
+		});
+		expect(configs).toHaveLength(2);
+		expect(configs[1].hardwareAcceleration).toBe("prefer-software");
+		expect(framesBeforeRetry).toBeGreaterThan(0);
+		expect(timestamps).toHaveLength(120);
+		for (let i = 0; i < timestamps.length; i++)
+			expect(timestamps[i]).toBeCloseTo((i * 1_000_000) / 30);
+		expect(mockDemuxerRead).toHaveBeenCalledTimes(2);
+		for (const close of closedFrames) expect(close).toHaveBeenCalledTimes(1);
+	});
+
+	it.each([
+		{
+			codec: "av01.0.04M.08",
+			errorName: "EncodingError",
+			cancelled: false,
+			setupFailure: false,
+		},
+		{ codec: "avc1.640034", errorName: "DataError", cancelled: false, setupFailure: false },
+		{ codec: "avc1.640034", errorName: "EncodingError", cancelled: true, setupFailure: false },
+		{ codec: "avc1.640034", errorName: "EncodingError", cancelled: false, setupFailure: true },
+	])("does not retry $codec / $errorName (cancelled: $cancelled, setup: $setupFailure)", async ({
+		codec,
+		errorName,
+		cancelled,
+		setupFailure,
+	}) => {
+		const info = await mockDemuxerGetMediaInfo();
+		mockDemuxerGetMediaInfo.mockResolvedValueOnce({
+			...info,
+			streams: info.streams.map((stream: Record<string, unknown>) => ({
+				...stream,
+				codec_string: codec,
+			})),
+		});
+		const configure = vi.fn();
+		const decoder = new StreamingVideoDecoder();
+		class NonRetryDecoder {
+			state: CodecState = "unconfigured";
+			decodeQueueSize = 0;
+			constructor(private callbacks: { error: (error: DOMException) => void }) {}
+			configure() {
+				configure();
+				if (setupFailure) throw new DOMException("setup failed", errorName);
+				this.state = "configured";
+			}
+			decode() {
+				if (cancelled) decoder.cancel();
+				this.state = "closed";
+				this.callbacks.error(new DOMException("decode failed", errorName));
+			}
+			async flush() {}
+			close() {
+				this.state = "closed";
+			}
+		}
+		vi.stubGlobal("VideoDecoder", NonRetryDecoder);
+		await decoder.loadMetadata("/tmp/no-retry.mp4");
+		await expect(decoder.decodeAll(30, undefined, undefined, vi.fn())).rejects.toThrow(
+			setupFailure ? "setup failed" : "decode failed",
+		);
+		expect(configure).toHaveBeenCalledTimes(1);
+		decoder.destroy();
 	});
 
 	it.each([