From 3aa6839701899796500ee95cd8c83ff7fdeecb08 Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:58:45 +1000 Subject: [PATCH] Recover Lightning decoder failures with a software retry --- src/lib/exporter/streamingDecodePipeline.ts | 63 +++++++- src/lib/exporter/streamingDecoder.test.ts | 157 ++++++++++++++++++-- 2 files changed, 203 insertions(+), 17 deletions(-) diff --git a/src/lib/exporter/streamingDecodePipeline.ts b/src/lib/exporter/streamingDecodePipeline.ts index acd24c80..d3ed0390 100644 --- a/src/lib/exporter/streamingDecodePipeline.ts +++ b/src/lib/exporter/streamingDecodePipeline.ts @@ -39,12 +39,70 @@ const STARTUP_MAX_PENDING_FRAMES = 28; export async function decodeVideoStream( context: StreamingDecodeContext, + targetFrameRate: number, + trimRegions: TrimRegion[] | undefined, + speedRegions: SpeedRegion[] | undefined, + onFrame: OnFrameCallback, + segmentsOverride?: VideoSegment[], +): Promise { + let emittedFrames = 0; + try { + await decodeVideoStreamAttempt( + context, + targetFrameRate, + trimRegions, + speedRegions, + async (...args) => { + await onFrame(...args); + emittedFrames++; + }, + segmentsOverride, + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const codec = context.metadata.codec.toLowerCase(); + // Only retry failures that completed the decode-error cleanup. Configuration + // and renderer/encoder callback failures must propagate without replay. + if ( + context.cancelled || + context.decoder !== null || + codec.includes("av01") || + codec.includes("av1") || + !/^\[VIDEO_DECODE_ENCODING_ERROR\]|^\[VIDEO_DECODER_RESOURCE_EXHAUSTED\]/.test(message) + ) { + throw error; + } + console.warn( + "[StreamingVideoDecoder] Decoder failed; replaying with software decoding", + error, + ); + // Replay from a keyframe to rebuild decoder state. Output already handed to + // the renderer/encoder must not be submitted again. + let replayedFrames = 0; + await decodeVideoStreamAttempt( + context, + targetFrameRate, + trimRegions, + speedRegions, + async (...args) => { + if (replayedFrames++ < emittedFrames) return; + await onFrame(...args); + }, + segmentsOverride, + true, + ); + } +} + +async function decodeVideoStreamAttempt( + context: StreamingDecodeContext, targetFrameRate: number, trimRegions: TrimRegion[] | undefined, speedRegions: SpeedRegion[] | undefined, onFrame: OnFrameCallback, segmentsOverride?: VideoSegment[], + forceSoftwareDecode = false, ): Promise { if (!context.demuxer || !context.metadata) { throw new Error("Must call loadMetadata() before decodeAll()"); @@ -52,7 +110,8 @@ export async function decodeVideoStream( const decoderConfig = await context.demuxer.getDecoderConfig("video"); const codec = context.metadata.codec.toLowerCase(); - const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1"); + const shouldPreferSoftwareDecode = + forceSoftwareDecode || codec.includes("av01") || codec.includes("av1"); const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({ duration: context.metadata.duration, streamDuration: context.metadata.streamDuration, @@ -157,7 +216,7 @@ export async function decodeVideoStream( try { context.decoder.configure(preferredDecoderConfig); } catch (error) { - if (!shouldPreferSoftwareDecode) { + if (!shouldPreferSoftwareDecode || forceSoftwareDecode) { throw buildVideoDecodeFailure(error, getDecoderFailureContext()); } // Fall back to default decoder config if software preference is unsupported. diff --git a/src/lib/exporter/streamingDecoder.test.ts b/src/lib/exporter/streamingDecoder.test.ts index d8d2c9e5..e11c6e61 100644 --- a/src/lib/exporter/streamingDecoder.test.ts +++ b/src/lib/exporter/streamingDecoder.test.ts @@ -139,18 +139,19 @@ describe("StreamingVideoDecoder decode failures", () => { codedWidth: 1920, codedHeight: 1080, }); - mockDemuxerRead.mockReturnValue( - new ReadableStream({ - start(controller) { - controller.enqueue({ - type: "key", - timestamp: 0, - duration: 33_333, - byteLength: 4, - }); - controller.close(); - }, - }), + mockDemuxerRead.mockImplementation( + () => + new ReadableStream({ + start(controller) { + controller.enqueue({ + type: "key", + timestamp: 0, + duration: 33_333, + byteLength: 4, + }); + controller.close(); + }, + }), ); Object.assign(globalThis, { window: { @@ -168,7 +169,7 @@ describe("StreamingVideoDecoder decode failures", () => { }); it("does not emit frozen remaining frames after a decoder error", async () => { - const frame = { timestamp: 0, close: vi.fn() } as unknown as VideoFrame; + const frames: Array<{ timestamp: number; close: ReturnType }> = []; class FailingVideoDecoder { state: CodecState = "unconfigured"; decodeQueueSize = 0; @@ -182,7 +183,9 @@ describe("StreamingVideoDecoder decode failures", () => { this.state = "configured"; } decode() { - this.callbacks.output(frame); + const frame = { timestamp: 0, close: vi.fn() }; + frames.push(frame); + this.callbacks.output(frame as unknown as VideoFrame); this.callbacks.error(new DOMException("bad frame", "EncodingError")); } async flush() {} @@ -200,7 +203,131 @@ describe("StreamingVideoDecoder decode failures", () => { "[VIDEO_DECODE_ENCODING_ERROR]", ); expect(onFrame).not.toHaveBeenCalled(); - expect(frame.close).toHaveBeenCalledTimes(1); + expect(frames).toHaveLength(2); + expect(mockDemuxerRead).toHaveBeenCalledTimes(2); + for (const frame of frames) expect(frame.close).toHaveBeenCalledTimes(1); + }); + + it.each([ + "EncodingError", + "QuotaExceededError", + ])("replays %s in software without duplicating exported frames", async (errorName) => { + mockDemuxerRead.mockImplementation( + () => + new ReadableStream({ + start(controller) { + for (let i = 0; i < 120; i++) + controller.enqueue({ timestamp: (i * 1_000_000) / 30 }); + controller.close(); + }, + }), + ); + const configs: VideoDecoderConfig[] = []; + const timestamps: number[] = []; + let framesBeforeRetry = 0; + const closedFrames: ReturnType[] = []; + class RecoveringDecoder { + state: CodecState = "unconfigured"; + decodeQueueSize = 0; + software = false; + constructor( + private callbacks: { + output: (frame: VideoFrame) => void; + error: (error: DOMException) => void; + }, + ) {} + configure(config: VideoDecoderConfig) { + configs.push(config); + this.software = config.hardwareAcceleration === "prefer-software"; + if (this.software) framesBeforeRetry = timestamps.length; + this.state = "configured"; + } + decode(chunk: EncodedVideoChunk) { + if (!this.software && chunk.timestamp >= 500_000) { + this.state = "closed"; + this.callbacks.error(new DOMException("driver decode failed", errorName)); + return; + } + const close = vi.fn(); + closedFrames.push(close); + this.callbacks.output({ + timestamp: chunk.timestamp, + close, + } as unknown as VideoFrame); + } + async flush() {} + close() { + this.state = "closed"; + } + } + vi.stubGlobal("VideoDecoder", RecoveringDecoder); + const decoder = new StreamingVideoDecoder(); + await decoder.loadMetadata("/tmp/driver-failure.mp4"); + await decoder.decodeAll(30, undefined, undefined, async (_frame, timestamp) => { + timestamps.push(timestamp); + }); + expect(configs).toHaveLength(2); + expect(configs[1].hardwareAcceleration).toBe("prefer-software"); + expect(framesBeforeRetry).toBeGreaterThan(0); + expect(timestamps).toHaveLength(120); + for (let i = 0; i < timestamps.length; i++) + expect(timestamps[i]).toBeCloseTo((i * 1_000_000) / 30); + expect(mockDemuxerRead).toHaveBeenCalledTimes(2); + for (const close of closedFrames) expect(close).toHaveBeenCalledTimes(1); + }); + + it.each([ + { + codec: "av01.0.04M.08", + errorName: "EncodingError", + cancelled: false, + setupFailure: false, + }, + { codec: "avc1.640034", errorName: "DataError", cancelled: false, setupFailure: false }, + { codec: "avc1.640034", errorName: "EncodingError", cancelled: true, setupFailure: false }, + { codec: "avc1.640034", errorName: "EncodingError", cancelled: false, setupFailure: true }, + ])("does not retry $codec / $errorName (cancelled: $cancelled, setup: $setupFailure)", async ({ + codec, + errorName, + cancelled, + setupFailure, + }) => { + const info = await mockDemuxerGetMediaInfo(); + mockDemuxerGetMediaInfo.mockResolvedValueOnce({ + ...info, + streams: info.streams.map((stream: Record) => ({ + ...stream, + codec_string: codec, + })), + }); + const configure = vi.fn(); + const decoder = new StreamingVideoDecoder(); + class NonRetryDecoder { + state: CodecState = "unconfigured"; + decodeQueueSize = 0; + constructor(private callbacks: { error: (error: DOMException) => void }) {} + configure() { + configure(); + if (setupFailure) throw new DOMException("setup failed", errorName); + this.state = "configured"; + } + decode() { + if (cancelled) decoder.cancel(); + this.state = "closed"; + this.callbacks.error(new DOMException("decode failed", errorName)); + } + async flush() {} + close() { + this.state = "closed"; + } + } + vi.stubGlobal("VideoDecoder", NonRetryDecoder); + await decoder.loadMetadata("/tmp/no-retry.mp4"); + await expect(decoder.decodeAll(30, undefined, undefined, vi.fn())).rejects.toThrow( + setupFailure ? "setup failed" : "decode failed", + ); + expect(configure).toHaveBeenCalledTimes(1); + decoder.destroy(); }); it.each([