diff --git a/src/lib/exporter/finalizationTimeout.test.ts b/src/lib/exporter/finalizationTimeout.test.ts index fae630f9..107d709f 100644 --- a/src/lib/exporter/finalizationTimeout.test.ts +++ b/src/lib/exporter/finalizationTimeout.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "vitest"; -import { getExportFinalizationTimeoutMs } from "./finalizationTimeout"; +import { + getExportFinalizationIdleTimeoutMs, + getExportFinalizationTimeoutMs, +} from "./finalizationTimeout"; describe("finalizationTimeout", () => { it("keeps non-audio finalization on the existing 10 minute timeout", () => { @@ -51,4 +54,24 @@ describe("finalizationTimeout", () => { }), ).toBe(600_000); }); + + it("derives a bounded idle watchdog window from the total timeout", () => { + expect( + getExportFinalizationIdleTimeoutMs({ + workload: "default", + }), + ).toBe(150_000); + expect( + getExportFinalizationIdleTimeoutMs({ + workload: "audio", + effectiveDurationSec: 1_200, + }), + ).toBe(300_000); + expect( + getExportFinalizationIdleTimeoutMs({ + workload: "audio", + effectiveDurationSec: 2_700, + }), + ).toBe(300_000); + }); }); diff --git a/src/lib/exporter/finalizationTimeout.ts b/src/lib/exporter/finalizationTimeout.ts index 804af0f4..69ed6d0d 100644 --- a/src/lib/exporter/finalizationTimeout.ts +++ b/src/lib/exporter/finalizationTimeout.ts @@ -3,6 +3,9 @@ export type FinalizationTimeoutWorkload = "default" | "audio"; const BASE_FINALIZATION_TIMEOUT_MS = 10 * 60_000; const AUDIO_TIMEOUT_HEADROOM_PER_OUTPUT_SECOND_MS = 500; const MAX_AUDIO_FINALIZATION_TIMEOUT_MS = 45 * 60_000; +const MIN_PROGRESS_IDLE_TIMEOUT_MS = 90_000; +const MAX_PROGRESS_IDLE_TIMEOUT_MS = 5 * 60_000; +const PROGRESS_IDLE_TIMEOUT_FRACTION = 0.25; export function getExportFinalizationTimeoutMs({ effectiveDurationSec, @@ -15,16 +18,38 @@ export function getExportFinalizationTimeoutMs({ return BASE_FINALIZATION_TIMEOUT_MS; } - if (!Number.isFinite(effectiveDurationSec) || (effectiveDurationSec ?? 0) <= 0) { + const safeEffectiveDurationSec = + typeof effectiveDurationSec === "number" ? effectiveDurationSec : Number.NaN; + if (!Number.isFinite(safeEffectiveDurationSec) || safeEffectiveDurationSec <= 0) { return BASE_FINALIZATION_TIMEOUT_MS; } // Audio finalization work scales with the output timeline, so long exports need // more headroom without making unrelated finalization hangs wait longer. - const safeEffectiveDurationSec = Math.max(0, effectiveDurationSec ?? 0); const adaptiveTimeoutMs = BASE_FINALIZATION_TIMEOUT_MS + safeEffectiveDurationSec * AUDIO_TIMEOUT_HEADROOM_PER_OUTPUT_SECOND_MS; return Math.min(adaptiveTimeoutMs, MAX_AUDIO_FINALIZATION_TIMEOUT_MS); } + +export function getExportFinalizationIdleTimeoutMs({ + effectiveDurationSec, + workload = "default", +}: { + effectiveDurationSec?: number | null; + workload?: FinalizationTimeoutWorkload; +}): number { + const totalTimeoutMs = getExportFinalizationTimeoutMs({ + effectiveDurationSec, + workload, + }); + + return Math.min( + Math.max( + Math.floor(totalTimeoutMs * PROGRESS_IDLE_TIMEOUT_FRACTION), + MIN_PROGRESS_IDLE_TIMEOUT_MS, + ), + MAX_PROGRESS_IDLE_TIMEOUT_MS, + ); +} diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 76f6331d..330cefcb 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -24,6 +24,7 @@ import { } from "./exportTuning"; import { type FinalizationTimeoutWorkload, + getExportFinalizationIdleTimeoutMs, getExportFinalizationTimeoutMs, } from "./finalizationTimeout"; import { FrameRenderer as ModernFrameRenderer } from "./modernFrameRenderer"; @@ -153,6 +154,7 @@ export class ModernVideoExporter { private nativeWriteTimeMs = 0; private finalizationTimeMs = 0; private processedFrameCount = 0; + private activeFinalizationProgressWatchdog: { refreshProgress: () => void } | null = null; private lastProgressSampleTimeMs = 0; private lastProgressSampleFrame = 0; @@ -473,6 +475,7 @@ export class ModernVideoExporter { ), "audio processing", "audio", + true, ); } } @@ -645,22 +648,52 @@ export class ModernVideoExporter { promise: Promise, stage: string, workload: FinalizationTimeoutWorkload = "default", + progressAware = false, ): Promise { let timeoutId: ReturnType | null = null; + let idleTimeoutId: ReturnType | null = null; const timeoutMs = getExportFinalizationTimeoutMs({ effectiveDurationSec: this.effectiveDurationSec, workload, }); + const idleTimeoutMs = progressAware + ? getExportFinalizationIdleTimeoutMs({ + effectiveDurationSec: this.effectiveDurationSec, + workload, + }) + : null; + const watchdog: { refreshProgress: () => void } | null = + progressAware && idleTimeoutMs + ? { + refreshProgress: () => undefined, + } + : null; try { return await Promise.race([ promise, new Promise((_, reject) => { + const rejectWithMessage = (message: string) => { + reject(new Error(message)); + }; + if (watchdog) { + this.activeFinalizationProgressWatchdog = watchdog; + const refreshProgress = () => { + if (idleTimeoutId) { + clearTimeout(idleTimeoutId); + } + idleTimeoutId = setTimeout(() => { + rejectWithMessage( + `Export timed out during ${stage} after ${Math.ceil(idleTimeoutMs! / 1000)} seconds without observable progress`, + ); + }, idleTimeoutMs!); + }; + watchdog.refreshProgress = refreshProgress; + refreshProgress(); + } timeoutId = setTimeout(() => { - reject( - new Error( - `Export timed out during ${stage} after ${Math.ceil(timeoutMs / 60_000)} minutes`, - ), + rejectWithMessage( + `Export timed out during ${stage} after ${Math.ceil(timeoutMs / 60_000)} minutes`, ); }, timeoutMs); }), @@ -669,6 +702,12 @@ export class ModernVideoExporter { if (timeoutId) { clearTimeout(timeoutId); } + if (idleTimeoutId) { + clearTimeout(idleTimeoutId); + } + if (this.activeFinalizationProgressWatchdog === watchdog) { + this.activeFinalizationProgressWatchdog = null; + } } } @@ -984,6 +1023,7 @@ export class ModernVideoExporter { ), `${NATIVE_EXPORT_ENGINE_NAME} edited audio rendering`, "audio", + true, ); editedAudioBuffer = await audioBlob.arrayBuffer(); editedAudioMimeType = audioBlob.type || null; @@ -1067,6 +1107,7 @@ export class ModernVideoExporter { ), "FFmpeg edited audio rendering", "audio", + true, ); editedAudioBuffer = await audioBlob.arrayBuffer(); editedAudioMimeType = audioBlob.type || null; @@ -1164,6 +1205,7 @@ export class ModernVideoExporter { renderProgress: number, audioProgress?: number, ) { + this.activeFinalizationProgressWatchdog?.refreshProgress(); this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress, audioProgress); } @@ -1606,6 +1648,7 @@ export class ModernVideoExporter { this.nativeWriteTimeMs = 0; this.finalizationTimeMs = 0; this.processedFrameCount = 0; + this.activeFinalizationProgressWatchdog = null; this.effectiveDurationSec = 0; this.lastProgressSampleTimeMs = 0; this.lastProgressSampleFrame = 0; diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 754407d5..ba015c3c 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -15,6 +15,7 @@ import type { import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder"; import { type FinalizationTimeoutWorkload, + getExportFinalizationIdleTimeoutMs, getExportFinalizationTimeoutMs, } from "./finalizationTimeout"; import { FrameRenderer } from "./frameRenderer"; @@ -111,6 +112,7 @@ export class VideoExporter { private nativeWriteError: Error | null = null; private maxNativeWriteInFlight = 1; private nativeEncoderError: Error | null = null; + private activeFinalizationProgressWatchdog: { refreshProgress: () => void } | null = null; constructor(config: VideoExporterConfig) { this.config = config; @@ -324,6 +326,7 @@ export class VideoExporter { ), "audio processing", "audio", + true, ); } } @@ -377,22 +380,52 @@ export class VideoExporter { promise: Promise, stage: string, workload: FinalizationTimeoutWorkload = "default", + progressAware = false, ): Promise { let timeoutId: ReturnType | null = null; + let idleTimeoutId: ReturnType | null = null; const timeoutMs = getExportFinalizationTimeoutMs({ effectiveDurationSec: this.effectiveDurationSec, workload, }); + const idleTimeoutMs = progressAware + ? getExportFinalizationIdleTimeoutMs({ + effectiveDurationSec: this.effectiveDurationSec, + workload, + }) + : null; + const watchdog: { refreshProgress: () => void } | null = + progressAware && idleTimeoutMs + ? { + refreshProgress: () => undefined, + } + : null; try { return await Promise.race([ promise, new Promise((_, reject) => { + const rejectWithMessage = (message: string) => { + reject(new Error(message)); + }; + if (watchdog) { + this.activeFinalizationProgressWatchdog = watchdog; + const refreshProgress = () => { + if (idleTimeoutId) { + clearTimeout(idleTimeoutId); + } + idleTimeoutId = setTimeout(() => { + rejectWithMessage( + `Export timed out during ${stage} after ${Math.ceil(idleTimeoutMs! / 1000)} seconds without observable progress`, + ); + }, idleTimeoutMs!); + }; + watchdog.refreshProgress = refreshProgress; + refreshProgress(); + } timeoutId = setTimeout(() => { - reject( - new Error( - `Export timed out during ${stage} after ${Math.ceil(timeoutMs / 60_000)} minutes`, - ), + rejectWithMessage( + `Export timed out during ${stage} after ${Math.ceil(timeoutMs / 60_000)} minutes`, ); }, timeoutMs); }), @@ -401,6 +434,12 @@ export class VideoExporter { if (timeoutId) { clearTimeout(timeoutId); } + if (idleTimeoutId) { + clearTimeout(idleTimeoutId); + } + if (this.activeFinalizationProgressWatchdog === watchdog) { + this.activeFinalizationProgressWatchdog = null; + } } } @@ -709,6 +748,7 @@ export class VideoExporter { ), "native edited audio rendering", "audio", + true, ); editedAudioBuffer = await audioBlob.arrayBuffer(); editedAudioMimeType = audioBlob.type || null; @@ -779,6 +819,7 @@ export class VideoExporter { ), "ffmpeg edited audio rendering", "audio", + true, ); editedAudioBuffer = await audioBlob.arrayBuffer(); editedAudioMimeType = audioBlob.type || null; @@ -890,6 +931,7 @@ export class VideoExporter { renderProgress: number, audioProgress?: number, ) { + this.activeFinalizationProgressWatchdog?.refreshProgress(); this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress, audioProgress); } @@ -1144,6 +1186,7 @@ export class VideoExporter { this.muxer = null; this.audioProcessor = null; + this.activeFinalizationProgressWatchdog = null; this.encodeQueue = 0; this.pendingMuxing = Promise.resolve(); this.nativePendingWrite = Promise.resolve();