import { execFile, spawn } from "node:child_process"; import { once } from "node:events"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import electron from "electron"; import ffmpegStatic from "ffmpeg-static"; const execFileAsync = promisify(execFile); const __dirname = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(__dirname, ".."); const mainEntry = path.join(repoRoot, "dist-electron", "main.cjs"); const rendererEntry = path.join(repoRoot, "dist", "index.html"); const width = parseEvenInteger(process.env.RECORDLY_BENCH_EXPORT_WIDTH ?? "1280", "Width"); const height = parseEvenInteger(process.env.RECORDLY_BENCH_EXPORT_HEIGHT ?? "720", "Height"); const frameRate = parsePositiveInteger(process.env.RECORDLY_BENCH_EXPORT_FPS ?? "60", "Frame rate"); const durationSeconds = parsePositiveInteger( process.env.RECORDLY_BENCH_EXPORT_DURATION ?? "15", "Duration", ); const timeoutMs = parsePositiveInteger( process.env.RECORDLY_BENCH_EXPORT_TIMEOUT_MS ?? "180000", "Timeout", ); const runsPerVariant = parsePositiveInteger(process.env.RECORDLY_BENCH_EXPORT_RUNS ?? "2", "Runs"); const useNativeExport = process.env.RECORDLY_BENCH_EXPORT_USE_NATIVE === "1"; const useWebcamOverlay = process.env.RECORDLY_BENCH_EXPORT_ENABLE_WEBCAM === "1"; const providedInputPath = process.env.RECORDLY_BENCH_EXPORT_INPUT ?? null; const providedWebcamInputPath = process.env.RECORDLY_BENCH_EXPORT_WEBCAM_INPUT ?? null; const keepTempArtifacts = process.env.RECORDLY_BENCH_EXPORT_KEEP_TEMP === "1"; const exportEncodingMode = parseExportEncodingMode( process.env.RECORDLY_BENCH_EXPORT_ENCODING_MODE ?? null, ); const exportQuality = parseExportQuality(process.env.RECORDLY_BENCH_EXPORT_QUALITY ?? null); const exportShadowIntensity = parseExportShadowIntensity( process.env.RECORDLY_BENCH_EXPORT_SHADOW_INTENSITY ?? null, ); const webcamWidth = parseEvenInteger( process.env.RECORDLY_BENCH_EXPORT_WEBCAM_WIDTH ?? "640", "Webcam width", ); const webcamHeight = parseEvenInteger( process.env.RECORDLY_BENCH_EXPORT_WEBCAM_HEIGHT ?? "360", "Webcam height", ); const webcamShadowIntensity = parseExportShadowIntensity( process.env.RECORDLY_BENCH_EXPORT_WEBCAM_SHADOW ?? null, ); const webcamSize = parseExportWebcamSize(process.env.RECORDLY_BENCH_EXPORT_WEBCAM_SIZE ?? null); const MODERN_BACKEND_SWEEP = ["auto", "webcodecs", "breeze"]; const exportPipeline = parseExportPipeline(process.env.RECORDLY_BENCH_EXPORT_PIPELINE ?? null); const exportBackend = parseExportBackend(process.env.RECORDLY_BENCH_EXPORT_BACKEND ?? null); const exportRenderBackend = parseRenderBackend( process.env.RECORDLY_BENCH_EXPORT_RENDER_BACKEND ?? null, ); const exportBackendList = parseExportBackendList( process.env.RECORDLY_BENCH_EXPORT_BACKENDS ?? null, ); const VARIANT_PRESETS = { adaptive: { name: "adaptive" }, baseline: { name: "baseline", maxEncodeQueue: 120, maxDecodeQueue: 10, maxPendingFrames: 24 }, tuned: { name: "tuned", maxEncodeQueue: 240, maxDecodeQueue: 12, maxPendingFrames: 32 }, }; const variantNameList = parseBenchmarkVariantList( process.env.RECORDLY_BENCH_EXPORT_VARIANTS ?? null, ); const variants = variantNameList ? variantNameList.map((variantName) => VARIANT_PRESETS[variantName]) : [VARIANT_PRESETS.baseline, VARIANT_PRESETS.tuned]; function collectUniqueStrings(values) { return [...new Set(values.filter((value) => typeof value === "string" && value.length > 0))]; } function parsePositiveInteger(rawValue, label) { const parsed = Number.parseInt(rawValue, 10); if (!Number.isInteger(parsed) || parsed <= 0) { throw new Error(`${label} must be a positive integer`); } return parsed; } function parseEvenInteger(rawValue, label) { const parsed = parsePositiveInteger(rawValue, label); if (parsed % 2 !== 0) { throw new Error(`${label} must be even`); } return parsed; } function parseExportPipeline(rawValue) { if (rawValue === null || rawValue === "") { return null; } if (rawValue === "legacy" || rawValue === "modern") { return rawValue; } throw new Error("RECORDLY_BENCH_EXPORT_PIPELINE must be 'legacy' or 'modern'"); } function parseExportBackend(rawValue) { if (rawValue === null || rawValue === "") { return null; } if (rawValue === "auto" || rawValue === "webcodecs" || rawValue === "breeze") { return rawValue; } throw new Error("RECORDLY_BENCH_EXPORT_BACKEND must be 'auto', 'webcodecs', or 'breeze'"); } function parseRenderBackend(rawValue) { if (rawValue === null || rawValue === "") { return null; } if (rawValue === "webgl" || rawValue === "webgpu") { return rawValue; } throw new Error("RECORDLY_BENCH_EXPORT_RENDER_BACKEND must be 'webgl' or 'webgpu'"); } function parseExportBackendList(rawValue) { if (rawValue === null || rawValue === "") { return null; } if (rawValue === "all") { return [...MODERN_BACKEND_SWEEP]; } const values = rawValue .split(",") .map((value) => value.trim()) .filter((value) => value.length > 0) .map((value) => parseExportBackend(value)) .filter((value) => value !== null); if (values.length === 0) { throw new Error( "RECORDLY_BENCH_EXPORT_BACKENDS must include at least one of: auto, webcodecs, breeze", ); } return [...new Set(values)]; } function parseBenchmarkVariantList(rawValue) { if (rawValue === null || rawValue === "") { return null; } const values = rawValue .split(",") .map((value) => value.trim()) .filter((value) => value.length > 0); if (values.length === 0) { throw new Error( "RECORDLY_BENCH_EXPORT_VARIANTS must include at least one of: adaptive, baseline, tuned", ); } for (const value of values) { if (!(value in VARIANT_PRESETS)) { throw new Error( "RECORDLY_BENCH_EXPORT_VARIANTS must include only: adaptive, baseline, tuned", ); } } return [...new Set(values)]; } function parseExportEncodingMode(rawValue) { if (rawValue === null || rawValue === "") { return null; } if (rawValue === "fast" || rawValue === "balanced" || rawValue === "quality") { return rawValue; } throw new Error("RECORDLY_BENCH_EXPORT_ENCODING_MODE must be 'fast', 'balanced', or 'quality'"); } function parseExportQuality(rawValue) { if (rawValue === null || rawValue === "") { return null; } if (rawValue === "medium" || rawValue === "good" || rawValue === "high" || rawValue === "source") { return rawValue; } throw new Error("RECORDLY_BENCH_EXPORT_QUALITY must be 'medium', 'good', 'high', or 'source'"); } function parseExportShadowIntensity(rawValue) { if (rawValue === null || rawValue === "") { return null; } const parsed = Number.parseFloat(rawValue); if (!Number.isFinite(parsed) || parsed < 0) { throw new Error("RECORDLY_BENCH_EXPORT_SHADOW_INTENSITY must be a non-negative number"); } return parsed; } function parseExportWebcamSize(rawValue) { if (rawValue === null || rawValue === "") { return null; } const parsed = Number.parseFloat(rawValue); if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 100) { throw new Error("RECORDLY_BENCH_EXPORT_WEBCAM_SIZE must be a number between 0 and 100"); } return parsed; } function summarizeSmokeProgress(progressSamples) { if (!Array.isArray(progressSamples) || progressSamples.length === 0) { return null; } const extractingSamples = progressSamples.filter( (sample) => sample?.phase === "extracting" && typeof sample?.currentFrame === "number" && sample.currentFrame > 1, ); const fpsSource = extractingSamples.length > 0 ? extractingSamples : progressSamples; const renderFpsSamples = fpsSource .map((sample) => sample?.renderFps) .filter((value) => typeof value === "number" && Number.isFinite(value)); const firstSample = progressSamples[0] ?? null; const lastSample = progressSamples.at(-1) ?? null; const firstExtractingSample = extractingSamples[0] ?? null; const lastExtractingSample = extractingSamples.at(-1) ?? null; return { samples: progressSamples.length, extractingSamples: extractingSamples.length, firstElapsedMs: typeof firstSample?.elapsedMs === "number" ? firstSample.elapsedMs : null, lastElapsedMs: typeof lastSample?.elapsedMs === "number" ? lastSample.elapsedMs : null, firstExtractingElapsedMs: typeof firstExtractingSample?.elapsedMs === "number" ? firstExtractingSample.elapsedMs : null, lastExtractingElapsedMs: typeof lastExtractingSample?.elapsedMs === "number" ? lastExtractingSample.elapsedMs : null, firstRenderFps: renderFpsSamples[0] ?? null, lastRenderFps: renderFpsSamples.at(-1) ?? null, minRenderFps: renderFpsSamples.length > 0 ? Math.min(...renderFpsSamples) : null, maxRenderFps: renderFpsSamples.length > 0 ? Math.max(...renderFpsSamples) : null, }; } async function ensureBuildArtifacts() { await fs.access(mainEntry); await fs.access(rendererEntry); } async function createFixtureVideo( ffmpegPath, targetPath, { fixtureWidth = width, fixtureHeight = height, includeAudio = true, videoFilter = `testsrc2=size=${fixtureWidth}x${fixtureHeight}:rate=${frameRate}`, } = {}, ) { const args = ["-y", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", videoFilter]; if (includeAudio) { args.push( "-f", "lavfi", "-i", "sine=frequency=880:sample_rate=48000", "-c:a", "aac", "-b:a", "128k", ); } else { args.push("-an"); } args.push( "-t", String(durationSeconds), "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p", "-movflags", "+faststart", targetPath, ); await execFileAsync(ffmpegPath, args, { timeout: 60_000, maxBuffer: 20 * 1024 * 1024, }); } function parseDurationSeconds(ffmpegOutput) { const match = ffmpegOutput.match(/Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/i); if (!match) { return null; } return ( Number.parseInt(match[1], 10) * 3600 + Number.parseInt(match[2], 10) * 60 + Number.parseFloat(match[3]) ); } async function inspectOutput(ffmpegPath, targetPath) { try { const { stderr } = await execFileAsync( ffmpegPath, ["-hide_banner", "-i", targetPath, "-f", "null", "-"], { timeout: 30_000, maxBuffer: 20 * 1024 * 1024, }, ); return parseDurationSeconds(stderr); } catch (error) { return parseDurationSeconds(String(error?.stderr ?? "")); } } async function readSmokeExportReport(outputPath) { const reportPath = `${outputPath}.report.json`; try { const reportContent = await fs.readFile(reportPath, "utf8"); return { reportPath, report: JSON.parse(reportContent), }; } catch { return null; } } function buildBenchmarkRequests() { if (exportBackendList) { return exportBackendList.map((backend) => ({ pipeline: exportPipeline, backend, label: backend, slug: backend, })); } if (exportBackend) { return [ { pipeline: exportPipeline, backend: exportBackend, label: exportBackend, slug: exportBackend, }, ]; } if (exportPipeline === "modern") { return MODERN_BACKEND_SWEEP.map((backend) => ({ pipeline: exportPipeline, backend, label: backend, slug: backend, })); } return [ { pipeline: exportPipeline, backend: null, label: "default", slug: "default", }, ]; } function formatTableCell(value) { if (Array.isArray(value)) { return value.length > 0 ? value.join(", ") : "-"; } if (value === null || value === undefined || value === "") { return "-"; } return String(value).replace(/\s+/g, " ").trim(); } function printTable(title, columns, rows) { if (!Array.isArray(rows) || rows.length === 0) { return; } const formattedRows = rows.map((row) => columns.map((column) => formatTableCell(column.getValue(row))), ); const widths = columns.map((column, columnIndex) => { const headerWidth = column.header.length; const rowWidth = Math.max(...formattedRows.map((row) => row[columnIndex].length)); return Math.max(headerWidth, rowWidth); }); const divider = `| ${widths.map((width) => "-".repeat(width)).join(" | ")} |`; console.log(`[benchmark-export-queues] ${title}`); console.log( `| ${columns .map((column, columnIndex) => column.header.padEnd(widths[columnIndex])) .join(" | ")} |`, ); console.log(divider); for (const row of formattedRows) { console.log( `| ${row.map((value, columnIndex) => value.padEnd(widths[columnIndex])).join(" | ")} |`, ); } } function formatMs(value) { return typeof value === "number" && Number.isFinite(value) ? `${Math.round(value)} ms` : "-"; } function formatDeltaMs(value) { if (typeof value !== "number" || !Number.isFinite(value)) { return "-"; } const roundedValue = Math.round(value); return `${roundedValue > 0 ? "+" : ""}${roundedValue} ms`; } function formatPercent(value) { return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(1)}%` : "-"; } function formatSeconds(value) { return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(2)} s` : "-"; } function formatMegabytes(value) { return typeof value === "number" && Number.isFinite(value) ? `${(value / (1024 * 1024)).toFixed(2)} MB` : "-"; } function formatBoolean(value) { return value ? "Yes" : "No"; } function calculateDelta(referenceValue, nextValue) { if ( typeof referenceValue !== "number" || !Number.isFinite(referenceValue) || typeof nextValue !== "number" || !Number.isFinite(nextValue) ) { return { deltaMs: null, deltaPercent: null }; } return { deltaMs: nextValue - referenceValue, deltaPercent: referenceValue > 0 ? ((nextValue - referenceValue) / referenceValue) * 100 : null, }; } function buildRequestedConfigRows(benchmarkRequests) { const rows = [ { key: "Width", value: width }, { key: "Height", value: height }, { key: "Frame rate", value: `${frameRate} FPS` }, { key: "Duration", value: `${durationSeconds} s` }, { key: "Timeout", value: formatMs(timeoutMs) }, { key: "Runs per variant", value: runsPerVariant }, { key: "Pipeline", value: exportPipeline ?? "default" }, { key: "Requested backends", value: benchmarkRequests.map((request) => request.label) }, { key: "Render backend", value: exportRenderBackend ?? "default" }, { key: "Backend sweep", value: formatBoolean(benchmarkRequests.length > 1) }, { key: "Encoding mode", value: exportEncodingMode ?? "default" }, { key: "Quality", value: exportQuality ?? "default" }, { key: "Shadow intensity", value: exportShadowIntensity ?? "default" }, { key: "Webcam enabled", value: formatBoolean(useWebcamOverlay) }, { key: "Experimental native override", value: formatBoolean(useNativeExport) }, ]; if (useWebcamOverlay) { rows.push( { key: "Webcam width", value: webcamWidth }, { key: "Webcam height", value: webcamHeight }, { key: "Webcam shadow", value: webcamShadowIntensity ?? "default" }, { key: "Webcam size", value: webcamSize ?? "default" }, ); } return rows; } function printRequestedConfigTable(benchmarkRequests) { printTable( "Requested config", [ { header: "Setting", getValue: (row) => row.key }, { header: "Value", getValue: (row) => row.value }, ], buildRequestedConfigRows(benchmarkRequests), ); } function buildTimingTableRows(benchmarkResults) { return benchmarkResults.flatMap((result) => result.summaries.map((summary) => ({ backend: result.request.backend ?? "default", pipeline: result.request.pipeline ?? "default", variant: summary.variant.name, averageElapsedMs: summary.averageElapsedMs, medianElapsedMs: summary.medianElapsedMs, averageSmokeElapsedMs: summary.averageSmokeElapsedMs, minElapsedMs: summary.minElapsedMs, maxElapsedMs: summary.maxElapsedMs, averageOutputDurationSeconds: summary.averageOutputDurationSeconds, averageSizeBytes: summary.averageSizeBytes, webcamEnabled: summary.webcamEnabled, })), ); } function printTimingSummaryTable(benchmarkResults) { printTable( "Timing summary", [ { header: "Pipeline", getValue: (row) => row.pipeline }, { header: "Backend", getValue: (row) => row.backend }, { header: "Variant", getValue: (row) => row.variant }, { header: "Avg total", getValue: (row) => formatMs(row.averageElapsedMs) }, { header: "Median total", getValue: (row) => formatMs(row.medianElapsedMs) }, { header: "Avg export", getValue: (row) => formatMs(row.averageSmokeElapsedMs) }, { header: "Min", getValue: (row) => formatMs(row.minElapsedMs) }, { header: "Max", getValue: (row) => formatMs(row.maxElapsedMs) }, { header: "Avg output", getValue: (row) => formatSeconds(row.averageOutputDurationSeconds), }, { header: "Avg size", getValue: (row) => formatMegabytes(row.averageSizeBytes) }, { header: "Webcam", getValue: (row) => formatBoolean(row.webcamEnabled) }, ], buildTimingTableRows(benchmarkResults), ); } function buildBackendDetailTableRows(benchmarkResults) { return benchmarkResults.flatMap((result) => result.summaries.map((summary) => ({ backend: result.request.backend ?? "default", pipeline: result.request.pipeline ?? "default", variant: summary.variant.name, encodeQueue: summary.variant.maxEncodeQueue, decodeQueue: summary.variant.maxDecodeQueue, pendingFrames: summary.variant.maxPendingFrames, observedRenderBackends: summary.observedRenderBackends, observedEncodeBackends: summary.observedEncodeBackends, observedEncoders: summary.observedEncoders, })), ); } function printBackendDetailTable(benchmarkResults) { printTable( "Observed backends", [ { header: "Pipeline", getValue: (row) => row.pipeline }, { header: "Backend", getValue: (row) => row.backend }, { header: "Variant", getValue: (row) => row.variant }, { header: "Encode Q", getValue: (row) => row.encodeQueue }, { header: "Decode Q", getValue: (row) => row.decodeQueue }, { header: "Pending", getValue: (row) => row.pendingFrames }, { header: "Render", getValue: (row) => row.observedRenderBackends }, { header: "Encode", getValue: (row) => row.observedEncodeBackends }, { header: "Encoder", getValue: (row) => row.observedEncoders }, ], buildBackendDetailTableRows(benchmarkResults), ); } function buildDeltaTableRows(benchmarkResults) { return benchmarkResults .map((result) => { const baseline = result.summaries.find( (summary) => summary.variant.name === "baseline", ); const tuned = result.summaries.find((summary) => summary.variant.name === "tuned"); if (!baseline || !tuned) { return null; } const averageDelta = calculateDelta(baseline.averageElapsedMs, tuned.averageElapsedMs); const medianDelta = calculateDelta(baseline.medianElapsedMs, tuned.medianElapsedMs); const exportDelta = calculateDelta( baseline.averageSmokeElapsedMs, tuned.averageSmokeElapsedMs, ); return { pipeline: result.request.pipeline ?? "default", backend: result.request.backend ?? "default", averageDeltaMs: averageDelta.deltaMs, averageDeltaPercent: averageDelta.deltaPercent, medianDeltaMs: medianDelta.deltaMs, medianDeltaPercent: medianDelta.deltaPercent, exportDeltaMs: exportDelta.deltaMs, exportDeltaPercent: exportDelta.deltaPercent, }; }) .filter(Boolean); } function printDeltaTable(benchmarkResults) { printTable( "Tuned vs baseline", [ { header: "Pipeline", getValue: (row) => row.pipeline }, { header: "Backend", getValue: (row) => row.backend }, { header: "Avg delta", getValue: (row) => `${formatDeltaMs(row.averageDeltaMs)} (${formatPercent(row.averageDeltaPercent)})`, }, { header: "Median delta", getValue: (row) => `${formatDeltaMs(row.medianDeltaMs)} (${formatPercent(row.medianDeltaPercent)})`, }, { header: "Export delta", getValue: (row) => `${formatDeltaMs(row.exportDeltaMs)} (${formatPercent(row.exportDeltaPercent)})`, }, ], buildDeltaTableRows(benchmarkResults), ); } async function runVariant( ffmpegPath, inputPath, webcamInputPath, benchmarkRequest, variant, runIndex, ) { const outputPath = path.join( path.dirname(inputPath), `${benchmarkRequest.slug}-${variant.name}-${runIndex + 1}-${Date.now()}.mp4`, ); const startedAt = performance.now(); const runLabel = `${benchmarkRequest.label}/${variant.name}#${runIndex + 1}`; const child = spawn(electron, [repoRoot], { cwd: repoRoot, env: { ...process.env, RECORDLY_SMOKE_EXPORT: "1", RECORDLY_SMOKE_EXPORT_INPUT: inputPath, RECORDLY_SMOKE_EXPORT_OUTPUT: outputPath, ...(useNativeExport ? { RECORDLY_SMOKE_EXPORT_USE_NATIVE: "1" } : {}), ...(exportEncodingMode ? { RECORDLY_SMOKE_EXPORT_ENCODING_MODE: exportEncodingMode } : {}), ...(exportQuality ? { RECORDLY_SMOKE_EXPORT_QUALITY: exportQuality } : {}), ...(exportShadowIntensity !== null ? { RECORDLY_SMOKE_EXPORT_SHADOW_INTENSITY: String(exportShadowIntensity) } : {}), ...(webcamInputPath ? { RECORDLY_SMOKE_EXPORT_WEBCAM_INPUT: webcamInputPath } : {}), ...(webcamShadowIntensity !== null ? { RECORDLY_SMOKE_EXPORT_WEBCAM_SHADOW: String(webcamShadowIntensity) } : {}), ...(webcamSize !== null ? { RECORDLY_SMOKE_EXPORT_WEBCAM_SIZE: String(webcamSize) } : {}), ...(benchmarkRequest.pipeline ? { RECORDLY_SMOKE_EXPORT_PIPELINE: benchmarkRequest.pipeline } : {}), ...(benchmarkRequest.backend ? { RECORDLY_SMOKE_EXPORT_BACKEND: benchmarkRequest.backend } : {}), ...(exportRenderBackend ? { RECORDLY_SMOKE_EXPORT_RENDER_BACKEND: exportRenderBackend } : {}), ...(typeof variant.maxEncodeQueue === "number" ? { RECORDLY_SMOKE_EXPORT_MAX_ENCODE_QUEUE: String(variant.maxEncodeQueue) } : {}), ...(typeof variant.maxDecodeQueue === "number" ? { RECORDLY_SMOKE_EXPORT_MAX_DECODE_QUEUE: String(variant.maxDecodeQueue) } : {}), ...(typeof variant.maxPendingFrames === "number" ? { RECORDLY_SMOKE_EXPORT_MAX_PENDING_FRAMES: String(variant.maxPendingFrames) } : {}), }, stdio: ["ignore", "pipe", "pipe"], }); let combinedOutput = ""; child.stdout.on("data", (chunk) => { const text = chunk.toString(); combinedOutput += text; process.stdout.write(`[${runLabel}] ${text}`); }); child.stderr.on("data", (chunk) => { const text = chunk.toString(); combinedOutput += text; process.stderr.write(`[${runLabel}] ${text}`); }); const timeout = setTimeout(() => { child.kill("SIGKILL"); }, timeoutMs); const [exitCode, signal] = await once(child, "close"); clearTimeout(timeout); if (exitCode !== 0) { const signalText = signal ? ` (signal ${signal})` : ""; throw new Error( `${variant.name} run ${runIndex + 1} failed with code ${exitCode ?? "unknown"}${signalText}\n${combinedOutput.trim()}`, ); } const smokeExportReport = await readSmokeExportReport(outputPath); let outputStats; try { outputStats = await fs.stat(outputPath); } catch (error) { const reportSuffix = smokeExportReport ? `\n${JSON.stringify(smokeExportReport.report)}` : ""; throw new Error( `${variant.name} run ${runIndex + 1} did not produce an output file: ${error instanceof Error ? error.message : String(error)}${reportSuffix}`, ); } if (outputStats.size <= 0) { const reportSuffix = smokeExportReport ? `\n${JSON.stringify(smokeExportReport.report)}` : ""; throw new Error( `${variant.name} run ${runIndex + 1} produced an empty output file${reportSuffix}`, ); } const elapsedMs = Math.round(performance.now() - startedAt); const outputDuration = await inspectOutput(ffmpegPath, outputPath); return { elapsedMs, outputPath, sizeBytes: outputStats.size, outputDuration, webcamEnabled: !!webcamInputPath, smokeExportReport: smokeExportReport?.report ?? null, smokeProgressSummary: summarizeSmokeProgress(smokeExportReport?.report?.progressSamples), }; } async function runBenchmarkRequest(ffmpegPath, inputPath, webcamInputPath, benchmarkRequest) { const summaries = []; for (const variant of variants) { const runs = []; for (let index = 0; index < runsPerVariant; index += 1) { console.log( `[benchmark-export-queues] Running ${benchmarkRequest.label}/${variant.name} (${index + 1}/${runsPerVariant}) with encode=${variant.maxEncodeQueue ?? "auto"} decode=${variant.maxDecodeQueue ?? "auto"} pending=${variant.maxPendingFrames ?? "auto"}`, ); runs.push( await runVariant( ffmpegPath, inputPath, webcamInputPath, benchmarkRequest, variant, index, ), ); } const runSummary = summarizeVariantRuns(runs); summaries.push({ variant, runs, ...runSummary, webcamEnabled: useWebcamOverlay, }); } return { request: benchmarkRequest, summaries, }; } function average(values) { return values.reduce((sum, value) => sum + value, 0) / values.length; } function median(values) { if (values.length === 0) { return 0; } const sorted = [...values].sort((left, right) => left - right); const middleIndex = Math.floor(sorted.length / 2); if (sorted.length % 2 === 0) { return (sorted[middleIndex - 1] + sorted[middleIndex]) / 2; } return sorted[middleIndex]; } function summarizeVariantRuns(runs) { const elapsedValues = runs.map((run) => run.elapsedMs); const sizeValues = runs.map((run) => run.sizeBytes); const outputDurationValues = runs .map((run) => run.outputDuration) .filter((value) => typeof value === "number" && Number.isFinite(value)); const smokeElapsedValues = runs .map((run) => run.smokeExportReport?.elapsedMs) .filter((value) => typeof value === "number" && Number.isFinite(value)); return { averageElapsedMs: Math.round(average(elapsedValues)), medianElapsedMs: Math.round(median(elapsedValues)), minElapsedMs: Math.min(...elapsedValues), maxElapsedMs: Math.max(...elapsedValues), averageSizeBytes: Math.round(average(sizeValues)), averageOutputDurationSeconds: outputDurationValues.length > 0 ? average(outputDurationValues) : null, averageSmokeElapsedMs: smokeElapsedValues.length > 0 ? Math.round(average(smokeElapsedValues)) : null, observedRenderBackends: collectUniqueStrings( runs.map((run) => run.smokeExportReport?.metrics?.renderBackend), ), observedEncodeBackends: collectUniqueStrings( runs.map((run) => run.smokeExportReport?.metrics?.encodeBackend), ), observedEncoders: collectUniqueStrings( runs.map((run) => run.smokeExportReport?.metrics?.encoderName), ), }; } async function main() { if (typeof ffmpegStatic !== "string" || ffmpegStatic.length === 0) { throw new Error("ffmpeg-static is unavailable for this platform"); } if (typeof electron !== "string" || electron.length === 0) { throw new Error("The Electron binary is unavailable in this workspace"); } await ensureBuildArtifacts(); const benchmarkRequests = buildBenchmarkRequests(); const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "recordly-export-queue-bench-")); const inputExtension = providedInputPath ? path.extname(providedInputPath) || ".mp4" : ".mp4"; const inputPath = path.join(tempDir, `input${inputExtension}`); const webcamExtension = providedWebcamInputPath ? path.extname(providedWebcamInputPath) || ".mp4" : ".mp4"; const webcamInputPath = useWebcamOverlay ? path.join(tempDir, `webcam${webcamExtension}`) : null; try { console.log("[benchmark-export-queues] Config"); console.log( JSON.stringify({ width, height, frameRate, durationSeconds, timeoutMs, runsPerVariant, requestedPipeline: exportPipeline, requestedBackend: exportBackend, requestedBackends: benchmarkRequests.map((request) => request.label), requestedRenderBackend: exportRenderBackend, backendSweepEnabled: benchmarkRequests.length > 1, requestedEncodingMode: exportEncodingMode, requestedQuality: exportQuality, requestedShadowIntensity: exportShadowIntensity, webcamEnabled: useWebcamOverlay, providedInput: providedInputPath, providedWebcamInput: providedWebcamInputPath, keepTempArtifacts, requestedWebcamShadowIntensity: webcamShadowIntensity, requestedWebcamSize: webcamSize, }), ); printRequestedConfigTable(benchmarkRequests); if (providedInputPath) { console.log(`[benchmark-export-queues] Using provided input video: ${providedInputPath}`); await fs.copyFile(providedInputPath, inputPath); } else { console.log(`[benchmark-export-queues] Generating fixture video: ${inputPath}`); await createFixtureVideo(ffmpegStatic, inputPath); } if (webcamInputPath) { if (providedWebcamInputPath) { console.log( `[benchmark-export-queues] Using provided webcam video: ${providedWebcamInputPath}`, ); await fs.copyFile(providedWebcamInputPath, webcamInputPath); } else { console.log( `[benchmark-export-queues] Generating webcam fixture video: ${webcamInputPath}`, ); await createFixtureVideo(ffmpegStatic, webcamInputPath, { fixtureWidth: webcamWidth, fixtureHeight: webcamHeight, includeAudio: false, videoFilter: `testsrc=size=${webcamWidth}x${webcamHeight}:rate=${frameRate}`, }); } } const benchmarkResults = []; for (const benchmarkRequest of benchmarkRequests) { benchmarkResults.push( await runBenchmarkRequest( ffmpegStatic, inputPath, webcamInputPath, benchmarkRequest, ), ); } console.log("[benchmark-export-queues] Summary"); for (const result of benchmarkResults) { for (const summary of result.summaries) { console.log( JSON.stringify({ requestedPipeline: result.request.pipeline, requestedBackend: result.request.backend, name: summary.variant.name, webcamEnabled: useWebcamOverlay, webcamShadowIntensity, webcamSize, maxEncodeQueue: summary.variant.maxEncodeQueue, maxDecodeQueue: summary.variant.maxDecodeQueue, maxPendingFrames: summary.variant.maxPendingFrames, averageElapsedMs: summary.averageElapsedMs, medianElapsedMs: summary.medianElapsedMs, minElapsedMs: summary.minElapsedMs, maxElapsedMs: summary.maxElapsedMs, averageSizeBytes: summary.averageSizeBytes, averageOutputDurationSeconds: summary.averageOutputDurationSeconds, averageSmokeElapsedMs: summary.averageSmokeElapsedMs, observedRenderBackends: summary.observedRenderBackends, observedEncodeBackends: summary.observedEncodeBackends, observedEncoders: summary.observedEncoders, runs: summary.runs.map((run) => ({ elapsedMs: run.elapsedMs, sizeBytes: run.sizeBytes, outputDuration: run.outputDuration, smokeExportReport: run.smokeExportReport, smokeProgressSummary: run.smokeProgressSummary, })), }), ); } } printTimingSummaryTable(benchmarkResults); printBackendDetailTable(benchmarkResults); printDeltaTable(benchmarkResults); for (const result of benchmarkResults) { if (result.summaries.length < 2) { continue; } const baseline = result.summaries[0]; const tuned = result.summaries[1]; const { deltaMs, deltaPercent: percent } = calculateDelta( baseline.averageElapsedMs, tuned.averageElapsedMs, ); const { deltaMs: medianDeltaMs, deltaPercent: medianPercent } = calculateDelta( baseline.medianElapsedMs, tuned.medianElapsedMs, ); const backendLabel = result.request.backend ?? "default"; console.log( `[benchmark-export-queues] ${backendLabel} tuned vs baseline: ${deltaMs}ms (${typeof percent === "number" ? percent.toFixed(1) : "-"}%)`, ); console.log( `[benchmark-export-queues] ${backendLabel} tuned vs baseline (median): ${medianDeltaMs}ms (${typeof medianPercent === "number" ? medianPercent.toFixed(1) : "-"}%)`, ); } } finally { if (keepTempArtifacts) { console.log(`[benchmark-export-queues] Preserved temp artifacts: ${tempDir}`); } else { await fs.rm(tempDir, { recursive: true, force: true }); } } } main().catch((error) => { console.error( `[benchmark-export-queues] ${error instanceof Error ? error.message : String(error)}`, ); process.exitCode = 1; });