diff --git a/.gitignore b/.gitignore index 3c1e1d47..605ab2f5 100644 --- a/.gitignore +++ b/.gitignore @@ -34,9 +34,13 @@ vite.config.js vite.config.d.ts # Native capture build artifacts -electron/native/wgc-capture/build/ -electron/native/cursor-monitor/build/ - -# Local debug helpers -tmp-*.ps1 -.tmp-*.ps1 +electron/native/wgc-capture/build/ +electron/native/cursor-monitor/build/ +electron/native/gpu-export-probe/build/ +electron/native/bin/*/whisper-* +electron/native/bin/*/whisper-runtime.json + +# Local debug helpers +tmp-*.ps1 +.tmp-*.ps1 +gpu-export-probe.mp4 diff --git a/electron-builder.json5 b/electron-builder.json5 index 4c9e1bba..bef41d48 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -17,10 +17,11 @@ "output": "release" }, "files": [ - "dist", - "dist-electron", - "electron/native", - "!*.png", + "dist", + "dist-electron", + "electron/native", + "!electron/native/**/build/**", + "!*.png", "!preview*.png", "!*.md", "!README.md", diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 8b634980..a1aa8880 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -82,6 +82,110 @@ interface RendererFfmpegAudioMuxMetrics { muxedVideoBytes?: number; } +interface RendererWindowsGpuExportSummary { + success?: boolean; + width?: number; + height?: number; + fps?: number; + seconds?: number; + mediaMs?: number; + frames?: number; + gpuDecodeSurface?: boolean; + webcamOverlay?: boolean; + cursorOverlay?: boolean; + zoomOverlay?: boolean; + surfacePoolSize?: number; + adapterIndex?: number; + adapterVendorId?: number; + adapterDeviceId?: number; + adapterDedicatedVideoMemoryMB?: number; + encoderBackend?: string; + encoderTuningApplied?: boolean; + nvencOutputBytes?: number; + initializeMs?: number; + initCoInitializeMs?: number; + initMfStartupMs?: number; + initD3DDeviceMs?: number; + initSourceReaderMs?: number; + initWebcamReaderMs?: number; + initVideoProcessorMs?: number; + initTexturesMs?: number; + initShaderPipelineMs?: number; + initSinkWriterMs?: number; + totalMs?: number; + readMs?: number; + clearMs?: number; + videoProcessMs?: number; + writeSampleMs?: number; + finalizeMs?: number; + realtimeMultiplier?: number; +} + +interface RendererNativeStaticLayoutChunkMetric { + index: number; + startSec: number; + durationSec: number; + backend: + | "cuda-overlay" + | "cuda-scale-cpu-pad" + | "cuda-static-composite" + | "nvidia-cuda-compositor" + | "windows-d3d11-compositor"; + elapsedMs: number; + outputBytes: number; + fallbackReason?: string; + windowsGpuSummary?: RendererWindowsGpuExportSummary; +} + +interface RendererNativeStaticLayoutMetrics extends RendererFfmpegAudioMuxMetrics { + chunkCount: number; + chunkDurationSec: number; + chunkExecMs: number; + concatExecMs?: number; + staticAssetExecMs?: number; + fallbackChunkCount: number; + videoOnlyBytes?: number; + chunks: RendererNativeStaticLayoutChunkMetric[]; +} + +interface RendererNativeStaticLayoutProgress { + sessionId?: string; + backend?: RendererNativeStaticLayoutChunkMetric["backend"]; + elapsedMs?: number; + averageFps?: number; + instantFps?: number; + intervalMs?: number; + intervalFrames?: number; + intervalDecodeWallMs?: number; + intervalEncodeMs?: number; + intervalPipelineWaitMs?: number; + intervalCompositeMs?: number; + intervalNvencMs?: number; + intervalPacketWriteMs?: number; + intervalWebcamDecodeMs?: number; + intervalWebcamCopyMs?: number; + intervalRoiCompositeFrames?: number; + intervalMonolithicCompositeFrames?: number; + intervalCopyCompositeFrames?: number; + currentFrame: number; + totalFrames: number; + percentage: number; +} + +interface RendererNativeVideoMetadataProbe { + width: number; + height: number; + duration: number; + mediaStartTime?: number; + streamStartTime?: number; + streamDuration?: number; + frameRate: number; + codec: string; + hasAudio: boolean; + audioCodec?: string; + audioSampleRate?: number; +} + interface Window { electronAPI: { hudOverlaySetIgnoreMouse: (ignore: boolean) => void; @@ -193,6 +297,82 @@ interface Window { generateWallpaperThumbnail: ( filePath: string, ) => Promise<{ success: boolean; data?: Uint8Array; error?: string }>; + probeNativeVideoMetadata: (filePath: string) => Promise<{ + success: boolean; + metadata?: RendererNativeVideoMetadataProbe; + error?: string; + }>; + nativeStaticLayoutExport: (options: { + sessionId?: string; + inputPath: string; + width: number; + height: number; + frameRate: number; + bitrate: number; + encodingMode: "fast" | "balanced" | "quality"; + durationSec: number; + contentWidth: number; + contentHeight: number; + offsetX: number; + offsetY: number; + backgroundColor: string; + backgroundImagePath?: string | null; + borderRadius?: number; + shadowIntensity?: number; + webcamInputPath?: string | null; + webcamLeft?: number; + webcamTop?: number; + webcamSize?: number; + webcamRadius?: number; + webcamShadowIntensity?: number; + webcamMirror?: boolean; + webcamTimeOffsetMs?: number; + cursorTelemetry?: Array<{ + timeMs: number; + cx: number; + cy: number; + cursorTypeIndex?: number; + bounceScale?: number; + }>; + cursorSize?: number; + cursorAtlasPngDataUrl?: string | null; + cursorAtlasEntries?: Array<{ + index: number; + x: number; + y: number; + width: number; + height: number; + anchorX: number; + anchorY: number; + aspectRatio: number; + }>; + zoomTelemetry?: Array<{ timeMs: number; scale: number; x: number; y: number }>; + chunkDurationSec?: number; + experimentalWindowsGpuCompositor?: boolean; + audioOptions?: { + audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; + audioSourcePath?: string | null; + audioSourceSampleRate?: number; + outputDurationSec?: number; + trimSegments?: Array<{ startMs: number; endMs: number }>; + editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; + editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; + editedAudioData?: ArrayBuffer; + editedAudioMimeType?: string | null; + }; + }) => Promise<{ + success: boolean; + tempPath?: string; + encoderName?: string; + error?: string; + metrics?: RendererNativeStaticLayoutMetrics; + }>; + nativeStaticLayoutExportCancel: (sessionId: string) => Promise<{ + success: boolean; + }>; + onNativeStaticLayoutExportProgress: ( + callback: (progress: RendererNativeStaticLayoutProgress) => void, + ) => () => void; nativeVideoExportStart: (options: { width: number; height: number; @@ -216,6 +396,7 @@ interface Window { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; audioSourceSampleRate?: number; + outputDurationSec?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; diff --git a/electron/ipc/export/native-video.test.ts b/electron/ipc/export/native-video.test.ts new file mode 100644 index 00000000..3e004f1b --- /dev/null +++ b/electron/ipc/export/native-video.test.ts @@ -0,0 +1,252 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("electron", () => ({ + app: { + getAppPath: () => process.cwd(), + getPath: () => process.env.TEMP ?? process.cwd(), + isPackaged: false, + }, +})); + +vi.mock("../ffmpeg/binary", () => ({ + getFfmpegBinaryPath: () => "ffmpeg", +})); + +import { + buildNativeVideoAudioMuxArgs, + normalizeNativeStaticLayoutBackground, + parseFfmpegDurationSeconds, + parseFfmpegFrameRate, + parseNativeVideoMetadataProbeOutput, + parseNvidiaCudaExportSummary, + parseWindowsGpuExportProgressLine, + parseWindowsGpuExportSummary, +} from "./native-video"; + +describe("normalizeNativeStaticLayoutBackground", () => { + it("falls back to a solid background when the configured image file is missing", async () => { + const normalized = await normalizeNativeStaticLayoutBackground({ + inputPath: "input.mp4", + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 8_000_000, + encodingMode: "quality", + durationSec: 10, + contentWidth: 1600, + contentHeight: 900, + offsetX: 160, + offsetY: 90, + backgroundColor: "#101010", + backgroundImagePath: "Z:\\recordly-missing-wallpaper\\midnight-8.jpg", + }); + + expect(normalized.backgroundImagePath).toBeNull(); + expect(normalized.backgroundColor).toBe("#ffffff"); + }); +}); + +describe("buildNativeVideoAudioMuxArgs", () => { + it("stream-copies source audio and preserves the requested video duration", () => { + const args = buildNativeVideoAudioMuxArgs("video.mp4", "source.mp4", "out.mp4", { + audioMode: "copy-source", + outputDurationSec: 60, + }); + + expect(args).toEqual( + expect.arrayContaining([ + "-map", + "0:v:0", + "-map", + "1:a:0", + "-c:v", + "copy", + "-c:a", + "copy", + "-t", + "60.000", + ]), + ); + expect(args).not.toContain("-shortest"); + }); + + it("does not shorten copy-source muxes when no explicit duration is available", () => { + const args = buildNativeVideoAudioMuxArgs("video.mp4", "source.mp4", "out.mp4", { + audioMode: "copy-source", + }); + + expect(args).toEqual(expect.arrayContaining(["-c:a", "copy"])); + expect(args).not.toContain("-shortest"); + }); + + it("keeps filtered audio on the AAC encode path", () => { + const args = buildNativeVideoAudioMuxArgs("video.mp4", "source.mp4", "out.mp4", { + audioMode: "trim-source", + trimSegments: [{ startMs: 0, endMs: 1_000 }], + outputDurationSec: 1, + }); + + expect(args).toEqual(expect.arrayContaining(["-filter_complex"])); + expect(args).toEqual(expect.arrayContaining(["-c:a", "aac", "-b:a", "192k"])); + }); +}); + +describe("parseWindowsGpuExportSummary", () => { + it("returns the last JSON summary from helper stdout", () => { + const summary = parseWindowsGpuExportSummary( + [ + "initializing", + '{"success":true,"frames":30,"totalMs":1000,"realtimeMultiplier":2}', + "cleanup", + '{"success":true,"frames":60,"surfacePoolSize":12,"readMs":12.5,"videoProcessMs":30,"writeSampleMs":40,"finalizeMs":5,"realtimeMultiplier":4}', + ].join("\n"), + ); + + expect(summary).toEqual({ + success: true, + frames: 60, + surfacePoolSize: 12, + readMs: 12.5, + videoProcessMs: 30, + writeSampleMs: 40, + finalizeMs: 5, + realtimeMultiplier: 4, + }); + }); + + it("returns null when helper stdout has no valid JSON summary", () => { + expect(parseWindowsGpuExportSummary("initializing\nnot-json")).toBeNull(); + expect(parseWindowsGpuExportSummary("")).toBeNull(); + }); +}); + +describe("parseNvidiaCudaExportSummary", () => { + it("parses the pretty JSON summary emitted by the CUDA lab wrapper", () => { + const summary = parseNvidiaCudaExportSummary( + [ + "preflight", + JSON.stringify( + { + success: true, + fps: 30, + durationSec: 10, + targetFrames: 300, + timingsMs: { nativeEncode: 920, mux: 45, endToEnd: 1400 }, + nativeSummary: { success: true, frames: 300, fps: 326.1 }, + }, + null, + 2, + ), + ].join("\n"), + ); + + expect(summary?.success).toBe(true); + expect(summary?.targetFrames).toBe(300); + expect(summary?.timingsMs?.nativeEncode).toBe(920); + expect(summary?.nativeSummary?.fps).toBe(326.1); + }); + + it("returns null when the wrapper output has no JSON object", () => { + expect(parseNvidiaCudaExportSummary("native probe failed before summary")).toBeNull(); + }); +}); + +describe("parseWindowsGpuExportProgressLine", () => { + it("parses bounded helper progress lines", () => { + expect( + parseWindowsGpuExportProgressLine( + 'PROGRESS {"currentFrame":30,"totalFrames":60,"percentage":50,"averageFps":240.5,"instantFps":180.25,"intervalMs":166.4,"intervalFrames":30,"intervalEncodeMs":120.2,"intervalPipelineWaitMs":46.2,"intervalMonolithicCompositeFrames":0}', + ), + ).toEqual({ + currentFrame: 30, + totalFrames: 60, + percentage: 50, + averageFps: 240.5, + instantFps: 180.25, + intervalMs: 166.4, + intervalFrames: 30, + intervalEncodeMs: 120.2, + intervalPipelineWaitMs: 46.2, + intervalMonolithicCompositeFrames: 0, + }); + }); + + it("ignores non-progress or malformed helper stderr", () => { + expect(parseWindowsGpuExportProgressLine("warning: encoder selected")).toBeNull(); + expect(parseWindowsGpuExportProgressLine("PROGRESS not-json")).toBeNull(); + expect( + parseWindowsGpuExportProgressLine( + 'PROGRESS {"currentFrame":1,"totalFrames":0,"percentage":999}', + ), + ).toBeNull(); + }); +}); + +describe("parseNativeVideoMetadataProbeOutput", () => { + it("parses FFmpeg input metadata with video and audio streams", () => { + const metadata = parseNativeVideoMetadataProbeOutput(` +Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'recording.mp4': + Metadata: + major_brand : isom + Duration: 00:06:04.25, start: 0.000000, bitrate: 3938 kb/s + Stream #0:0[0x1](und): Video: h264 (High) (avc1 / 0x31637661), yuv420p(progressive), 1920x1080, 3720 kb/s, 46.05 fps, 60 tbr, 90k tbn (default) + Stream #0:1[0x2](und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, stereo, fltp, 192 kb/s (default) +`); + + expect(metadata).toEqual({ + width: 1920, + height: 1080, + duration: 364.25, + mediaStartTime: 0, + streamStartTime: 0, + streamDuration: 364.25, + frameRate: 46.05, + codec: "h264 (High) (avc1 / 0x31637661)", + hasAudio: true, + audioCodec: "aac (LC) (mp4a / 0x6134706D)", + audioSampleRate: 48000, + }); + }); + + it("parses video-only metadata and falls back to tbr when fps is absent", () => { + const metadata = parseNativeVideoMetadataProbeOutput(` +Input #0, matroska,webm, from 'recording.webm': + Duration: 00:00:10.50, start: 0.023000, bitrate: 1000 kb/s + Stream #0:0: Video: vp9, yuv420p, 1280x720, 30 tbr, 1k tbn +`); + + expect(metadata).toEqual({ + width: 1280, + height: 720, + duration: 10.5, + mediaStartTime: 0.023, + streamStartTime: 0.023, + streamDuration: 10.5, + frameRate: 30, + codec: "vp9", + hasAudio: false, + audioCodec: undefined, + audioSampleRate: undefined, + }); + }); + + it("rejects output without usable video metadata", () => { + expect(parseNativeVideoMetadataProbeOutput("Duration: N/A")).toBeNull(); + expect(parseNativeVideoMetadataProbeOutput("not a media file")).toBeNull(); + }); +}); + +describe("parseFfmpegDurationSeconds", () => { + it("parses HH:MM:SS timestamps", () => { + expect(parseFfmpegDurationSeconds("01:02:03.5")).toBe(3723.5); + expect(parseFfmpegDurationSeconds("bad")).toBeNull(); + }); +}); + +describe("parseFfmpegFrameRate", () => { + it("prefers fps and falls back to tbr", () => { + expect(parseFfmpegFrameRate("Video: h264, 1920x1080, 59.94 fps, 60 tbr")).toBe(59.94); + expect(parseFfmpegFrameRate("Video: h264, 1920x1080, 30 tbr")).toBe(30); + expect(parseFfmpegFrameRate("Video: h264")).toBeNull(); + }); +}); diff --git a/electron/ipc/export/native-video.ts b/electron/ipc/export/native-video.ts index a9205369..d7598f28 100644 --- a/electron/ipc/export/native-video.ts +++ b/electron/ipc/export/native-video.ts @@ -1,31 +1,45 @@ import type { ChildProcessByStdio } from "node:child_process"; import { execFile, spawn } from "node:child_process"; import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { performance } from "node:perf_hooks"; import type { Readable, Writable } from "node:stream"; import { promisify } from "node:util"; import type { WebContents } from "electron"; -import { app } from "electron"; +import { app, powerSaveBlocker } from "electron"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import type { NativeExportEncodingMode, + NativeStaticLayoutBackend, + NativeStaticLayoutExportArgsConfig, NativeVideoAudioMuxMetrics, NativeVideoExportFinishOptions, } from "../nativeVideoExport"; import { buildEditedTrackSourceAudioFilter, + buildNativeConcatArgs, + buildNativeCudaOverlayStaticLayoutArgs, + buildNativeCudaScaleCpuPadStaticLayoutArgs, + buildNativePrecompositedStaticLayoutArgs, + buildNativeStaticBackgroundRenderArgs, + buildNativeStaticLayoutChunks, buildNativeVideoExportArgs, buildTrimmedSourceAudioFilter, + createNativeSquircleMaskPgmBuffer, getEditedAudioExtension, getNativeVideoInputByteSize, getPreferredNativeVideoEncoders, + isNativeCudaOutOfMemory, parseAvailableFfmpegEncoders, } from "../nativeVideoExport"; import { cachedNativeVideoEncoder, setCachedNativeVideoEncoder } from "../state"; const execFileAsync = promisify(execFile); const getNowMs = () => performance.now(); +const formatFfmpegSeconds = (milliseconds: number) => (milliseconds / 1000).toFixed(3); +const MISSING_NATIVE_STATIC_BACKGROUND_COLOR = "#ffffff"; +const NATIVE_EXPORT_HIGH_PRIORITY = os.constants.priority.PRIORITY_HIGH; export type NativeVideoExportSession = { ffmpegProcess: ChildProcessByStdio; @@ -46,6 +60,213 @@ export type NativeVideoExportSession = { export const nativeVideoExportSessions = new Map(); +export interface NativeStaticLayoutExportOptions { + sessionId?: string; + inputPath: string; + width: number; + height: number; + frameRate: number; + bitrate: number; + encodingMode: NativeExportEncodingMode; + durationSec: number; + contentWidth: number; + contentHeight: number; + offsetX: number; + offsetY: number; + backgroundColor: string; + backgroundImagePath?: string | null; + borderRadius?: number; + shadowIntensity?: number; + webcamInputPath?: string | null; + webcamLeft?: number; + webcamTop?: number; + webcamSize?: number; + webcamRadius?: number; + webcamShadowIntensity?: number; + webcamMirror?: boolean; + webcamTimeOffsetMs?: number; + cursorTelemetry?: Array<{ + timeMs: number; + cx: number; + cy: number; + cursorTypeIndex?: number; + bounceScale?: number; + }>; + cursorTelemetryPath?: string | null; + cursorSize?: number; + cursorAtlasPngDataUrl?: string | null; + cursorAtlasPath?: string | null; + cursorAtlasEntries?: Array<{ + index: number; + x: number; + y: number; + width: number; + height: number; + anchorX: number; + anchorY: number; + aspectRatio: number; + }>; + cursorAtlasMetadataPath?: string | null; + zoomTelemetry?: Array<{ timeMs: number; scale: number; x: number; y: number }>; + zoomTelemetryPath?: string | null; + chunkDurationSec?: number; + experimentalWindowsGpuCompositor?: boolean; + audioOptions?: NativeVideoExportFinishOptions; +} + +export interface NativeStaticLayoutExportProgress { + sessionId?: string; + backend?: NativeStaticLayoutBackend; + elapsedMs?: number; + averageFps?: number; + instantFps?: number; + intervalMs?: number; + intervalFrames?: number; + intervalDecodeWallMs?: number; + intervalEncodeMs?: number; + intervalPipelineWaitMs?: number; + intervalCompositeMs?: number; + intervalNvencMs?: number; + intervalPacketWriteMs?: number; + intervalWebcamDecodeMs?: number; + intervalWebcamCopyMs?: number; + intervalRoiCompositeFrames?: number; + intervalMonolithicCompositeFrames?: number; + intervalCopyCompositeFrames?: number; + currentFrame: number; + totalFrames: number; + percentage: number; +} + +export interface WindowsGpuExportSummary { + success?: boolean; + width?: number; + height?: number; + fps?: number; + seconds?: number; + mediaMs?: number; + frames?: number; + gpuDecodeSurface?: boolean; + webcamOverlay?: boolean; + cursorOverlay?: boolean; + cursorAtlas?: boolean; + zoomOverlay?: boolean; + surfacePoolSize?: number; + adapterIndex?: number; + adapterVendorId?: number; + adapterDeviceId?: number; + adapterDedicatedVideoMemoryMB?: number; + encoderBackend?: string; + encoderTuningApplied?: boolean; + nvencOutputBytes?: number; + initializeMs?: number; + initCoInitializeMs?: number; + initMfStartupMs?: number; + initD3DDeviceMs?: number; + initSourceReaderMs?: number; + initWebcamReaderMs?: number; + initVideoProcessorMs?: number; + initTexturesMs?: number; + initShaderPipelineMs?: number; + initSinkWriterMs?: number; + totalMs?: number; + readMs?: number; + clearMs?: number; + videoProcessMs?: number; + writeSampleMs?: number; + finalizeMs?: number; + realtimeMultiplier?: number; +} + +export interface NvidiaCudaExportSummary { + success?: boolean; + inputPath?: string; + outputPath?: string; + fps?: number; + bitrateMbps?: number; + durationSec?: number; + targetFrames?: number; + timingsMs?: { + demux?: number; + backgroundConvert?: number; + cursorAtlas?: number; + webcamConvert?: number; + webcamDemux?: number; + nativeEncode?: number; + mux?: number; + endToEnd?: number; + }; + nativeSummary?: { + success?: boolean; + frames?: number; + totalMs?: number; + fps?: number; + measuredFps?: number; + mappedDisplayFrames?: number; + selectedDisplayFrames?: number; + skippedDisplayFrames?: number; + roiCompositeFrames?: number; + monolithicCompositeFrames?: number; + copyCompositeFrames?: number; + cursorAtlas?: boolean; + webcamOverlay?: boolean; + zoomOverlay?: boolean; + zoomSamples?: number; + }; + nativeProcessPriorityBoosted?: boolean; + appRuntimeGuard?: { + powerGuardStarted?: boolean; + wrapperProcessPriorityBoosted?: boolean; + nativeProcessPriorityBoosted?: boolean; + }; + gpuSamples?: unknown[]; + gpuSummary?: unknown; + outputVideo?: unknown; + outputAudio?: unknown; +} + +export interface NativeVideoMetadataProbe { + width: number; + height: number; + duration: number; + mediaStartTime?: number; + streamStartTime?: number; + streamDuration?: number; + frameRate: number; + codec: string; + hasAudio: boolean; + audioCodec?: string; + audioSampleRate?: number; +} + +export interface NativeStaticLayoutChunkMetric { + index: number; + startSec: number; + durationSec: number; + backend: NativeStaticLayoutBackend; + elapsedMs: number; + outputBytes: number; + fallbackReason?: string; + windowsGpuSummary?: WindowsGpuExportSummary; + nvidiaCudaSummary?: NvidiaCudaExportSummary; +} + +export interface NativeStaticLayoutExportMetrics extends NativeVideoAudioMuxMetrics { + chunkCount: number; + chunkDurationSec: number; + chunkExecMs: number; + concatExecMs?: number; + staticAssetExecMs?: number; + fallbackChunkCount: number; + videoOnlyBytes?: number; + chunks: NativeStaticLayoutChunkMetric[]; +} + +export interface NativeStaticLayoutExportSession { + terminating: boolean; + currentProcess: ReturnType | null; +} + export function cleanupNativeVideoExportSessions() { for (const [sessionId, session] of nativeVideoExportSessions) { session.terminating = true; @@ -63,6 +284,378 @@ export function cleanupNativeVideoExportSessions() { } nativeVideoExportSessions.delete(sessionId); } + + for (const [sessionId, session] of nativeStaticLayoutExportSessions) { + session.terminating = true; + try { + session.currentProcess?.kill("SIGKILL"); + } catch { + /* process may already be exited */ + } + nativeStaticLayoutExportSessions.delete(sessionId); + } +} + +export function parseWindowsGpuExportSummary(stdout: string): WindowsGpuExportSummary | null { + const summaryLine = stdout + .trim() + .split(/\r?\n/) + .reverse() + .find((line) => line.trim().startsWith("{")); + if (!summaryLine) { + return null; + } + + try { + const parsed = JSON.parse(summaryLine) as WindowsGpuExportSummary; + return parsed && typeof parsed === "object" ? parsed : null; + } catch { + return null; + } +} + +export function parseNvidiaCudaExportSummary(stdout: string): NvidiaCudaExportSummary | null { + const trimmed = stdout.trim(); + const startIndex = trimmed.indexOf("{"); + const endIndex = trimmed.lastIndexOf("}"); + if (startIndex === -1 || endIndex <= startIndex) { + return null; + } + + try { + const parsed = JSON.parse( + trimmed.slice(startIndex, endIndex + 1), + ) as NvidiaCudaExportSummary; + return parsed && typeof parsed === "object" ? parsed : null; + } catch { + return null; + } +} + +function shouldPersistNvidiaCudaExportDiagnostics() { + return ( + process.env.RECORDLY_NVIDIA_CUDA_EXPORT_DIAGNOSTICS === "1" || + process.env.RECORDLY_NATIVE_EXPORT_DIAGNOSTICS === "1" + ); +} + +function getSafeDiagnosticsFileSegment(value: string | undefined) { + const safe = (value || "session").replace(/[^a-zA-Z0-9._-]+/g, "-").slice(0, 96); + return safe || "session"; +} + +function getCliArgValue(args: string[], name: string) { + const index = args.indexOf(name); + if (index === -1 || index + 1 >= args.length) { + return null; + } + return args[index + 1] || null; +} + +async function persistNvidiaCudaExportDiagnostics(params: { + args: string[]; + code: number | null; + elapsedMs: number; + outputPath: string; + sessionId?: string; + signal: NodeJS.Signals | null; + startedAtIso: string; + stderr: string; + stdout: string; + summary: NvidiaCudaExportSummary | null; +}) { + if (!shouldPersistNvidiaCudaExportDiagnostics()) { + return; + } + + const diagnosticsDirectory = path.join(app.getPath("userData"), "native-export-diagnostics"); + const filePrefix = `${Date.now()}-${getSafeDiagnosticsFileSegment(params.sessionId)}`; + const manifest = { + backend: "nvidia-cuda-compositor", + startedAt: params.startedAtIso, + completedAt: new Date().toISOString(), + elapsedMs: Number(params.elapsedMs.toFixed(2)), + sessionId: params.sessionId ?? null, + outputPath: params.outputPath, + exitCode: params.code, + signal: params.signal, + args: params.args, + summary: params.summary, + }; + + try { + await fs.mkdir(diagnosticsDirectory, { recursive: true }); + const artifactsDirectory = path.join(diagnosticsDirectory, `${filePrefix}.artifacts`); + const artifactArgs = [ + "--cursor-json", + "--cursor-atlas-png", + "--cursor-atlas-metadata", + "--zoom-telemetry", + ]; + const artifactCopies = artifactArgs + .map((argName) => { + const sourcePath = getCliArgValue(params.args, argName); + return sourcePath + ? { + argName, + sourcePath, + outputPath: path.join( + artifactsDirectory, + `${getSafeDiagnosticsFileSegment(argName.replace(/^--/, ""))}-${path.basename(sourcePath)}`, + ), + } + : null; + }) + .filter((artifact): artifact is NonNullable => Boolean(artifact)); + await fs.mkdir(artifactsDirectory, { recursive: true }).catch(() => undefined); + await Promise.allSettled([ + fs.writeFile( + path.join(diagnosticsDirectory, `${filePrefix}.manifest.json`), + `${JSON.stringify(manifest, null, 2)}\n`, + ), + fs.writeFile( + path.join(diagnosticsDirectory, `${filePrefix}.stdout.json`), + params.stdout, + ), + fs.writeFile( + path.join(diagnosticsDirectory, `${filePrefix}.stderr.log`), + params.stderr, + ), + ...artifactCopies.map((artifact) => + fs.copyFile(artifact.sourcePath, artifact.outputPath).catch(() => undefined), + ), + ]); + } catch (error) { + console.warn("[native-static-layout-export] Failed to persist NVIDIA CUDA diagnostics", error); + } +} + +export function parseWindowsGpuExportProgressLine( + line: string, +): NativeStaticLayoutExportProgress | null { + const trimmed = line.trim(); + const prefix = "PROGRESS "; + if (!trimmed.startsWith(prefix)) { + return null; + } + + try { + const parsed = JSON.parse(trimmed.slice(prefix.length)) as { + currentFrame?: unknown; + totalFrames?: unknown; + percentage?: unknown; + averageFps?: unknown; + instantFps?: unknown; + intervalMs?: unknown; + intervalFrames?: unknown; + intervalDecodeWallMs?: unknown; + intervalEncodeMs?: unknown; + intervalPipelineWaitMs?: unknown; + intervalCompositeMs?: unknown; + intervalNvencMs?: unknown; + intervalPacketWriteMs?: unknown; + intervalWebcamDecodeMs?: unknown; + intervalWebcamCopyMs?: unknown; + intervalRoiCompositeFrames?: unknown; + intervalMonolithicCompositeFrames?: unknown; + intervalCopyCompositeFrames?: unknown; + }; + const currentFrame = Number(parsed.currentFrame); + const totalFrames = Number(parsed.totalFrames); + const percentage = Number(parsed.percentage); + if ( + !Number.isFinite(currentFrame) || + !Number.isFinite(totalFrames) || + !Number.isFinite(percentage) || + totalFrames <= 0 + ) { + return null; + } + + const progress: NativeStaticLayoutExportProgress = { + currentFrame: Math.max(0, Math.floor(currentFrame)), + totalFrames: Math.max(1, Math.floor(totalFrames)), + percentage: Math.min(100, Math.max(0, percentage)), + }; + const optionalNumberFields = [ + "averageFps", + "instantFps", + "intervalMs", + "intervalFrames", + "intervalDecodeWallMs", + "intervalEncodeMs", + "intervalPipelineWaitMs", + "intervalCompositeMs", + "intervalNvencMs", + "intervalPacketWriteMs", + "intervalWebcamDecodeMs", + "intervalWebcamCopyMs", + "intervalRoiCompositeFrames", + "intervalMonolithicCompositeFrames", + "intervalCopyCompositeFrames", + ] as const; + for (const field of optionalNumberFields) { + const value = Number(parsed[field]); + if (Number.isFinite(value) && value >= 0) { + progress[field] = value; + } + } + return progress; + } catch { + return null; + } +} + +function startNativeStaticLayoutExportPowerGuard() { + try { + const blockerId = powerSaveBlocker.start("prevent-app-suspension"); + return { + started: true, + release: () => { + if (powerSaveBlocker.isStarted(blockerId)) { + powerSaveBlocker.stop(blockerId); + } + }, + }; + } catch (error) { + console.warn("[native-static-layout-export] Failed to start power guard", error); + return { + started: false, + release: () => undefined, + }; + } +} + +function setNativeStaticLayoutExportProcessPriority( + pid: number | undefined, + label: string, +) { + if (!pid) { + return false; + } + + try { + os.setPriority(pid, NATIVE_EXPORT_HIGH_PRIORITY); + return true; + } catch (error) { + console.warn( + `[native-static-layout-export] Failed to raise ${label} priority`, + error, + ); + return false; + } +} + +export const nativeStaticLayoutExportSessions = new Map(); + +export function parseFfmpegDurationSeconds(value: string): number | null { + const parts = value.trim().split(":"); + if (parts.length !== 3) { + return null; + } + + const [hours, minutes, seconds] = parts.map(Number); + if (![hours, minutes, seconds].every(Number.isFinite)) { + return null; + } + + return hours * 3600 + minutes * 60 + seconds; +} + +export function parseFfmpegFrameRate(line: string): number | null { + const fpsMatch = line.match(/,\s*([0-9]+(?:\.[0-9]+)?)\s*fps\b/i); + if (fpsMatch) { + const frameRate = Number(fpsMatch[1]); + return Number.isFinite(frameRate) && frameRate > 0 ? frameRate : null; + } + + const tbrMatch = line.match(/,\s*([0-9]+(?:\.[0-9]+)?)\s*tbr\b/i); + if (tbrMatch) { + const frameRate = Number(tbrMatch[1]); + return Number.isFinite(frameRate) && frameRate > 0 ? frameRate : null; + } + + return null; +} + +export function parseNativeVideoMetadataProbeOutput( + output: string, +): NativeVideoMetadataProbe | null { + const durationMatch = output.match( + /Duration:\s*([0-9:.]+),\s*start:\s*(-?[0-9]+(?:\.[0-9]+)?)/i, + ); + const duration = durationMatch ? parseFfmpegDurationSeconds(durationMatch[1]) : null; + if (!duration || duration <= 0) { + return null; + } + + const mediaStartTime = durationMatch ? Number(durationMatch[2]) : 0; + const lines = output.split(/\r?\n/); + const videoLine = lines.find((line) => /\bVideo:\s*/i.test(line)); + if (!videoLine) { + return null; + } + + const dimensionsMatch = videoLine.match(/,\s*([0-9]{2,5})x([0-9]{2,5})(?:[,\s]|$)/); + if (!dimensionsMatch) { + return null; + } + + const width = Number(dimensionsMatch[1]); + const height = Number(dimensionsMatch[2]); + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return null; + } + + const videoCodecMatch = videoLine.match(/Video:\s*([^,\r\n]+)/i); + const videoStartMatch = videoLine.match(/\bstart:\s*(-?[0-9]+(?:\.[0-9]+)?)/i); + const frameRate = parseFfmpegFrameRate(videoLine) ?? 60; + const audioLine = lines.find((line) => /\bAudio:\s*/i.test(line)); + const audioCodecMatch = audioLine?.match(/Audio:\s*([^,\r\n]+)/i); + const audioSampleRateMatch = audioLine?.match(/,\s*([0-9]+)\s*Hz\b/i); + + return { + width, + height, + duration, + mediaStartTime: Number.isFinite(mediaStartTime) ? mediaStartTime : 0, + streamStartTime: videoStartMatch ? Number(videoStartMatch[1]) : mediaStartTime, + streamDuration: duration, + frameRate, + codec: videoCodecMatch?.[1]?.trim() || "unknown", + hasAudio: Boolean(audioLine), + audioCodec: audioCodecMatch?.[1]?.trim(), + audioSampleRate: audioSampleRateMatch ? Number(audioSampleRateMatch[1]) : undefined, + }; +} + +export async function probeNativeVideoMetadata( + ffmpegPath: string, + inputPath: string, +): Promise { + let output = ""; + try { + const result = await execFileAsync(ffmpegPath, ["-hide_banner", "-i", inputPath], { + timeout: 30_000, + maxBuffer: 4 * 1024 * 1024, + }); + output = `${result.stdout}\n${result.stderr}`; + } catch (error) { + const processOutput = error as { stdout?: unknown; stderr?: unknown }; + output = [processOutput.stdout, processOutput.stderr] + .filter((value): value is string => typeof value === "string") + .join("\n"); + if (!output) { + throw error; + } + } + + const metadata = parseNativeVideoMetadataProbeOutput(output); + if (!metadata) { + throw new Error("Unable to parse native video metadata from FFmpeg output"); + } + + return metadata; } export function getNativeVideoExportMaxQueuedWriteBytes(inputByteSize: number) { @@ -70,6 +663,76 @@ export function getNativeVideoExportMaxQueuedWriteBytes(inputByteSize: number) { return Math.min(64 * 1024 * 1024, Math.max(16 * 1024 * 1024, inputByteSize * 4)); } +async function runFfmpegWithMetrics( + ffmpegPath: string, + args: string[], + timeoutMs: number, + session?: NativeStaticLayoutExportSession, +): Promise<{ + success: boolean; + elapsedMs: number; + stderr: string; + code: number | null; + signal: NodeJS.Signals | null; +}> { + const startedAt = getNowMs(); + return await new Promise((resolve) => { + const child = spawn(ffmpegPath, args, { + stdio: ["ignore", "ignore", "pipe"], + }); + if (session) { + session.currentProcess = child; + if (session.terminating) { + child.kill("SIGKILL"); + } + } + let stderr = ""; + let settled = false; + const timeout = setTimeout(() => { + if (settled) return; + try { + child.kill("SIGKILL"); + } catch { + // ignore + } + }, timeoutMs); + + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + child.once("error", (error) => { + if (settled) return; + settled = true; + if (session?.currentProcess === child) { + session.currentProcess = null; + } + clearTimeout(timeout); + resolve({ + success: false, + elapsedMs: getNowMs() - startedAt, + stderr: error instanceof Error ? error.message : String(error), + code: null, + signal: null, + }); + }); + child.once("close", (code, signal) => { + if (settled) return; + settled = true; + if (session?.currentProcess === child) { + session.currentProcess = null; + } + clearTimeout(timeout); + resolve({ + success: code === 0, + elapsedMs: getNowMs() - startedAt, + stderr, + code, + signal, + }); + }); + }); +} + export function isHardwareAcceleratedVideoEncoder(encoderName: string) { return /(videotoolbox|nvenc|qsv|amf|mf)/i.test(encoderName); } @@ -262,6 +925,1476 @@ export async function writeNativeVideoExportFrame( } } +function toConcatFileLine(filePath: string) { + const normalized = filePath.replace(/\\/g, "/").replace(/'/g, "'\\''"); + return `file '${normalized}'`; +} + +function getStaticLayoutChunkOutputPath(directory: string, index: number) { + return path.join(directory, `chunk-${String(index).padStart(4, "0")}.mp4`); +} + +function shouldUsePrecompositedStaticLayout(options: NativeStaticLayoutExportOptions) { + return Boolean( + options.backgroundImagePath || + (options.borderRadius ?? 0) > 0.5 || + (options.shadowIntensity ?? 0) > 0, + ); +} + +export async function normalizeNativeStaticLayoutBackground( + options: NativeStaticLayoutExportOptions, +): Promise { + if (!options.backgroundImagePath || (await pathExists(options.backgroundImagePath))) { + return options; + } + + console.warn( + "[native-static-layout-export] Background image is missing; using solid fallback", + { + backgroundImagePath: options.backgroundImagePath, + }, + ); + return { + ...options, + backgroundColor: MISSING_NATIVE_STATIC_BACKGROUND_COLOR, + backgroundImagePath: null, + }; +} + +function getFfmpegFailureMessage(result: Awaited>) { + const suffix = result.signal ? ` (signal ${result.signal})` : ""; + const status = result.code === null ? "unknown" : String(result.code); + return result.stderr.trim() || `FFmpeg exited with code ${status}${suffix}`; +} + +function clampUnit(value: number) { + if (!Number.isFinite(value)) { + return 0; + } + + return Math.min(1, Math.max(0, value)); +} + +function formatCliNumber(value: number) { + return Number.isInteger(value) + ? String(value) + : value.toFixed(6).replace(/0+$/, "").replace(/\.$/, ""); +} + +async function pathExists(filePath: string) { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +function getNativeBinPlatformArch() { + return process.arch === "arm64" ? "win32-arm64" : "win32-x64"; +} + +async function resolveExperimentalWindowsGpuExporterPath() { + if (process.platform !== "win32") { + return null; + } + + const executableNames = ["recordly-gpu-export.exe", "gpu-export-probe.exe"]; + const candidates: string[] = []; + const configuredPath = process.env.RECORDLY_WINDOWS_GPU_EXPORT_EXE; + if (configuredPath) { + candidates.push(configuredPath); + } + + const nativeBinDir = path.join("electron", "native", "bin", getNativeBinPlatformArch()); + const resourcesPath = (process as NodeJS.Process & { resourcesPath?: string }).resourcesPath; + if (resourcesPath) { + for (const executableName of executableNames) { + candidates.push( + path.join(resourcesPath, "app.asar.unpacked", nativeBinDir, executableName), + path.join(resourcesPath, nativeBinDir, executableName), + ); + } + } + + for (const executableName of executableNames) { + candidates.push( + path.join(process.cwd(), nativeBinDir, executableName), + path.join( + app.getAppPath().replace(/app\.asar$/, "app.asar.unpacked"), + nativeBinDir, + executableName, + ), + path.join(process.cwd(), ".tmp", "gpu-export-probe-build", "Release", executableName), + path.join( + process.cwd(), + "electron", + "native", + "gpu-export-probe", + "build", + "Release", + executableName, + ), + ); + } + + for (const candidate of candidates) { + if (await pathExists(candidate)) { + return candidate; + } + } + + return null; +} + +function isExperimentalNvidiaCudaExportEnabled(options: NativeStaticLayoutExportOptions) { + return Boolean( + process.platform === "win32" && + process.env.RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT === "1" && + options.experimentalWindowsGpuCompositor, + ); +} + +async function resolveExperimentalNvidiaCudaExportScriptPath() { + if (process.platform !== "win32") { + return null; + } + + const candidates: string[] = []; + const configuredPath = process.env.RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT; + if (configuredPath) { + candidates.push(configuredPath); + } + candidates.push( + path.join(process.cwd(), ".tmp", "nvdec-nvenc-probe", "run-mp4-pipeline.mjs"), + path.join(app.getAppPath(), ".tmp", "nvdec-nvenc-probe", "run-mp4-pipeline.mjs"), + ); + + for (const candidate of candidates) { + if (await pathExists(candidate)) { + return candidate; + } + } + + return null; +} + +function resolveExperimentalNvidiaCudaNodeCommand() { + const configuredNodePath = process.env.RECORDLY_NVIDIA_CUDA_NODE_EXE; + if (configuredNodePath) { + return { + command: configuredNodePath, + env: {}, + }; + } + + return { + command: process.execPath, + env: { + ELECTRON_RUN_AS_NODE: "1", + }, + }; +} + +function convertHexColorToNv12(color: string) { + const hex = color.trim().match(/^#?([0-9a-f]{6})$/i)?.[1] ?? "101010"; + const r = Number.parseInt(hex.slice(0, 2), 16); + const g = Number.parseInt(hex.slice(2, 4), 16); + const b = Number.parseInt(hex.slice(4, 6), 16); + const clampByte = (value: number) => Math.max(0, Math.min(255, Math.round(value))); + + return { + y: clampByte(((66 * r + 129 * g + 25 * b + 128) >> 8) + 16), + u: clampByte(((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128), + v: clampByte(((112 * r - 94 * g - 18 * b + 128) >> 8) + 128), + }; +} + +function getNvidiaCudaBitrateMbps(options: NativeStaticLayoutExportOptions) { + return Math.max(1, Math.round(options.bitrate / 1_000_000)); +} + +function buildExperimentalWindowsGpuStaticLayoutArgs( + options: NativeStaticLayoutExportOptions, + outputPath: string, +) { + const shadowPixels = Math.round(clampUnit(options.shadowIntensity ?? 0) * 64); + const pixelCount = options.width * options.height; + const surfacePoolSize = pixelCount <= 1920 * 1080 ? 12 : 8; + const args = [ + "--input", + options.inputPath, + "--output", + outputPath, + "--width", + String(options.width), + "--height", + String(options.height), + "--fps", + String(options.frameRate), + "--seconds", + formatCliNumber(options.durationSec), + "--bitrate", + String(options.bitrate), + "--shader-composite", + "--radius", + formatCliNumber(Math.max(0, options.borderRadius ?? 0)), + "--shadow", + formatCliNumber(shadowPixels), + "--content-left", + String(Math.round(options.offsetX)), + "--content-top", + String(Math.round(options.offsetY)), + "--content-width", + String(Math.round(options.contentWidth)), + "--content-height", + String(Math.round(options.contentHeight)), + "--background-color", + options.backgroundColor, + "--surface-pool-size", + String(surfacePoolSize), + ]; + + if (options.backgroundImagePath) { + args.push("--background-image", options.backgroundImagePath); + } + if (options.webcamInputPath) { + const webcamShadowPixels = Math.round(clampUnit(options.webcamShadowIntensity ?? 0) * 64); + args.push( + "--webcam-input", + options.webcamInputPath, + "--webcam-left", + String(Math.round(options.webcamLeft ?? 0)), + "--webcam-top", + String(Math.round(options.webcamTop ?? 0)), + "--webcam-size", + String(Math.round(options.webcamSize ?? 0)), + "--webcam-radius", + formatCliNumber(Math.max(0, options.webcamRadius ?? 0)), + "--webcam-shadow", + formatCliNumber(webcamShadowPixels), + "--webcam-time-offset-ms", + formatCliNumber(options.webcamTimeOffsetMs ?? 0), + ); + if (options.webcamMirror !== false) { + args.push("--webcam-mirror"); + } + } + if (options.cursorTelemetryPath) { + args.push( + "--cursor-telemetry", + options.cursorTelemetryPath, + "--cursor-size", + formatCliNumber(Math.max(1, options.cursorSize ?? 84)), + ); + if (options.cursorAtlasPath && options.cursorAtlasMetadataPath) { + args.push( + "--cursor-atlas", + options.cursorAtlasPath, + "--cursor-atlas-metadata", + options.cursorAtlasMetadataPath, + ); + } + } + if (options.zoomTelemetryPath) { + args.push("--zoom-telemetry", options.zoomTelemetryPath); + } + + return args; +} + +async function prepareWindowsGpuCursorTelemetry( + options: NativeStaticLayoutExportOptions, + outputPath: string, +) { + const telemetry = options.cursorTelemetry; + if (!telemetry || telemetry.length === 0) { + return null; + } + + const lines = telemetry + .filter((sample) => { + return ( + Number.isFinite(sample.timeMs) && + Number.isFinite(sample.cx) && + Number.isFinite(sample.cy) + ); + }) + .map((sample) => { + const timeMs = Math.max(0, sample.timeMs); + const cx = Math.min(1, Math.max(0, sample.cx)); + const cy = Math.min(1, Math.max(0, sample.cy)); + const cursorTypeIndex = Math.max( + 0, + Math.min(8, Math.round(sample.cursorTypeIndex ?? 0)), + ); + const bounceScale = Math.min(2, Math.max(0.1, sample.bounceScale ?? 1)); + return [ + formatCliNumber(timeMs), + formatCliNumber(cx), + formatCliNumber(cy), + String(cursorTypeIndex), + formatCliNumber(bounceScale), + ].join(","); + }); + + if (lines.length === 0) { + return null; + } + + await fs.writeFile(outputPath, lines.join("\n"), "utf8"); + return outputPath; +} + +async function prepareWindowsGpuCursorAtlas( + options: NativeStaticLayoutExportOptions, + atlasPath: string, + metadataPath: string, +) { + const dataUrl = options.cursorAtlasPngDataUrl; + const entries = options.cursorAtlasEntries; + if (!dataUrl || !entries?.length) { + return null; + } + + const match = /^data:image\/png;base64,([A-Za-z0-9+/=]+)$/.exec(dataUrl); + if (!match) { + return null; + } + + const metadataLines = entries + .filter((entry) => { + return ( + Number.isInteger(entry.index) && + Number.isFinite(entry.x) && + Number.isFinite(entry.y) && + Number.isFinite(entry.width) && + Number.isFinite(entry.height) && + Number.isFinite(entry.anchorX) && + Number.isFinite(entry.anchorY) && + Number.isFinite(entry.aspectRatio) && + entry.width > 0 && + entry.height > 0 + ); + }) + .map((entry) => + [ + String(Math.max(0, Math.min(8, entry.index))), + formatCliNumber(Math.max(0, entry.x)), + formatCliNumber(Math.max(0, entry.y)), + formatCliNumber(Math.max(1, entry.width)), + formatCliNumber(Math.max(1, entry.height)), + formatCliNumber(Math.min(1, Math.max(0, entry.anchorX))), + formatCliNumber(Math.min(1, Math.max(0, entry.anchorY))), + formatCliNumber(Math.max(0.01, entry.aspectRatio)), + ].join(","), + ); + if (metadataLines.length === 0) { + return null; + } + + await fs.writeFile(atlasPath, Buffer.from(match[1], "base64")); + await fs.writeFile(metadataPath, metadataLines.join("\n"), "utf8"); + return { atlasPath, metadataPath }; +} + +async function prepareNvidiaCudaCursorTelemetry( + options: NativeStaticLayoutExportOptions, + outputPath: string, +) { + const telemetry = options.cursorTelemetry; + if (!telemetry || telemetry.length === 0) { + return null; + } + + const samples = telemetry + .filter((sample) => { + return ( + Number.isFinite(sample.timeMs) && + Number.isFinite(sample.cx) && + Number.isFinite(sample.cy) + ); + }) + .map((sample) => ({ + timeMs: Math.max(0, sample.timeMs), + cx: Math.min(1, Math.max(0, sample.cx)), + cy: Math.min(1, Math.max(0, sample.cy)), + cursorTypeIndex: Math.max(0, Math.min(8, Math.round(sample.cursorTypeIndex ?? 0))), + bounceScale: Math.min(2, Math.max(0.1, sample.bounceScale ?? 1)), + })); + if (samples.length === 0) { + return null; + } + + await fs.writeFile(outputPath, JSON.stringify({ samples }), "utf8"); + return outputPath; +} + +async function prepareNvidiaCudaCursorAtlas( + options: NativeStaticLayoutExportOptions, + atlasPath: string, + metadataPath: string, +) { + const dataUrl = options.cursorAtlasPngDataUrl; + const entries = options.cursorAtlasEntries; + if (!dataUrl || !entries?.length) { + return null; + } + + const match = /^data:image\/png;base64,([A-Za-z0-9+/=]+)$/.exec(dataUrl); + if (!match) { + return null; + } + + const metadataLines = entries + .filter((entry) => { + return ( + Number.isInteger(entry.index) && + Number.isFinite(entry.x) && + Number.isFinite(entry.y) && + Number.isFinite(entry.width) && + Number.isFinite(entry.height) && + Number.isFinite(entry.anchorX) && + Number.isFinite(entry.anchorY) && + Number.isFinite(entry.aspectRatio) && + entry.width > 0 && + entry.height > 0 + ); + }) + .map((entry) => + [ + String(Math.max(0, Math.min(8, entry.index))), + formatCliNumber(Math.max(0, entry.x)), + formatCliNumber(Math.max(0, entry.y)), + formatCliNumber(Math.max(1, entry.width)), + formatCliNumber(Math.max(1, entry.height)), + formatCliNumber(Math.min(1, Math.max(0, entry.anchorX))), + formatCliNumber(Math.min(1, Math.max(0, entry.anchorY))), + formatCliNumber(Math.max(0.01, entry.aspectRatio)), + ].join("\t"), + ); + if (metadataLines.length === 0) { + return null; + } + + await fs.writeFile(atlasPath, Buffer.from(match[1], "base64")); + await fs.writeFile(metadataPath, `${metadataLines.join("\n")}\n`, "utf8"); + return { atlasPath, metadataPath }; +} + +async function prepareWindowsGpuZoomTelemetry( + options: NativeStaticLayoutExportOptions, + outputPath: string, +) { + const telemetry = options.zoomTelemetry; + if (!telemetry || telemetry.length === 0) { + return null; + } + + const lines = telemetry + .filter((sample) => { + return ( + Number.isFinite(sample.timeMs) && + Number.isFinite(sample.scale) && + Number.isFinite(sample.x) && + Number.isFinite(sample.y) + ); + }) + .map((sample) => { + const timeMs = Math.max(0, sample.timeMs); + const scale = Math.max(0.01, sample.scale); + return [ + formatCliNumber(timeMs), + formatCliNumber(scale), + formatCliNumber(sample.x), + formatCliNumber(sample.y), + ].join(","); + }); + + if (lines.length === 0) { + return null; + } + + await fs.writeFile(outputPath, lines.join("\n"), "utf8"); + return outputPath; +} + +function shouldCreateWindowsGpuWebcamProxy(filePath: string) { + const extension = path.extname(filePath).toLowerCase(); + return extension !== ".mp4" && extension !== ".m4v" && extension !== ".mov"; +} + +function buildWindowsGpuWebcamProxyArgs( + options: NativeStaticLayoutExportOptions, + outputPath: string, +) { + if (!options.webcamInputPath) { + throw new Error("Windows GPU webcam proxy requires an input path"); + } + + const frameRate = Math.max(1, Math.round(options.frameRate)); + const proxyBitrate = Math.max(2_000_000, Math.min(6_000_000, Math.round(options.bitrate / 3))); + return [ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-i", + options.webcamInputPath, + "-map", + "0:v:0", + "-an", + "-t", + formatCliNumber(options.durationSec), + "-vf", + `scale=trunc(iw/2)*2:trunc(ih/2)*2,setsar=1,fps=${frameRate}`, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-tune", + "zerolatency", + "-b:v", + String(proxyBitrate), + "-maxrate", + String(Math.round(proxyBitrate * 1.2)), + "-bufsize", + String(proxyBitrate * 2), + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + outputPath, + ]; +} + +async function prepareWindowsGpuWebcamInput( + ffmpegPath: string, + options: NativeStaticLayoutExportOptions, + outputPath: string, + session: NativeStaticLayoutExportSession, +) { + if (!options.webcamInputPath || !shouldCreateWindowsGpuWebcamProxy(options.webcamInputPath)) { + return { inputPath: options.webcamInputPath ?? null, elapsedMs: 0 }; + } + + const result = await runFfmpegWithMetrics( + ffmpegPath, + buildWindowsGpuWebcamProxyArgs(options, outputPath), + Math.max(5 * 60 * 1000, options.durationSec * 1000), + session, + ); + if (!result.success) { + throw new Error(getFfmpegFailureMessage(result)); + } + + return { inputPath: outputPath, elapsedMs: result.elapsedMs }; +} + +function buildExperimentalNvidiaCudaStaticLayoutArgs( + options: NativeStaticLayoutExportOptions, + outputPath: string, + workDir: string, +) { + const background = convertHexColorToNv12(options.backgroundColor); + const shadowIntensityPct = Math.round(clampUnit(options.shadowIntensity ?? 0) * 100); + const shadowOffsetY = + shadowIntensityPct > 0 ? Math.max(1, Math.round(options.height * 0.012)) : 0; + const args = [ + "--input", + options.inputPath, + "--output", + outputPath, + "--work-dir", + workDir, + "--fps", + String(Math.max(1, Math.round(options.frameRate))), + "--bitrate-mbps", + String(getNvidiaCudaBitrateMbps(options)), + "--duration-sec", + formatCliNumber(options.durationSec), + "--stream-sync", + "--prewarm-ms", + process.env.RECORDLY_NVIDIA_CUDA_PREWARM_MS || "500", + "--content-x", + String(Math.round(options.offsetX)), + "--content-y", + String(Math.round(options.offsetY)), + "--content-width", + String(Math.round(options.contentWidth)), + "--content-height", + String(Math.round(options.contentHeight)), + "--radius", + String(Math.round(Math.max(0, options.borderRadius ?? 0))), + "--background-y", + String(background.y), + "--background-u", + String(background.u), + "--background-v", + String(background.v), + ]; + if (!canMuxNvidiaCudaSourceAudioInline(options)) { + args.push("--video-only"); + } + + if (options.backgroundImagePath) { + args.push("--background-image", options.backgroundImagePath); + } + if (shadowOffsetY > 0 && shadowIntensityPct > 0) { + args.push( + "--shadow-offset-y", + String(shadowOffsetY), + "--shadow-intensity-pct", + String(shadowIntensityPct), + ); + } + if (options.webcamInputPath && (options.webcamSize ?? 0) > 0) { + args.push( + "--webcam-input", + options.webcamInputPath, + "--webcam-x", + String(Math.round(options.webcamLeft ?? 0)), + "--webcam-y", + String(Math.round(options.webcamTop ?? 0)), + "--webcam-size", + String(Math.round(options.webcamSize ?? 0)), + "--webcam-radius", + String(Math.round(Math.max(0, options.webcamRadius ?? 0))), + "--webcam-stream", + ); + if (options.webcamMirror !== false) { + args.push("--webcam-mirror"); + } + } + if (options.cursorTelemetryPath) { + args.push( + "--cursor-json", + options.cursorTelemetryPath, + "--cursor-height", + String(Math.round(Math.max(1, options.cursorSize ?? 84))), + "--cursor-style", + "external", + ); + if (options.cursorAtlasPath && options.cursorAtlasMetadataPath) { + args.push( + "--cursor-atlas-png", + options.cursorAtlasPath, + "--cursor-atlas-metadata", + options.cursorAtlasMetadataPath, + ); + } + } + if (options.zoomTelemetryPath) { + args.push("--zoom-telemetry", options.zoomTelemetryPath); + } + if (process.env.RECORDLY_NVIDIA_CUDA_SAMPLE_GPU === "1") { + args.push("--sample-gpu"); + } + + return args; +} + +function canMuxNvidiaCudaSourceAudioInline(options: NativeStaticLayoutExportOptions) { + const audioOptions = options.audioOptions; + if (audioOptions?.audioMode !== "copy-source" || !audioOptions.audioSourcePath) { + return false; + } + + return path.resolve(audioOptions.audioSourcePath) === path.resolve(options.inputPath); +} + +async function runExperimentalNvidiaCudaStaticLayoutExport( + ffmpegPath: string, + options: NativeStaticLayoutExportOptions, + outputPath: string, + chunkDirectory: string, + session: NativeStaticLayoutExportSession, + onProgress?: (progress: NativeStaticLayoutExportProgress) => void, +) { + const scriptPath = await resolveExperimentalNvidiaCudaExportScriptPath(); + if (!scriptPath) { + throw new Error("Experimental NVIDIA CUDA export script is not available"); + } + + const nodeCommand = resolveExperimentalNvidiaCudaNodeCommand(); + const workDir = path.join(chunkDirectory, "nvidia-cuda-work"); + const args = [ + scriptPath, + ...buildExperimentalNvidiaCudaStaticLayoutArgs(options, outputPath, workDir), + ]; + const startedAt = getNowMs(); + const startedAtIso = new Date().toISOString(); + const timeoutMs = Math.max(20 * 60 * 1000, options.durationSec * 2000); + const ffmpegDirectory = path.dirname(ffmpegPath); + const pathKey = process.platform === "win32" ? "Path" : "PATH"; + const env = { + ...process.env, + ...nodeCommand.env, + RECORDLY_NVIDIA_CUDA_EXPORT_HIGH_PRIORITY: + process.env.RECORDLY_NVIDIA_CUDA_EXPORT_HIGH_PRIORITY ?? "1", + [pathKey]: `${ffmpegDirectory}${path.delimiter}${process.env[pathKey] ?? ""}`, + }; + const powerGuard = startNativeStaticLayoutExportPowerGuard(); + + return await new Promise<{ + elapsedMs: number; + stdout: string; + stderr: string; + summary: NvidiaCudaExportSummary; + }>((resolve, reject) => { + const child = spawn(nodeCommand.command, args, { + env, + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }); + const childPriorityApplied = setNativeStaticLayoutExportProcessPriority( + child.pid, + "NVIDIA CUDA export wrapper", + ); + console.info("[native-static-layout-export] NVIDIA CUDA runtime guard started", { + childPriorityApplied, + powerGuardStarted: powerGuard.started, + }); + session.currentProcess = child; + if (session.terminating) { + child.kill("SIGKILL"); + } + + let stdout = ""; + let stderr = ""; + let stderrLineBuffer = ""; + let settled = false; + const timeout = setTimeout(() => { + if (settled) return; + child.kill("SIGKILL"); + }, timeoutMs); + + child.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk: Buffer) => { + const text = chunk.toString(); + stderr += text; + stderrLineBuffer += text; + const lines = stderrLineBuffer.split(/\r?\n/); + stderrLineBuffer = lines.pop() ?? ""; + for (const line of lines) { + const progress = parseWindowsGpuExportProgressLine(line); + if (!progress) { + continue; + } + const elapsedMs = Math.max(0, getNowMs() - startedAt); + const averageFps = + typeof progress.averageFps === "number" && + Number.isFinite(progress.averageFps) && + progress.averageFps > 0 + ? progress.averageFps + : elapsedMs > 0 && progress.currentFrame > 0 + ? (progress.currentFrame * 1000) / elapsedMs + : undefined; + onProgress?.({ + ...progress, + sessionId: options.sessionId, + backend: "nvidia-cuda-compositor", + elapsedMs, + averageFps, + }); + } + }); + child.once("error", (error) => { + if (settled) return; + settled = true; + if (session.currentProcess === child) { + session.currentProcess = null; + } + clearTimeout(timeout); + powerGuard.release(); + reject(error); + }); + child.once("close", async (code, signal) => { + if (settled) return; + settled = true; + if (session.currentProcess === child) { + session.currentProcess = null; + } + clearTimeout(timeout); + powerGuard.release(); + + if (session.terminating) { + reject(new Error("Native static layout export was cancelled")); + return; + } + + const elapsedMs = getNowMs() - startedAt; + const summary = parseNvidiaCudaExportSummary(stdout); + if (summary) { + summary.appRuntimeGuard = { + powerGuardStarted: powerGuard.started, + wrapperProcessPriorityBoosted: childPriorityApplied, + nativeProcessPriorityBoosted: summary.nativeProcessPriorityBoosted, + }; + } + await Promise.allSettled([ + fs.writeFile(path.join(chunkDirectory, "nvidia-cuda-export.stdout.json"), stdout), + fs.writeFile(path.join(chunkDirectory, "nvidia-cuda-export.stderr.log"), stderr), + summary + ? fs.writeFile( + path.join(chunkDirectory, "nvidia-cuda-export.summary.json"), + `${JSON.stringify(summary, null, 2)}\n`, + ) + : Promise.resolve(), + ]); + await persistNvidiaCudaExportDiagnostics({ + args, + code, + elapsedMs, + outputPath, + sessionId: options.sessionId, + signal, + startedAtIso, + stderr, + stdout, + summary, + }); + if (code !== 0 || !summary?.success) { + const suffix = signal ? ` (signal ${signal})` : ""; + reject( + new Error( + stderr.trim() || + stdout.trim() || + `Experimental NVIDIA CUDA exporter exited with code ${code ?? "unknown"}${suffix}`, + ), + ); + return; + } + + resolve({ + elapsedMs, + stdout, + stderr, + summary, + }); + }); + }); +} + +async function runExperimentalWindowsGpuStaticLayoutExport( + options: NativeStaticLayoutExportOptions, + outputPath: string, + session: NativeStaticLayoutExportSession, + onProgress?: (progress: NativeStaticLayoutExportProgress) => void, +) { + const executablePath = await resolveExperimentalWindowsGpuExporterPath(); + if (!executablePath) { + throw new Error("Experimental Windows GPU exporter is not built"); + } + + const args = buildExperimentalWindowsGpuStaticLayoutArgs(options, outputPath); + const startedAt = getNowMs(); + const timeoutMs = Math.max(15 * 60 * 1000, options.durationSec * 1000); + + return await new Promise<{ + elapsedMs: number; + stdout: string; + stderr: string; + summary: WindowsGpuExportSummary; + }>((resolve, reject) => { + const child = spawn(executablePath, args, { + stdio: ["ignore", "pipe", "pipe"], + }); + session.currentProcess = child; + if (session.terminating) { + child.kill("SIGKILL"); + } + + let stdout = ""; + let stderr = ""; + let stderrLineBuffer = ""; + let settled = false; + const timeout = setTimeout(() => { + if (settled) return; + child.kill("SIGKILL"); + }, timeoutMs); + + child.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk: Buffer) => { + const text = chunk.toString(); + stderr += text; + stderrLineBuffer += text; + const lines = stderrLineBuffer.split(/\r?\n/); + stderrLineBuffer = lines.pop() ?? ""; + for (const line of lines) { + const progress = parseWindowsGpuExportProgressLine(line); + if (!progress) { + continue; + } + const elapsedMs = Math.max(0, getNowMs() - startedAt); + onProgress?.({ + ...progress, + sessionId: options.sessionId, + backend: "windows-d3d11-compositor", + elapsedMs, + averageFps: + elapsedMs > 0 && progress.currentFrame > 0 + ? (progress.currentFrame * 1000) / elapsedMs + : undefined, + }); + } + }); + child.once("error", (error) => { + if (settled) return; + settled = true; + if (session.currentProcess === child) { + session.currentProcess = null; + } + clearTimeout(timeout); + reject(error); + }); + child.once("close", (code, signal) => { + if (settled) return; + settled = true; + if (session.currentProcess === child) { + session.currentProcess = null; + } + clearTimeout(timeout); + + if (session.terminating) { + reject(new Error("Native static layout export was cancelled")); + return; + } + + const summary = parseWindowsGpuExportSummary(stdout); + + if (code !== 0 || !summary?.success) { + const suffix = signal ? ` (signal ${signal})` : ""; + reject( + new Error( + stderr.trim() || + stdout.trim() || + `Experimental Windows GPU exporter exited with code ${code ?? "unknown"}${suffix}`, + ), + ); + return; + } + + resolve({ + elapsedMs: getNowMs() - startedAt, + stdout, + stderr, + summary, + }); + }); + }); +} + +export async function exportNativeStaticLayoutVideo( + ffmpegPath: string, + options: NativeStaticLayoutExportOptions, + onProgress?: (progress: NativeStaticLayoutExportProgress) => void, +) { + if (options.width % 2 !== 0 || options.height % 2 !== 0) { + throw new Error("Native static layout export requires even output dimensions"); + } + if (!Number.isFinite(options.durationSec) || options.durationSec <= 0) { + throw new Error("Native static layout export requires a positive duration"); + } + options = await normalizeNativeStaticLayoutBackground(options); + if ( + options.webcamInputPath && + !(options.experimentalWindowsGpuCompositor && process.platform === "win32") + ) { + throw new Error("Native webcam overlay requires the Windows GPU compositor"); + } + if ( + options.zoomTelemetry?.length && + !(options.experimentalWindowsGpuCompositor && process.platform === "win32") + ) { + throw new Error("Native zoom telemetry requires the Windows GPU compositor"); + } + + const chunkDurationSec = Math.max(1, Math.min(300, options.chunkDurationSec ?? 120)); + const chunks = buildNativeStaticLayoutChunks(options.durationSec, chunkDurationSec); + if (chunks.length === 0) { + throw new Error("Native static layout export produced no chunks"); + } + + const sessionId = + options.sessionId ?? + `recordly-static-layout-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const session: NativeStaticLayoutExportSession = { + terminating: false, + currentProcess: null, + }; + const chunkDirectory = path.join(app.getPath("temp"), sessionId); + const concatListPath = path.join(chunkDirectory, "chunks.txt"); + const videoOnlyPath = path.join(app.getPath("temp"), `${sessionId}.mp4`); + let outputPathToKeep: string | null = null; + const metrics: NativeStaticLayoutExportMetrics = { + chunkCount: chunks.length, + chunkDurationSec, + chunkExecMs: 0, + fallbackChunkCount: 0, + chunks: [], + }; + + try { + nativeStaticLayoutExportSessions.set(sessionId, session); + await fs.mkdir(chunkDirectory, { recursive: true }); + const fullConfig: NativeStaticLayoutExportArgsConfig = { + inputPath: options.inputPath, + outputPath: videoOnlyPath, + width: options.width, + height: options.height, + frameRate: options.frameRate, + bitrate: options.bitrate, + encodingMode: options.encodingMode, + contentWidth: options.contentWidth, + contentHeight: options.contentHeight, + offsetX: options.offsetX, + offsetY: options.offsetY, + backgroundColor: options.backgroundColor, + backgroundImagePath: options.backgroundImagePath, + borderRadius: options.borderRadius, + shadowIntensity: options.shadowIntensity, + durationSec: options.durationSec, + }; + const usePrecompositedLayout = shouldUsePrecompositedStaticLayout(options); + let didRenderVideo = false; + let didMuxAudioInline = false; + + if (options.experimentalWindowsGpuCompositor && process.platform === "win32") { + try { + if (session.terminating) { + throw new Error("Native static layout export was cancelled"); + } + let experimentalGpuOptions = options; + if (options.webcamInputPath) { + const webcamProxyPath = path.join(chunkDirectory, "webcam-proxy.mp4"); + const webcamInput = await prepareWindowsGpuWebcamInput( + ffmpegPath, + options, + webcamProxyPath, + session, + ); + if (webcamInput.elapsedMs > 0) { + metrics.staticAssetExecMs = + (metrics.staticAssetExecMs ?? 0) + webcamInput.elapsedMs; + } + if ( + webcamInput.inputPath && + webcamInput.inputPath !== options.webcamInputPath + ) { + experimentalGpuOptions = { + ...options, + webcamInputPath: webcamInput.inputPath, + }; + } + } + if (options.cursorTelemetry?.length) { + const cursorTelemetryPath = await prepareWindowsGpuCursorTelemetry( + options, + path.join(chunkDirectory, "cursor-telemetry.csv"), + ); + if (cursorTelemetryPath) { + experimentalGpuOptions = { + ...experimentalGpuOptions, + cursorTelemetryPath, + }; + } + const cursorAtlas = await prepareWindowsGpuCursorAtlas( + options, + path.join(chunkDirectory, "cursor-atlas.png"), + path.join(chunkDirectory, "cursor-atlas.csv"), + ); + if (cursorAtlas) { + experimentalGpuOptions = { + ...experimentalGpuOptions, + cursorAtlasPath: cursorAtlas.atlasPath, + cursorAtlasMetadataPath: cursorAtlas.metadataPath, + }; + } + } + if (options.zoomTelemetry?.length) { + const zoomTelemetryPath = await prepareWindowsGpuZoomTelemetry( + options, + path.join(chunkDirectory, "zoom-telemetry.csv"), + ); + if (zoomTelemetryPath) { + experimentalGpuOptions = { + ...experimentalGpuOptions, + zoomTelemetryPath, + }; + } + } + let experimentalNvidiaCudaOptions = experimentalGpuOptions; + let shouldTryNvidiaCuda = isExperimentalNvidiaCudaExportEnabled(options); + if (shouldTryNvidiaCuda && options.cursorTelemetry?.length) { + const cursorTelemetryPath = await prepareNvidiaCudaCursorTelemetry( + options, + path.join(chunkDirectory, "cursor-telemetry.json"), + ); + const cursorAtlas = await prepareNvidiaCudaCursorAtlas( + options, + path.join(chunkDirectory, "cursor-atlas-nvidia.png"), + path.join(chunkDirectory, "cursor-atlas-nvidia.tsv"), + ); + shouldTryNvidiaCuda = Boolean(cursorTelemetryPath && cursorAtlas); + if (cursorTelemetryPath && cursorAtlas) { + experimentalNvidiaCudaOptions = { + ...experimentalGpuOptions, + cursorTelemetryPath, + cursorAtlasPath: cursorAtlas.atlasPath, + cursorAtlasMetadataPath: cursorAtlas.metadataPath, + }; + } + } + + if (shouldTryNvidiaCuda) { + try { + const shouldMuxAudioInline = + canMuxNvidiaCudaSourceAudioInline(experimentalNvidiaCudaOptions); + const cudaResult = await runExperimentalNvidiaCudaStaticLayoutExport( + ffmpegPath, + experimentalNvidiaCudaOptions, + videoOnlyPath, + chunkDirectory, + session, + onProgress, + ); + console.info( + "[native-static-layout-export] NVIDIA CUDA compositor completed", + { + elapsedMs: cudaResult.elapsedMs, + fps: cudaResult.summary.fps, + targetFrames: cudaResult.summary.targetFrames, + durationSec: cudaResult.summary.durationSec, + nativeEncodeMs: cudaResult.summary.timingsMs?.nativeEncode, + muxMs: cudaResult.summary.timingsMs?.mux, + endToEndMs: cudaResult.summary.timingsMs?.endToEnd, + nativeFps: + cudaResult.summary.nativeSummary?.measuredFps ?? + cudaResult.summary.nativeSummary?.fps, + mappedDisplayFrames: + cudaResult.summary.nativeSummary?.mappedDisplayFrames, + selectedDisplayFrames: + cudaResult.summary.nativeSummary?.selectedDisplayFrames, + skippedDisplayFrames: + cudaResult.summary.nativeSummary?.skippedDisplayFrames, + roiCompositeFrames: + cudaResult.summary.nativeSummary?.roiCompositeFrames, + monolithicCompositeFrames: + cudaResult.summary.nativeSummary?.monolithicCompositeFrames, + copyCompositeFrames: + cudaResult.summary.nativeSummary?.copyCompositeFrames, + webcamOverlay: cudaResult.summary.nativeSummary?.webcamOverlay, + cursorAtlas: cudaResult.summary.nativeSummary?.cursorAtlas, + zoomOverlay: cudaResult.summary.nativeSummary?.zoomOverlay, + zoomSamples: cudaResult.summary.nativeSummary?.zoomSamples, + }, + ); + const outputStat = await fs.stat(videoOnlyPath); + metrics.chunkCount = 1; + metrics.chunkDurationSec = options.durationSec; + metrics.chunkExecMs += cudaResult.elapsedMs; + metrics.chunks.push({ + index: 0, + startSec: 0, + durationSec: options.durationSec, + backend: "nvidia-cuda-compositor", + elapsedMs: cudaResult.elapsedMs, + outputBytes: outputStat.size, + nvidiaCudaSummary: cudaResult.summary, + }); + didRenderVideo = true; + didMuxAudioInline = shouldMuxAudioInline; + } catch (error) { + if (session.terminating) { + throw error; + } + metrics.fallbackChunkCount++; + console.warn( + "[native-static-layout-export] Experimental NVIDIA CUDA compositor unavailable; falling back to Windows GPU compositor:", + error, + ); + await removeTemporaryExportFile(videoOnlyPath); + } + } + + if (!didRenderVideo) { + const gpuResult = await runExperimentalWindowsGpuStaticLayoutExport( + experimentalGpuOptions, + videoOnlyPath, + session, + onProgress, + ); + console.info("[native-static-layout-export] Windows GPU compositor completed", { + elapsedMs: gpuResult.elapsedMs, + width: gpuResult.summary.width, + height: gpuResult.summary.height, + fps: gpuResult.summary.fps, + frames: gpuResult.summary.frames, + realtimeMultiplier: gpuResult.summary.realtimeMultiplier, + surfacePoolSize: gpuResult.summary.surfacePoolSize, + gpuDecodeSurface: gpuResult.summary.gpuDecodeSurface, + adapterIndex: gpuResult.summary.adapterIndex, + encoderBackend: gpuResult.summary.encoderBackend, + encoderTuningApplied: gpuResult.summary.encoderTuningApplied, + readMs: gpuResult.summary.readMs, + videoProcessMs: gpuResult.summary.videoProcessMs, + writeSampleMs: gpuResult.summary.writeSampleMs, + finalizeMs: gpuResult.summary.finalizeMs, + webcamOverlay: gpuResult.summary.webcamOverlay, + cursorOverlay: gpuResult.summary.cursorOverlay, + cursorAtlas: gpuResult.summary.cursorAtlas, + zoomOverlay: gpuResult.summary.zoomOverlay, + }); + const outputStat = await fs.stat(videoOnlyPath); + metrics.chunkCount = 1; + metrics.chunkDurationSec = options.durationSec; + metrics.chunkExecMs += gpuResult.elapsedMs; + metrics.chunks.push({ + index: 0, + startSec: 0, + durationSec: options.durationSec, + backend: "windows-d3d11-compositor", + elapsedMs: gpuResult.elapsedMs, + outputBytes: outputStat.size, + windowsGpuSummary: gpuResult.summary, + }); + didRenderVideo = true; + } + } catch (error) { + if (session.terminating) { + throw error; + } + metrics.fallbackChunkCount++; + console.warn( + "[native-static-layout-export] Experimental Windows GPU compositor unavailable; falling back to FFmpeg static layout:", + error, + ); + await removeTemporaryExportFile(videoOnlyPath); + if (options.webcamInputPath || options.zoomTelemetry?.length) { + throw error; + } + } + } + + if (!didRenderVideo && usePrecompositedLayout) { + const maskPath = path.join(chunkDirectory, "layout-mask.pgm"); + const staticBackgroundPath = path.join(chunkDirectory, "layout-background.png"); + await fs.writeFile( + maskPath, + createNativeSquircleMaskPgmBuffer( + options.contentWidth, + options.contentHeight, + options.borderRadius ?? 0, + ), + ); + + const backgroundResult = await runFfmpegWithMetrics( + ffmpegPath, + buildNativeStaticBackgroundRenderArgs({ + ...fullConfig, + inputPath: options.inputPath, + outputPath: staticBackgroundPath, + maskPath, + }), + 2 * 60 * 1000, + session, + ); + metrics.staticAssetExecMs = backgroundResult.elapsedMs; + if (!backgroundResult.success) { + throw new Error(getFfmpegFailureMessage(backgroundResult)); + } + + const fullResult = await runFfmpegWithMetrics( + ffmpegPath, + buildNativePrecompositedStaticLayoutArgs({ + ...fullConfig, + staticBackgroundPath, + maskPath, + }), + 15 * 60 * 1000, + session, + ); + metrics.chunkExecMs += fullResult.elapsedMs; + if (!fullResult.success) { + throw new Error(getFfmpegFailureMessage(fullResult)); + } + + const outputStat = await fs.stat(videoOnlyPath); + metrics.chunkCount = 1; + metrics.chunkDurationSec = options.durationSec; + metrics.chunks.push({ + index: 0, + startSec: 0, + durationSec: options.durationSec, + backend: "cuda-static-composite", + elapsedMs: fullResult.elapsedMs, + outputBytes: outputStat.size, + }); + } else if (!didRenderVideo) { + const primaryResult = await runFfmpegWithMetrics( + ffmpegPath, + buildNativeCudaOverlayStaticLayoutArgs(fullConfig), + 15 * 60 * 1000, + session, + ); + let fullResult = primaryResult; + let fullBackend: NativeStaticLayoutBackend = "cuda-overlay"; + let fallbackReason: string | undefined; + if (!primaryResult.success) { + fullBackend = "cuda-scale-cpu-pad"; + fallbackReason = isNativeCudaOutOfMemory(primaryResult.stderr) + ? "cuda-oom" + : "cuda-overlay-failed"; + metrics.fallbackChunkCount++; + fullResult = await runFfmpegWithMetrics( + ffmpegPath, + buildNativeCudaScaleCpuPadStaticLayoutArgs(fullConfig), + 15 * 60 * 1000, + session, + ); + } + metrics.chunkExecMs += fullResult.elapsedMs; + if (fullResult !== primaryResult) { + metrics.chunkExecMs += primaryResult.elapsedMs; + } + + if (fullResult.success) { + const outputStat = await fs.stat(videoOnlyPath); + metrics.chunkCount = 1; + metrics.chunkDurationSec = options.durationSec; + metrics.chunks.push({ + index: 0, + startSec: 0, + durationSec: options.durationSec, + backend: fullBackend, + elapsedMs: fullResult.elapsedMs, + outputBytes: outputStat.size, + fallbackReason, + }); + } else if (isNativeCudaOutOfMemory(fullResult.stderr)) { + const concatLines: string[] = []; + + for (const chunk of chunks) { + if (session.terminating) { + throw new Error("Native static layout export was cancelled"); + } + + const outputPath = getStaticLayoutChunkOutputPath(chunkDirectory, chunk.index); + const baseConfig: NativeStaticLayoutExportArgsConfig = { + inputPath: options.inputPath, + outputPath, + width: options.width, + height: options.height, + frameRate: options.frameRate, + bitrate: options.bitrate, + encodingMode: options.encodingMode, + contentWidth: options.contentWidth, + contentHeight: options.contentHeight, + offsetX: options.offsetX, + offsetY: options.offsetY, + backgroundColor: options.backgroundColor, + startSec: chunk.startSec, + durationSec: chunk.durationSec, + }; + const primary = await runFfmpegWithMetrics( + ffmpegPath, + buildNativeCudaOverlayStaticLayoutArgs(baseConfig), + 15 * 60 * 1000, + session, + ); + let backend: NativeStaticLayoutBackend = "cuda-overlay"; + let result = primary; + let fallbackReason: string | undefined; + + if (!primary.success && isNativeCudaOutOfMemory(primary.stderr)) { + backend = "cuda-scale-cpu-pad"; + fallbackReason = "cuda-oom"; + metrics.fallbackChunkCount++; + result = await runFfmpegWithMetrics( + ffmpegPath, + buildNativeCudaScaleCpuPadStaticLayoutArgs(baseConfig), + 15 * 60 * 1000, + session, + ); + } + + metrics.chunkExecMs += result.elapsedMs; + if (!result.success) { + throw new Error(getFfmpegFailureMessage(result)); + } + + const outputStat = await fs.stat(outputPath); + metrics.chunks.push({ + index: chunk.index, + startSec: chunk.startSec, + durationSec: chunk.durationSec, + backend, + elapsedMs: result.elapsedMs, + outputBytes: outputStat.size, + fallbackReason, + }); + concatLines.push(toConcatFileLine(outputPath)); + } + + metrics.chunkCount = chunks.length; + await fs.writeFile(concatListPath, `${concatLines.join("\n")}\n`, "utf8"); + if (session.terminating) { + throw new Error("Native static layout export was cancelled"); + } + const concatStartedAt = getNowMs(); + const concatResult = await runFfmpegWithMetrics( + ffmpegPath, + buildNativeConcatArgs({ listPath: concatListPath, outputPath: videoOnlyPath }), + 15 * 60 * 1000, + session, + ); + metrics.concatExecMs = getNowMs() - concatStartedAt; + if (!concatResult.success) { + throw new Error(getFfmpegFailureMessage(concatResult)); + } + } else { + throw new Error(getFfmpegFailureMessage(fullResult)); + } + } + + const videoOnlyStat = await fs.stat(videoOnlyPath); + metrics.videoOnlyBytes = videoOnlyStat.size; + if (didMuxAudioInline) { + outputPathToKeep = videoOnlyPath; + return { + outputPath: videoOnlyPath, + metrics, + }; + } + const finalized = await muxNativeVideoExportAudio( + videoOnlyPath, + options.audioOptions ?? {}, + ); + Object.assign(metrics, finalized.metrics); + outputPathToKeep = finalized.outputPath; + return { + outputPath: finalized.outputPath, + metrics, + }; + } catch (error) { + await removeTemporaryExportFile(videoOnlyPath); + await removeTemporaryExportFile(videoOnlyPath.replace(/\.mp4$/, "-final.mp4")); + throw error; + } finally { + nativeStaticLayoutExportSessions.delete(sessionId); + await fs.rm(chunkDirectory, { force: true, recursive: true }).catch(() => undefined); + if (outputPathToKeep !== videoOnlyPath) { + await removeTemporaryExportFile(videoOnlyPath); + } + } +} + export async function enqueueNativeVideoExportFrameWrite( session: NativeVideoExportSession, frameData: Uint8Array | ArrayBuffer, @@ -372,6 +2505,66 @@ export async function resolveNativeVideoEncoder( throw new Error("No usable FFmpeg encoder was available for native export"); } +export function buildNativeVideoAudioMuxArgs( + videoPath: string, + audioInputPath: string, + outputPath: string, + options: NativeVideoExportFinishOptions, +) { + const audioMode = options.audioMode ?? "none"; + const useEditedTrackFiltergraph = + audioMode === "edited-track" && options.editedTrackStrategy === "filtergraph-fast-path"; + const args = [ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-i", + videoPath, + "-i", + audioInputPath, + ]; + + if (audioMode === "trim-source") { + const filter = buildTrimmedSourceAudioFilter(options.trimSegments ?? []); + if (filter) { + args.push("-filter_complex", filter, "-map", "0:v:0", "-map", "[aout]"); + } else { + args.push("-map", "0:v:0", "-map", "1:a:0"); + } + } else if (useEditedTrackFiltergraph) { + const filter = buildEditedTrackSourceAudioFilter( + options.editedTrackSegments ?? [], + options.audioSourceSampleRate ?? 0, + ); + if (!filter) { + throw new Error("Edited-track filtergraph inputs are incomplete for native export"); + } + args.push("-filter_complex", filter, "-map", "0:v:0", "-map", "[aout]"); + } else { + args.push("-map", "0:v:0", "-map", "1:a:0"); + } + + args.push("-c:v", "copy"); + if (audioMode === "copy-source") { + args.push("-c:a", "copy"); + } else { + args.push("-c:a", "aac", "-b:a", "192k"); + } + if ( + typeof options.outputDurationSec === "number" && + Number.isFinite(options.outputDurationSec) && + options.outputDurationSec > 0 + ) { + args.push("-t", formatFfmpegSeconds(options.outputDurationSec * 1000)); + } else if (audioMode !== "copy-source") { + args.push("-shortest"); + } + args.push("-movflags", "+faststart", outputPath); + + return args; +} + export async function muxNativeVideoExportAudio( videoPath: string, options: NativeVideoExportFinishOptions, @@ -420,49 +2613,7 @@ export async function muxNativeVideoExportAudio( `${path.basename(videoPath, path.extname(videoPath))}-final.mp4`, ); - const args = [ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-i", - videoPath, - "-i", - audioInputPath, - ]; - - if (audioMode === "trim-source") { - const filter = buildTrimmedSourceAudioFilter(options.trimSegments ?? []); - if (filter) { - args.push("-filter_complex", filter, "-map", "0:v:0", "-map", "[aout]"); - } else { - args.push("-map", "0:v:0", "-map", "1:a:0"); - } - } else if (useEditedTrackFiltergraph) { - const filter = buildEditedTrackSourceAudioFilter( - options.editedTrackSegments ?? [], - options.audioSourceSampleRate ?? 0, - ); - if (!filter) { - throw new Error("Edited-track filtergraph inputs are incomplete for native export"); - } - args.push("-filter_complex", filter, "-map", "0:v:0", "-map", "[aout]"); - } else { - args.push("-map", "0:v:0", "-map", "1:a:0"); - } - - args.push( - "-c:v", - "copy", - "-c:a", - "aac", - "-b:a", - "192k", - "-shortest", - "-movflags", - "+faststart", - outputPath, - ); + const args = buildNativeVideoAudioMuxArgs(videoPath, audioInputPath, outputPath, options); try { const ffmpegExecStartedAt = getNowMs(); @@ -471,6 +2622,12 @@ export async function muxNativeVideoExportAudio( maxBuffer: 20 * 1024 * 1024, }); metrics.ffmpegExecMs = getNowMs() - ffmpegExecStartedAt; + console.info("[native-video-export] Audio mux completed", { + ffmpegExecMs: metrics.ffmpegExecMs, + audioMode: options.audioMode, + tempVideoBytes: metrics.tempVideoBytes, + muxedVideoBytes: metrics.muxedVideoBytes, + }); await removeTemporaryExportFile(videoPath); return { outputPath, diff --git a/electron/ipc/ffmpeg/filters.test.ts b/electron/ipc/ffmpeg/filters.test.ts index 3569d729..9660fd29 100644 --- a/electron/ipc/ffmpeg/filters.test.ts +++ b/electron/ipc/ffmpeg/filters.test.ts @@ -70,6 +70,34 @@ describe("getAudioSyncAdjustment", () => { ]); }); + it("pads the remaining tail after a measured late start delay", () => { + const filterParts: string[] = []; + appendSyncedAudioFilter(filterParts, "[1:a]", "aout", { + mode: "delay", + delayMs: 18051, + tempoRatio: 1, + durationDeltaMs: 1631070, + }); + + expect(filterParts).toEqual([ + "[1:a]adelay=18051|18051,apad=pad_dur=1613.019,aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS[aout]", + ]); + }); + + it("does not pad when a measured late start already explains the duration gap", () => { + const filterParts: string[] = []; + appendSyncedAudioFilter(filterParts, "[1:a]", "aout", { + mode: "delay", + delayMs: 10000, + tempoRatio: 1, + durationDeltaMs: 10000, + }); + + expect(filterParts).toEqual([ + "[1:a]adelay=10000|10000,aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS[aout]", + ]); + }); + it("can add a small gain boost before resampling", () => { const filterParts: string[] = []; appendSyncedAudioFilter( diff --git a/electron/ipc/ffmpeg/filters.ts b/electron/ipc/ffmpeg/filters.ts index dfc59fbb..7e437525 100644 --- a/electron/ipc/ffmpeg/filters.ts +++ b/electron/ipc/ffmpeg/filters.ts @@ -114,6 +114,13 @@ export function appendSyncedAudioFilter( filters.push(`adelay=${adjustment.delayMs}|${adjustment.delayMs}`); } + if ( + adjustment.mode === "delay" && + adjustment.durationDeltaMs > adjustment.delayMs + 20 + ) { + filters.push(`apad=pad_dur=${formatFfmpegSeconds(adjustment.durationDeltaMs - adjustment.delayMs)}`); + } + if (adjustment.mode === "tempo") { filters.push(...buildAtempoFilters(adjustment.tempoRatio)); } diff --git a/electron/ipc/nativeVideoExport.test.ts b/electron/ipc/nativeVideoExport.test.ts index 6f7fbb4a..2f5ab12e 100644 --- a/electron/ipc/nativeVideoExport.test.ts +++ b/electron/ipc/nativeVideoExport.test.ts @@ -2,7 +2,15 @@ import { describe, expect, it } from "vitest"; import { ATEMPO_FILTER_EPSILON } from "./ffmpeg/filters"; import { buildEditedTrackSourceAudioFilter, + buildNativeConcatArgs, + buildNativeCudaOverlayStaticLayoutArgs, + buildNativeCudaScaleCpuPadStaticLayoutArgs, + buildNativePrecompositedStaticLayoutArgs, + buildNativeStaticBackgroundRenderArgs, + buildNativeStaticLayoutChunks, buildTrimmedSourceAudioFilter, + createNativeSquircleMaskPgmBuffer, + isNativeCudaOutOfMemory, } from "./nativeVideoExport"; describe("buildTrimmedSourceAudioFilter", () => { @@ -118,3 +126,150 @@ describe("buildEditedTrackSourceAudioFilter", () => { ).toBeNull(); }); }); + +describe("native static layout command builders", () => { + const baseConfig = { + inputPath: "input.mp4", + outputPath: "chunk.mp4", + width: 1920, + height: 1080, + frameRate: 60, + bitrate: 8_000_000, + encodingMode: "fast" as const, + contentWidth: 1536, + contentHeight: 864, + offsetX: 192, + offsetY: 108, + backgroundColor: "#101010", + startSec: 120, + durationSec: 60, + }; + + it("builds the primary CUDA overlay layout command", () => { + const args = buildNativeCudaOverlayStaticLayoutArgs(baseConfig); + + expect(args).toContain("-filter_complex"); + expect(args).toContain( + "color=c=0x101010:s=1920x1080:r=60:d=60.000,format=nv12,hwupload_cuda[bg];" + + "[0:v]scale_cuda=w=1536:h=864:format=nv12,fps=60[fg];" + + "[bg][fg]overlay_cuda=192:108:shortest=0:repeatlast=1:eof_action=repeat,trim=duration=60.000,setpts=PTS-STARTPTS[out]", + ); + expect(args).toContain("h264_nvenc"); + expect(args).toContain("p1"); + expect(args).not.toContain("yuv420p"); + expect(args).toEqual(expect.arrayContaining(["-ss", "120.000", "-t", "60.000"])); + }); + + it("builds the stable CUDA scale plus CPU pad fallback command", () => { + const args = buildNativeCudaScaleCpuPadStaticLayoutArgs(baseConfig); + + expect(args).toEqual( + expect.arrayContaining([ + "-vf", + "scale_cuda=w=1536:h=864:format=nv12:passthrough=0,hwdownload,format=nv12,fps=60,pad=w=1920:h=1080:x=192:y=108:color=0x101010", + "-map", + "0:v:0", + "-an", + ]), + ); + }); + + it("sanitizes unsupported background colors to the safe dark fallback", () => { + const args = buildNativeCudaScaleCpuPadStaticLayoutArgs({ + ...baseConfig, + backgroundColor: "linear-gradient(red, blue)", + }); + + expect(args).toContain( + "scale_cuda=w=1536:h=864:format=nv12:passthrough=0,hwdownload,format=nv12,fps=60,pad=w=1920:h=1080:x=192:y=108:color=0x101010", + ); + }); + + it("builds a precomposited background command for image wallpaper and shadow", () => { + const args = buildNativeStaticBackgroundRenderArgs({ + ...baseConfig, + outputPath: "background.png", + backgroundImagePath: "wallpaper.jpg", + maskPath: "mask.pgm", + shadowIntensity: 0.67, + }); + const filterComplex = args[args.indexOf("-filter_complex") + 1]; + + expect(args).toEqual(expect.arrayContaining(["-i", "wallpaper.jpg", "-i", "mask.pgm"])); + expect(filterComplex).toContain( + "scale=w=1920:h=1080:force_original_aspect_ratio=increase,crop=w=1920:h=1080", + ); + expect(filterComplex).toContain("split=3"); + expect(filterComplex).toContain("gblur=sigma=32.16:steps=2"); + expect(filterComplex).toContain("overlay=x=119:y=35:format=auto"); + expect(args).toEqual(expect.arrayContaining(["-frames:v", "1", "background.png"])); + }); + + it("builds a precomposited static layout command with a squircle alpha mask", () => { + const args = buildNativePrecompositedStaticLayoutArgs({ + ...baseConfig, + staticBackgroundPath: "background.png", + maskPath: "mask.pgm", + borderRadius: 12.5, + }); + const filterComplex = args[args.indexOf("-filter_complex") + 1]; + + expect(args).toEqual(expect.arrayContaining(["-i", "background.png", "-i", "mask.pgm"])); + expect(filterComplex).toContain( + "scale_cuda=w=1536:h=864:format=nv12:passthrough=0,hwdownload,format=nv12,fps=60,format=rgba", + ); + expect(filterComplex).toContain("[fgbase][mask]alphamerge[fg]"); + expect(filterComplex).toContain("overlay=x=192:y=108:format=auto"); + expect(args).toContain("h264_nvenc"); + expect(args).toEqual(expect.arrayContaining(["-pix_fmt", "yuv420p"])); + }); + + it("creates an opaque PGM mask for square video corners and a partial mask for radius", () => { + const squareMask = createNativeSquircleMaskPgmBuffer(4, 4, 0); + expect(squareMask.subarray(squareMask.length - 16)).toEqual(Buffer.alloc(16, 255)); + + const roundedMask = createNativeSquircleMaskPgmBuffer(8, 8, 4); + const header = Buffer.from("P5\n8 8\n255\n", "ascii"); + const pixels = roundedMask.subarray(header.length); + expect(pixels[0]).toBeLessThan(255); + expect(pixels[4 * 8 + 4]).toBe(255); + }); + + it("splits long exports into bounded chunks", () => { + expect(buildNativeStaticLayoutChunks(367.5, 120)).toEqual([ + { index: 0, startSec: 0, durationSec: 120 }, + { index: 1, startSec: 120, durationSec: 120 }, + { index: 2, startSec: 240, durationSec: 120 }, + { index: 3, startSec: 360, durationSec: 7.5 }, + ]); + }); + + it("builds concat args for already encoded chunks", () => { + expect(buildNativeConcatArgs({ listPath: "chunks.txt", outputPath: "out.mp4" })).toEqual([ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "concat", + "-safe", + "0", + "-i", + "chunks.txt", + "-c", + "copy", + "-movflags", + "+faststart", + "out.mp4", + ]); + }); + + it("detects CUDA OOM as a retryable fast-path failure", () => { + expect( + isNativeCudaOutOfMemory( + "cu->cuMemAlloc(&data, size) failed -> CUDA_ERROR_OUT_OF_MEMORY: out of memory", + ), + ).toBe(true); + expect(isNativeCudaOutOfMemory("FFmpeg exited with code 1")).toBe(false); + }); +}); diff --git a/electron/ipc/nativeVideoExport.ts b/electron/ipc/nativeVideoExport.ts index 3a8d812b..c2c3d5ed 100644 --- a/electron/ipc/nativeVideoExport.ts +++ b/electron/ipc/nativeVideoExport.ts @@ -1,3 +1,8 @@ +import { + getShadowFilterPadding, + VIDEO_SHADOW_LAYER_PROFILES, +} from "../../src/lib/exporter/shadowProfile"; +import { getSquirclePathPoints } from "../../src/lib/geometry/squircle"; import { ATEMPO_FILTER_EPSILON, buildAtempoFilters } from "./ffmpeg/filters"; const NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL = 4; @@ -33,6 +38,7 @@ export interface NativeVideoExportFinishOptions { audioMode?: NativeVideoExportAudioMode; audioSourcePath?: string | null; audioSourceSampleRate?: number; + outputDurationSec?: number; trimSegments?: NativeVideoExportAudioSegment[]; editedTrackStrategy?: NativeVideoExportEditedTrackStrategy; editedTrackSegments?: NativeVideoExportEditedTrackSegment[]; @@ -50,6 +56,41 @@ export interface NativeVideoAudioMuxMetrics { muxedVideoBytes?: number; } +export type NativeStaticLayoutBackend = + | "cuda-overlay" + | "cuda-scale-cpu-pad" + | "cuda-static-composite" + | "nvidia-cuda-compositor" + | "windows-d3d11-compositor"; + +export interface NativeStaticLayoutExportArgsConfig { + inputPath: string; + outputPath: string; + width: number; + height: number; + frameRate: number; + bitrate: number; + encodingMode: NativeExportEncodingMode; + contentWidth: number; + contentHeight: number; + offsetX: number; + offsetY: number; + backgroundColor: string; + backgroundImagePath?: string | null; + staticBackgroundPath?: string | null; + maskPath?: string | null; + borderRadius?: number; + shadowIntensity?: number; + startSec?: number; + durationSec?: number; +} + +export interface NativeStaticLayoutChunk { + index: number; + startSec: number; + durationSec: number; +} + export function getNativeVideoInputByteSize(width: number, height: number): number { return width * height * NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL; } @@ -107,6 +148,121 @@ function getBitrateArgs(bitrate: number): string[] { ]; } +function getNvencStaticLayoutModeArgs(encodingMode: NativeExportEncodingMode): string[] { + const lowLatencyRateControlArgs = [ + "-rc", + "vbr", + "-multipass", + "disabled", + "-rc-lookahead", + "0", + "-surfaces", + "32", + ]; + switch (encodingMode) { + case "quality": + return ["-preset", "p1", "-tune", "hq", ...lowLatencyRateControlArgs]; + case "balanced": + return ["-preset", "p1", "-tune", "ll", ...lowLatencyRateControlArgs]; + case "fast": + default: + return ["-preset", "p1", "-tune", "ull", ...lowLatencyRateControlArgs]; + } +} + +function formatFfmpegColor(value: string): string { + const trimmed = value.trim(); + const hex = trimmed.match(/^#?([0-9a-f]{6})$/i)?.[1]; + return hex ? `0x${hex.toLowerCase()}` : "0x101010"; +} + +function clampUnitInterval(value: number): number { + if (!Number.isFinite(value)) { + return 0; + } + + return Math.min(1, Math.max(0, value)); +} + +function formatFfmpegNumber(value: number): string { + return Number.isInteger(value) ? String(value) : value.toFixed(6).replace(/0+$/, "").replace(/\.$/, ""); +} + +function isPointInsidePolygon(x: number, y: number, points: Array<{ x: number; y: number }>) { + let inside = false; + for (let index = 0, previousIndex = points.length - 1; index < points.length; previousIndex = index++) { + const current = points[index]; + const previous = points[previousIndex]; + const intersects = + (current.y > y) !== (previous.y > y) && + x < ((previous.x - current.x) * (y - current.y)) / (previous.y - current.y) + current.x; + + if (intersects) { + inside = !inside; + } + } + + return inside; +} + +export function createNativeSquircleMaskPgmBuffer( + width: number, + height: number, + radius: number, +): Buffer { + const safeWidth = Math.max(1, Math.round(width)); + const safeHeight = Math.max(1, Math.round(height)); + const clampedRadius = Math.min(Math.max(0, radius), Math.min(safeWidth, safeHeight) / 2); + const header = Buffer.from(`P5\n${safeWidth} ${safeHeight}\n255\n`, "ascii"); + const pixels = Buffer.alloc(safeWidth * safeHeight, 255); + + if (clampedRadius <= 0.5) { + return Buffer.concat([header, pixels]); + } + + const points = getSquirclePathPoints({ + x: 0, + y: 0, + width: safeWidth, + height: safeHeight, + radius: clampedRadius, + }); + const samples = [ + [0.25, 0.25], + [0.75, 0.25], + [0.25, 0.75], + [0.75, 0.75], + ] as const; + + for (let y = 0; y < safeHeight; y += 1) { + for (let x = 0; x < safeWidth; x += 1) { + let coveredSamples = 0; + for (const [sampleX, sampleY] of samples) { + if (isPointInsidePolygon(x + sampleX, y + sampleY, points)) { + coveredSamples += 1; + } + } + pixels[y * safeWidth + x] = Math.round((coveredSamples / samples.length) * 255); + } + } + + return Buffer.concat([header, pixels]); +} + +function pushFfmpegTimeSliceArgs( + args: string[], + startSec: number | undefined, + durationSec: number | undefined, +) { + if (Number.isFinite(startSec) && (startSec ?? 0) > 0) { + args.push("-ss", formatFfmpegSeconds((startSec ?? 0) * 1000)); + } + + if (Number.isFinite(durationSec) && (durationSec ?? 0) > 0) { + args.push("-t", formatFfmpegSeconds((durationSec ?? 0) * 1000)); + } +} + export function buildNativeVideoExportArgs( encoder: string, options: NativeVideoExportStartOptions, @@ -145,6 +301,274 @@ export function buildNativeVideoExportArgs( return args; } +export function buildNativeCudaOverlayStaticLayoutArgs( + config: NativeStaticLayoutExportArgsConfig, +): string[] { + const backgroundColor = formatFfmpegColor(config.backgroundColor); + const durationSec = formatFfmpegSeconds(Math.max(0.001, config.durationSec ?? 1) * 1000); + const args = ["-y", "-hide_banner", "-loglevel", "error"]; + pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec); + args.push( + "-hwaccel", + "cuda", + "-hwaccel_output_format", + "cuda", + "-i", + config.inputPath, + "-filter_complex", + `color=c=${backgroundColor}:s=${config.width}x${config.height}:r=${config.frameRate}:d=${durationSec},format=nv12,hwupload_cuda[bg];[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12,fps=${config.frameRate}[fg];[bg][fg]overlay_cuda=${config.offsetX}:${config.offsetY}:shortest=0:repeatlast=1:eof_action=repeat,trim=duration=${durationSec},setpts=PTS-STARTPTS[out]`, + "-map", + "[out]", + "-an", + "-r", + String(config.frameRate), + "-c:v", + "h264_nvenc", + ...getNvencStaticLayoutModeArgs(config.encodingMode), + ...getBitrateArgs(config.bitrate), + "-movflags", + "+faststart", + config.outputPath, + ); + return args; +} + +export function buildNativeCudaScaleCpuPadStaticLayoutArgs( + config: NativeStaticLayoutExportArgsConfig, +): string[] { + const backgroundColor = formatFfmpegColor(config.backgroundColor); + const args = ["-y", "-hide_banner", "-loglevel", "error"]; + pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec); + args.push( + "-hwaccel", + "cuda", + "-hwaccel_output_format", + "cuda", + "-i", + config.inputPath, + "-vf", + `scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12:passthrough=0,hwdownload,format=nv12,fps=${config.frameRate},pad=w=${config.width}:h=${config.height}:x=${config.offsetX}:y=${config.offsetY}:color=${backgroundColor}`, + "-map", + "0:v:0", + "-an", + "-r", + String(config.frameRate), + "-c:v", + "h264_nvenc", + ...getNvencStaticLayoutModeArgs(config.encodingMode), + ...getBitrateArgs(config.bitrate), + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + config.outputPath, + ); + return args; +} + +export function buildNativeStaticBackgroundRenderArgs( + config: NativeStaticLayoutExportArgsConfig, +): string[] { + const backgroundColor = formatFfmpegColor(config.backgroundColor); + const args = ["-y", "-hide_banner", "-loglevel", "error"]; + if (config.backgroundImagePath) { + args.push("-i", config.backgroundImagePath); + } else { + args.push( + "-f", + "lavfi", + "-i", + `color=c=${backgroundColor}:s=${config.width}x${config.height}:d=1`, + ); + } + + const shadowStrength = clampUnitInterval(config.shadowIntensity ?? 0); + const shadowLayers = + shadowStrength > 0 && config.maskPath + ? VIDEO_SHADOW_LAYER_PROFILES.map((layer) => ({ + offsetY: layer.offsetScale * shadowStrength, + alpha: clampUnitInterval(layer.alphaScale * shadowStrength), + blur: Math.max(0, layer.blurScale * shadowStrength), + })).filter((layer) => layer.alpha > 0) + : []; + + if (shadowLayers.length > 0 && config.maskPath) { + args.push("-i", config.maskPath); + } + + const filterParts = [ + config.backgroundImagePath + ? `[0:v]scale=w=${config.width}:h=${config.height}:force_original_aspect_ratio=increase,crop=w=${config.width}:h=${config.height},setsar=1,format=rgba[bg0]` + : "[0:v]format=rgba[bg0]", + ]; + let currentBackgroundLabel = "bg0"; + + if (shadowLayers.length > 0) { + filterParts.push( + `[1:v]format=gray,split=${shadowLayers.length}${shadowLayers + .map((_, index) => `[shadow_mask_source_${index}]`) + .join("")}`, + ); + } + + shadowLayers.forEach((layer, index) => { + const padding = getShadowFilterPadding(layer.blur, layer.offsetY); + const paddedWidth = config.contentWidth + padding * 2; + const paddedHeight = config.contentHeight + padding * 2; + const positionedMaskLabel = `shadow_mask_positioned_${index}`; + const shadowMaskLabel = `shadow_mask_${index}`; + const shadowColorLabel = `shadow_color_${index}`; + const shadowLabel = `shadow_${index}`; + const nextBackgroundLabel = `bg${index + 1}`; + const blurFilter = + layer.blur > 0 ? `,gblur=sigma=${formatFfmpegNumber(layer.blur)}:steps=2` : ""; + + filterParts.push( + `[shadow_mask_source_${index}]lut=y=val*${formatFfmpegNumber( + layer.alpha, + )},pad=w=${paddedWidth}:h=${paddedHeight}:x=${padding}:y=${Math.round( + padding + layer.offsetY, + )}:color=black${blurFilter}[${positionedMaskLabel}]`, + `[${positionedMaskLabel}]format=gray[${shadowMaskLabel}]`, + `color=c=black:s=${paddedWidth}x${paddedHeight}:d=1,format=rgba[${shadowColorLabel}]`, + `[${shadowColorLabel}][${shadowMaskLabel}]alphamerge[${shadowLabel}]`, + `[${currentBackgroundLabel}][${shadowLabel}]overlay=x=${ + config.offsetX - padding + }:y=${config.offsetY - padding}:format=auto[${nextBackgroundLabel}]`, + ); + currentBackgroundLabel = nextBackgroundLabel; + }); + + filterParts.push(`[${currentBackgroundLabel}]format=rgba[out]`); + args.push( + "-filter_complex", + filterParts.join(";"), + "-map", + "[out]", + "-frames:v", + "1", + config.outputPath, + ); + return args; +} + +export function buildNativePrecompositedStaticLayoutArgs( + config: NativeStaticLayoutExportArgsConfig, +): string[] { + if (!config.staticBackgroundPath) { + throw new Error("Native precomposited static layout requires a static background path"); + } + + const durationSec = formatFfmpegSeconds(Math.max(0.001, config.durationSec ?? 1) * 1000); + const useMask = Boolean(config.maskPath && (config.borderRadius ?? 0) > 0.5); + const args = ["-y", "-hide_banner", "-loglevel", "error"]; + pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec); + args.push( + "-hwaccel", + "cuda", + "-hwaccel_output_format", + "cuda", + "-i", + config.inputPath, + "-loop", + "1", + "-framerate", + String(config.frameRate), + "-t", + durationSec, + "-i", + config.staticBackgroundPath, + ); + + if (useMask && config.maskPath) { + args.push( + "-loop", + "1", + "-framerate", + String(config.frameRate), + "-t", + durationSec, + "-i", + config.maskPath, + ); + } + + const foregroundFilter = `[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12:passthrough=0,hwdownload,format=nv12,fps=${config.frameRate},format=rgba[fgbase]`; + const maskFilter = useMask + ? ";[2:v]format=gray[mask];[fgbase][mask]alphamerge[fg]" + : ""; + const foregroundLabel = useMask ? "fg" : "fgbase"; + const filterComplex = `${foregroundFilter}${maskFilter};[1:v]format=rgba[bg];[bg][${foregroundLabel}]overlay=x=${config.offsetX}:y=${config.offsetY}:format=auto,trim=duration=${durationSec},setpts=PTS-STARTPTS,format=yuv420p[out]`; + + args.push( + "-filter_complex", + filterComplex, + "-map", + "[out]", + "-an", + "-r", + String(config.frameRate), + "-c:v", + "h264_nvenc", + ...getNvencStaticLayoutModeArgs(config.encodingMode), + ...getBitrateArgs(config.bitrate), + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + config.outputPath, + ); + return args; +} + +export function buildNativeConcatArgs(config: { + listPath: string; + outputPath: string; +}): string[] { + return [ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "concat", + "-safe", + "0", + "-i", + config.listPath, + "-c", + "copy", + "-movflags", + "+faststart", + config.outputPath, + ]; +} + +export function buildNativeStaticLayoutChunks( + durationSec: number, + chunkDurationSec: number, +): NativeStaticLayoutChunk[] { + if (!Number.isFinite(durationSec) || durationSec <= 0) { + return []; + } + + const safeChunkDuration = Math.max(1, Math.min(300, Math.floor(chunkDurationSec))); + const chunks: NativeStaticLayoutChunk[] = []; + for (let startSec = 0, index = 0; startSec < durationSec; startSec += safeChunkDuration, index++) { + chunks.push({ + index, + startSec, + durationSec: Math.min(safeChunkDuration, durationSec - startSec), + }); + } + + return chunks; +} + +export function isNativeCudaOutOfMemory(stderr: string): boolean { + return /CUDA_ERROR_OUT_OF_MEMORY|cuMemAlloc.+out of memory/i.test(stderr); +} + function formatFfmpegSeconds(milliseconds: number): string { return (milliseconds / 1000).toFixed(3); } diff --git a/electron/ipc/paths/binaries.ts b/electron/ipc/paths/binaries.ts index 45ec80da..45a944fa 100644 --- a/electron/ipc/paths/binaries.ts +++ b/electron/ipc/paths/binaries.ts @@ -63,6 +63,10 @@ export function resolvePreferredWindowsNativeHelperPath( ); const prebundledPath = getPrebundledNativeHelperPath(binaryName); + if (app.isPackaged && existsSync(prebundledPath)) { + return prebundledPath; + } + if (existsSync(buildOutputPath)) { return buildOutputPath; } diff --git a/electron/ipc/project/manager.test.ts b/electron/ipc/project/manager.test.ts index 51fca45d..f769eec4 100644 --- a/electron/ipc/project/manager.test.ts +++ b/electron/ipc/project/manager.test.ts @@ -115,4 +115,26 @@ describe("local media path policy", () => { await expect(fs.readFile(thumbnailPath, "utf8")).resolves.toBe("png-thumbnail"); }); + + it("loads project files that start with a UTF-8 byte order mark", async () => { + const videoPath = path.join(tempPath, "recording.mp4"); + const projectPath = path.join(tempPath, "recording.recordly"); + await fs.writeFile(videoPath, "test-video"); + await fs.writeFile( + projectPath, + `\uFEFF${JSON.stringify({ + version: 1, + videoPath, + editor: {}, + })}`, + "utf-8", + ); + + const { loadProjectFromPath } = await import("./manager"); + + const result = await loadProjectFromPath(projectPath); + expect(result.success).toBe(true); + expect(result.path).toBe(projectPath); + expect(result.project).toMatchObject({ videoPath }); + }); }); diff --git a/electron/ipc/project/manager.ts b/electron/ipc/project/manager.ts index 9e61605a..07ccc5be 100644 --- a/electron/ipc/project/manager.ts +++ b/electron/ipc/project/manager.ts @@ -1,33 +1,33 @@ -import { constants as fsConstants } from "node:fs"; -import { existsSync } from "node:fs"; +import { existsSync, constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { app } from "electron"; import { RECORDINGS_DIR, USER_DATA_PATH } from "../../appPaths"; import { isSupportedLocalMediaPath } from "../../mediaTypes"; import { - PROJECT_FILE_EXTENSION, LEGACY_PROJECT_FILE_EXTENSIONS, - PROJECTS_DIRECTORY_NAME, - PROJECT_THUMBNAIL_SUFFIX, - RECENT_PROJECTS_FILE, MAX_RECENT_PROJECTS, + PROJECT_FILE_EXTENSION, + PROJECT_THUMBNAIL_SUFFIX, + PROJECTS_DIRECTORY_NAME, + RECENT_PROJECTS_FILE, RECORDINGS_SETTINGS_FILE, } from "../constants"; -import type { ProjectLibraryEntry, RecordingSessionData } from "../types"; import { + approvedLocalReadPaths, currentProjectPath, setCurrentProjectPath, - setCurrentVideoPath, setCurrentRecordingSession, - approvedLocalReadPaths, + setCurrentVideoPath, setCustomRecordingsDir, setRecordingsDirLoaded, } from "../state"; +import type { ProjectLibraryEntry, RecordingSessionData } from "../types"; import { + getRecordingsDir, normalizePath, normalizeVideoSourcePath, - getRecordingsDir, + parseJsonWithByteOrderMark, } from "../utils"; @@ -247,7 +247,7 @@ export async function saveProjectThumbnail(projectPath: string, thumbnailDataUrl export async function loadRecentProjectPaths() { try { const content = await fs.readFile(RECENT_PROJECTS_FILE, "utf-8"); - const parsed = JSON.parse(content) as { paths?: unknown }; + const parsed = parseJsonWithByteOrderMark<{ paths?: unknown }>(content); return Array.isArray(parsed.paths) ? parsed.paths.filter( (value): value is string => @@ -364,7 +364,7 @@ export async function loadProjectFromPath(projectPath: string) { let project: unknown; try { const content = await fs.readFile(normalizedPath, "utf-8"); - project = JSON.parse(content); + project = parseJsonWithByteOrderMark(content); } catch (error) { return { success: false, diff --git a/electron/ipc/project/session.ts b/electron/ipc/project/session.ts index 2f0001c1..3c126e6d 100644 --- a/electron/ipc/project/session.ts +++ b/electron/ipc/project/session.ts @@ -3,7 +3,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { RECORDING_SESSION_MANIFEST_SUFFIX } from "../constants"; import type { RecordingSessionData, RecordingSessionManifest } from "../types"; -import { normalizeVideoSourcePath } from "../utils"; +import { normalizeVideoSourcePath, parseJsonWithByteOrderMark } from "../utils"; function normalizeRecordingTimeOffsetMs(value: unknown): number { return typeof value === "number" && Number.isFinite(value) ? Math.round(value) : 0; @@ -51,7 +51,8 @@ export async function resolveRecordingSessionManifest( try { const content = await fs.readFile(manifestPath, "utf-8"); - const parsed = JSON.parse(content) as Partial; + const parsed = + parseJsonWithByteOrderMark>(content); if (parsed.version !== 1 && parsed.version !== 2) { return null; } diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.ts index dca8f3fc..87c16b1f 100644 --- a/electron/ipc/recording/diagnostics.test.ts +++ b/electron/ipc/recording/diagnostics.test.ts @@ -140,7 +140,7 @@ describe("getCompanionAudioFallbackPaths", () => { await Promise.all([ fs.writeFile(videoPath, "video"), fs.writeFile(micPath, "mic"), - fs.writeFile(`${micPath}.json`, JSON.stringify({ startDelayMs: 2750 })), + fs.writeFile(`${micPath}.json`, `\ufeff${JSON.stringify({ startDelayMs: 2750 })}`), ]); execFileMock.mockImplementation( @@ -166,6 +166,17 @@ describe("getCompanionAudioFallbackPaths", () => { }); }); + it("scales audio mux timeout for long recordings", async () => { + const { getRecordingAudioMuxTimeoutMs } = await import("./diagnostics"); + + expect(getRecordingAudioMuxTimeoutMs(0)).toBe(5 * 60 * 1000); + expect(getRecordingAudioMuxTimeoutMs(29 * 60 + 29.41)).toBeGreaterThan(120000); + expect(getRecordingAudioMuxTimeoutMs(29 * 60 + 29.41)).toBeCloseTo( + (29 * 60 + 29.41) * 1000 + 60 * 1000, + 0, + ); + }); + it("ignores invalid sidecar timing metadata values", async () => { const micPath = path.join(tempRoot, "recording.mic.wav"); await Promise.all([ diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts index fb7b6c4a..b445d704 100644 --- a/electron/ipc/recording/diagnostics.ts +++ b/electron/ipc/recording/diagnostics.ts @@ -5,9 +5,12 @@ import { COMPANION_AUDIO_LAYOUTS } from "../constants"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import { lastNativeCaptureDiagnostics, setLastNativeCaptureDiagnostics } from "../state"; import type { CompanionAudioCandidate, NativeCaptureDiagnostics } from "../types"; +import { parseJsonWithByteOrderMark } from "../utils"; const execFileAsync = promisify(execFile); export const MIN_VALID_RECORDED_VIDEO_BYTES = 1024; +export const RECORDING_AUDIO_MUX_MIN_TIMEOUT_MS = 5 * 60 * 1000; +export const RECORDING_AUDIO_MUX_MAX_TIMEOUT_MS = 2 * 60 * 60 * 1000; type CompanionAudioTimingMetadata = { startDelayMs?: number; @@ -53,6 +56,18 @@ export function parseFfmpegDurationSeconds(stderr: string) { return hours * 3600 + minutes * 60 + seconds; } +export function getRecordingAudioMuxTimeoutMs(durationSeconds: number) { + if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) { + return RECORDING_AUDIO_MUX_MIN_TIMEOUT_MS; + } + + const realtimeMuxBudgetMs = Math.ceil(durationSeconds * 1000) + 60 * 1000; + return Math.min( + RECORDING_AUDIO_MUX_MAX_TIMEOUT_MS, + Math.max(RECORDING_AUDIO_MUX_MIN_TIMEOUT_MS, realtimeMuxBudgetMs), + ); +} + /** Probe the duration of a media file (in seconds) using the container header. */ export async function probeMediaDurationSeconds(filePath: string): Promise { const ffmpegPath = getFfmpegBinaryPath(); @@ -108,7 +123,8 @@ async function readCompanionAudioTimingMetadata( ): Promise { try { const raw = await fs.readFile(`${companionPath}.json`, "utf8"); - const parsed = JSON.parse(raw) as CompanionAudioTimingMetadata | null; + const parsed = + parseJsonWithByteOrderMark(raw); if (!parsed || typeof parsed !== "object") { return null; } diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts index 3ebe321b..769d22ca 100644 --- a/electron/ipc/recording/mac.ts +++ b/electron/ipc/recording/mac.ts @@ -10,27 +10,28 @@ import { import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import { appendSyncedAudioFilter, getAudioSyncAdjustment } from "../ffmpeg/filters"; import { - nativeScreenRecordingActive, - setNativeScreenRecordingActive, - setNativeCaptureProcess, - nativeCaptureOutputBuffer, - nativeCaptureTargetPath, - setNativeCaptureTargetPath, - nativeCaptureStopRequested, - setNativeCaptureStopRequested, - nativeCaptureSystemAudioPath, - setNativeCaptureSystemAudioPath, - nativeCaptureMicrophonePath, - setNativeCaptureMicrophonePath, lastNativeCaptureDiagnostics, - setCurrentVideoPath, - setCurrentProjectPath, + nativeCaptureMicrophonePath, + nativeCaptureOutputBuffer, + nativeCaptureStopRequested, + nativeCaptureSystemAudioPath, + nativeCaptureTargetPath, + nativeScreenRecordingActive, selectedSource, + setCurrentProjectPath, + setCurrentVideoPath, + setNativeCaptureMicrophonePath, + setNativeCaptureProcess, + setNativeCaptureStopRequested, + setNativeCaptureSystemAudioPath, + setNativeCaptureTargetPath, + setNativeScreenRecordingActive, } from "../state"; import type { AudioSyncAdjustment } from "../types"; import { isAutoRecordingPath, moveFileWithOverwrite } from "../utils"; import { getFileSizeIfPresent, + getRecordingAudioMuxTimeoutMs, getUsableCompanionAudioCandidates, probeMediaDurationSeconds, recordNativeCaptureDiagnostics, @@ -163,6 +164,7 @@ export async function muxNativeMacRecordingWithAudio( } const videoDuration = await probeMediaDurationSeconds(videoPath); + const muxTimeoutMs = getRecordingAudioMuxTimeoutMs(videoDuration); const audioAdjustments: Map = new Map(); if (videoDuration > 0) { @@ -207,6 +209,9 @@ export async function muxNativeMacRecordingWithAudio( filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]"); args = [ "-y", + "-hide_banner", + "-nostdin", + "-nostats", ...inputs, "-filter_complex", filterParts.join(";"), @@ -234,6 +239,9 @@ export async function muxNativeMacRecordingWithAudio( appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment); args = [ "-y", + "-hide_banner", + "-nostdin", + "-nostats", ...inputs, "-filter_complex", filterParts.join(";"), @@ -255,7 +263,10 @@ export async function muxNativeMacRecordingWithAudio( console.log("[mux] Running ffmpeg:", ffmpegPath, args.join(" ")); try { - await execFileAsync(ffmpegPath, args, { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }); + await execFileAsync(ffmpegPath, args, { + timeout: muxTimeoutMs, + maxBuffer: 20 * 1024 * 1024, + }); await validateRecordedVideo(mixedOutputPath); } catch (error) { const execError = error as NodeJS.ErrnoException & { stderr?: string }; diff --git a/electron/ipc/recording/prune.ts b/electron/ipc/recording/prune.ts index 2ade345d..f7ee5e27 100644 --- a/electron/ipc/recording/prune.ts +++ b/electron/ipc/recording/prune.ts @@ -15,6 +15,7 @@ import { isAutoRecordingPath, normalizePath, normalizeVideoSourcePath, + parseJsonWithByteOrderMark, } from "../utils"; export async function hasSiblingProjectFile(videoPath: string) { @@ -71,10 +72,22 @@ async function loadSavedProjectMediaPaths() { }) .map(async (entry) => { const projectPath = path.join(projectsDir, entry.name); - const rawProject = JSON.parse(await fs.readFile(projectPath, "utf-8")) as { + let rawProject: { videoPath?: unknown; editor?: { webcam?: { sourcePath?: unknown } }; }; + try { + rawProject = parseJsonWithByteOrderMark<{ + videoPath?: unknown; + editor?: { webcam?: { sourcePath?: unknown } }; + }>(await fs.readFile(projectPath, "utf-8")); + } catch (error) { + console.warn("[prune] Skipping unreadable project while pruning recordings", { + projectPath, + error, + }); + return; + } const candidatePaths = [ rawProject.videoPath, rawProject.editor?.webcam?.sourcePath, diff --git a/electron/ipc/recording/windows.ts b/electron/ipc/recording/windows.ts index e0336413..4d498192 100644 --- a/electron/ipc/recording/windows.ts +++ b/electron/ipc/recording/windows.ts @@ -27,6 +27,7 @@ import type { AudioSyncAdjustment, PauseSegment } from "../types"; import { moveFileWithOverwrite } from "../utils"; import { getCompanionAudioStartDelayMs, + getRecordingAudioMuxTimeoutMs, probeMediaDurationSeconds, validateRecordedVideo, } from "./diagnostics"; @@ -194,6 +195,7 @@ export async function muxNativeWindowsVideoWithAudio( if (audioInputs.length === 0) return; const videoDuration = await probeMediaDurationSeconds(videoPath); + const muxTimeoutMs = getRecordingAudioMuxTimeoutMs(videoDuration); const audioAdjustments: Map = new Map(); if (videoDuration > 0) { @@ -278,6 +280,9 @@ export async function muxNativeWindowsVideoWithAudio( ffmpegPath, [ "-y", + "-hide_banner", + "-nostdin", + "-nostats", ...inputs, "-filter_complex", filterParts.join(";"), @@ -294,7 +299,7 @@ export async function muxNativeWindowsVideoWithAudio( "-shortest", mixedOutputPath, ], - { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, + { timeout: muxTimeoutMs, maxBuffer: 20 * 1024 * 1024 }, ); } else { const pauseFilter = buildPausedAudioFilter( @@ -329,6 +334,9 @@ export async function muxNativeWindowsVideoWithAudio( ffmpegPath, [ "-y", + "-hide_banner", + "-nostdin", + "-nostats", ...inputs, "-filter_complex", filterParts.join(";"), @@ -345,7 +353,7 @@ export async function muxNativeWindowsVideoWithAudio( "-shortest", mixedOutputPath, ], - { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, + { timeout: muxTimeoutMs, maxBuffer: 20 * 1024 * 1024 }, ); } diff --git a/electron/ipc/register/export.test.ts b/electron/ipc/register/export.test.ts new file mode 100644 index 00000000..29ddb00a --- /dev/null +++ b/electron/ipc/register/export.test.ts @@ -0,0 +1,62 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("electron", () => ({ + app: { + getAppPath: () => process.cwd(), + getPath: () => process.env.TEMP ?? process.cwd(), + isPackaged: false, + }, + BrowserWindow: { + fromWebContents: () => null, + }, + dialog: { + showSaveDialog: vi.fn(), + }, + ipcMain: { + handle: vi.fn(), + }, + powerSaveBlocker: { + isStarted: () => true, + start: () => 1, + stop: vi.fn(), + }, +})); + +vi.mock("../ffmpeg/binary", () => ({ + getFfmpegBinaryPath: () => "ffmpeg", +})); + +import { moveExportedTempFile } from "./export"; + +const tempDirs: string[] = []; + +async function makeTempDir() { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "recordly-export-move-")); + tempDirs.push(dir); + return dir; +} + +afterEach(async () => { + await Promise.allSettled( + tempDirs.splice(0).map((dir) => fs.rm(dir, { force: true, recursive: true })), + ); +}); + +describe("moveExportedTempFile", () => { + it("moves an app-managed export temp file to the selected destination", async () => { + const dir = await makeTempDir(); + const tempPath = path.join(dir, "export-temp.mp4"); + const destinationPath = path.join(dir, "export-final.mp4"); + await fs.writeFile(tempPath, "recordly-export"); + + await moveExportedTempFile(tempPath, destinationPath); + + await expect(fs.readFile(destinationPath, "utf8")).resolves.toBe( + "recordly-export", + ); + await expect(fs.access(tempPath)).rejects.toThrow(); + }); +}); diff --git a/electron/ipc/register/export.ts b/electron/ipc/register/export.ts index 63b6f6df..05af28a9 100644 --- a/electron/ipc/register/export.ts +++ b/electron/ipc/register/export.ts @@ -15,6 +15,7 @@ import { } from "../export/exportStream"; import { enqueueNativeVideoExportFrameWrite, + exportNativeStaticLayoutVideo, flushNativeVideoExportPendingWriteRequests, getNativeVideoExportMaxQueuedWriteBytes, getNativeVideoExportSessionError, @@ -22,8 +23,11 @@ import { isIgnorableNativeVideoExportStreamError, muxExportedVideoAudioBuffer, muxNativeVideoExportAudio, + type NativeStaticLayoutExportOptions, type NativeVideoExportSession, + nativeStaticLayoutExportSessions, nativeVideoExportSessions, + probeNativeVideoMetadata, removeTemporaryExportFile, resolveNativeVideoEncoder, sendNativeVideoExportWriteFrameResult, @@ -39,7 +43,13 @@ import { } from "../nativeVideoExport"; import { approveUserPath } from "../utils"; -async function moveExportedTempFile(tempPath: string, destinationPath: string) { +function getPartialExportDestinationPath(destinationPath: string) { + const parsed = path.parse(destinationPath); + const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + return path.join(parsed.dir, `.recordly-partial-${parsed.name}-${suffix}${parsed.ext}`); +} + +export async function moveExportedTempFile(tempPath: string, destinationPath: string) { await fs.mkdir(path.dirname(destinationPath), { recursive: true }); try { await fs.rename(tempPath, destinationPath); @@ -53,17 +63,33 @@ async function moveExportedTempFile(tempPath: string, destinationPath: string) { // exporting to a different volume still works. } - await fs.copyFile(tempPath, destinationPath); + const partialDestinationPath = getPartialExportDestinationPath(destinationPath); try { - await fs.rm(tempPath, { force: true }); - } catch (unlinkError) { - // Copy succeeded, so the export itself is safe; surface the leaked temp - // path instead of silently swallowing the failure so operators can - // reclaim disk space manually if the OS temp reaper misses it. - console.warn( - `[export] Failed to remove temp file after cross-volume copy (${tempPath}):`, - unlinkError, - ); + await fs.copyFile(tempPath, partialDestinationPath); + try { + await fs.rename(partialDestinationPath, destinationPath); + } catch (renameError) { + const code = (renameError as NodeJS.ErrnoException).code; + if (code !== "EEXIST" && code !== "EPERM") { + throw renameError; + } + await fs.rm(destinationPath, { force: true }); + await fs.rename(partialDestinationPath, destinationPath); + } + try { + await fs.rm(tempPath, { force: true }); + } catch (unlinkError) { + // Copy succeeded, so the export itself is safe; surface the leaked temp + // path instead of silently swallowing the failure so operators can + // reclaim disk space manually if the OS temp reaper misses it. + console.warn( + `[export] Failed to remove temp file after cross-volume copy (${tempPath}):`, + unlinkError, + ); + } + } catch (error) { + await fs.rm(partialDestinationPath, { force: true }).catch(() => undefined); + throw error; } } @@ -218,6 +244,86 @@ export function registerExportHandlers() { }, ); + ipcMain.handle("probe-native-video-metadata", async (_, filePath: string) => { + try { + if (typeof filePath !== "string" || filePath.trim().length === 0) { + throw new Error("Native metadata probe requires a file path"); + } + + const metadata = await probeNativeVideoMetadata(getFfmpegBinaryPath(), filePath); + return { + success: true, + metadata, + }; + } catch (error) { + console.warn("[probe-native-video-metadata] Failed:", error); + return { + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } + }); + + ipcMain.handle( + "native-static-layout-export", + async (event, options: NativeStaticLayoutExportOptions) => { + try { + if (!options || typeof options.inputPath !== "string") { + throw new Error("Native static layout export requires an input path"); + } + + const result = await exportNativeStaticLayoutVideo( + getFfmpegBinaryPath(), + options, + (progress) => { + if (event.sender.isDestroyed()) { + return; + } + + event.sender.send("native-static-layout-export-progress", progress); + }, + ); + registerOwnedExportPath(result.outputPath); + const primaryBackend = result.metrics.chunks[0]?.backend; + return { + success: true, + tempPath: result.outputPath, + encoderName: + primaryBackend === "nvidia-cuda-compositor" + ? "nvidia-cuda-compositor" + : primaryBackend === "windows-d3d11-compositor" + ? "windows-d3d11-compositor" + : result.metrics.chunkCount > 1 + ? "chunked-h264-nvenc" + : "static-layout-h264-nvenc", + metrics: result.metrics, + }; + } catch (error) { + console.warn("[native-static-layout-export] Failed:", error); + return { + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } + }, + ); + + ipcMain.handle("native-static-layout-export-cancel", async (_, sessionId: string) => { + const session = nativeStaticLayoutExportSessions.get(sessionId); + if (!session) { + return { success: true }; + } + + session.terminating = true; + try { + session.currentProcess?.kill("SIGKILL"); + } catch { + // Process may already be closed. + } + + return { success: true }; + }); + ipcMain.on( "native-video-export-write-frame-async", ( diff --git a/electron/ipc/register/project.ts b/electron/ipc/register/project.ts index 570d7b58..c56af7ac 100644 --- a/electron/ipc/register/project.ts +++ b/electron/ipc/register/project.ts @@ -1,45 +1,46 @@ -import { constants as fsConstants } from "node:fs"; import { randomUUID } from "node:crypto"; +import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { dialog, ipcMain, shell } from "electron"; import { RECORDINGS_DIR } from "../../appPaths"; import { buildMediaUrl, getMediaServerBaseUrl } from "../../mediaServer"; import { - PROJECT_FILE_EXTENSION, LEGACY_PROJECT_FILE_EXTENSIONS, + PROJECT_FILE_EXTENSION, } from "../constants"; -import { - currentProjectPath, - setCurrentProjectPath, - currentVideoPath, - setCurrentVideoPath, - currentRecordingSession, - setCurrentRecordingSession, -} from "../state"; import { getProjectsDir, getProjectThumbnailPath, isPathInsideDirectory, isTrustedProjectPath, listProjectLibraryEntries, - loadRecentProjectPaths, loadProjectFromPath, + loadRecentProjectPaths, persistRecordingsDirectorySetting, + rememberRecentProject, replaceApprovedSessionLocalReadPaths, resolveApprovedLocalMediaPath, - rememberRecentProject, - saveRecentProjectPaths, saveProjectThumbnail, + saveRecentProjectPaths, } from "../project/manager"; +import { persistRecordingSessionManifest, resolveRecordingSession } from "../project/session"; import { + currentProjectPath, + currentRecordingSession, + currentVideoPath, + setCurrentProjectPath, + setCurrentRecordingSession, + setCurrentVideoPath, +} from "../state"; +import { + approveUserPath, + getRecordingsDir, getTelemetryPathForVideo, isAutoRecordingPath, - getRecordingsDir, - approveUserPath, normalizeVideoSourcePath, + parseJsonWithByteOrderMark, } from "../utils"; -import { persistRecordingSessionManifest, resolveRecordingSession } from "../project/session"; function normalizeRecordingTimeOffsetMs(value: unknown): number { return typeof value === "number" && Number.isFinite(value) ? Math.round(value) : 0; @@ -158,7 +159,7 @@ async function ensureNamedProjectSaveDoesNotOverwriteDifferentProject( try { const existingProjectRaw = await fs.readFile(targetProjectPath, "utf-8"); - const existingProjectData = JSON.parse(existingProjectRaw) as unknown; + const existingProjectData = parseJsonWithByteOrderMark(existingProjectRaw); const existingProjectId = getProjectId(existingProjectData); const existingVideoPath = getProjectVideoPath(existingProjectData); @@ -607,7 +608,7 @@ export function registerProjectHandlers() { await fs.unlink(resolvedPath); // Also delete the cursor telemetry sidecar if it exists const telemetryPath = getTelemetryPathForVideo(resolvedPath); - await fs.unlink(telemetryPath).catch(() => {}); + await fs.unlink(telemetryPath).catch(() => undefined); const currentResolved = currentVideoPath ? await fs.realpath(currentVideoPath).catch(() => currentVideoPath) : null; diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 0e491b80..8ca4720f 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -134,6 +134,7 @@ import { getTelemetryPathForVideo, moveFileWithOverwrite, normalizeVideoSourcePath, + parseJsonWithByteOrderMark, parseWindowId, } from "../utils"; import { resolveWindowsCaptureDisplay } from "../windowsCaptureSelection"; @@ -1341,11 +1342,15 @@ export function registerRecordingHandlers( const telemetryPath = getTelemetryPathForVideo(targetVideoPath); try { const content = await fs.readFile(telemetryPath, "utf-8"); - const parsed = JSON.parse(content); + const parsed = parseJsonWithByteOrderMark(content); + const parsedObject = + parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as { samples?: unknown }) + : null; const rawSamples = Array.isArray(parsed) ? parsed - : Array.isArray(parsed?.samples) - ? parsed.samples + : Array.isArray(parsedObject?.samples) + ? parsedObject.samples : []; const samples: CursorTelemetryPoint[] = rawSamples diff --git a/electron/ipc/register/settings.ts b/electron/ipc/register/settings.ts index 9d323ac5..3f05a590 100644 --- a/electron/ipc/register/settings.ts +++ b/electron/ipc/register/settings.ts @@ -3,20 +3,21 @@ import { app, ipcMain } from "electron"; import { hideCursor } from "../../cursorHider"; import { closeCountdownWindow, createCountdownWindow, getCountdownWindow } from "../../windows"; import { - SHORTCUTS_FILE, - RECORDINGS_SETTINGS_FILE, COUNTDOWN_SETTINGS_FILE, + RECORDINGS_SETTINGS_FILE, + SHORTCUTS_FILE, } from "../constants"; import { - countdownTimer, - setCountdownTimer, countdownCancelled, - setCountdownCancelled, countdownInProgress, - setCountdownInProgress, countdownRemaining, + countdownTimer, + setCountdownCancelled, + setCountdownInProgress, setCountdownRemaining, + setCountdownTimer, } from "../state"; +import { parseJsonWithByteOrderMark } from "../utils"; export function registerSettingsHandlers() { ipcMain.handle('app:getVersion', () => { @@ -42,7 +43,7 @@ export function registerSettingsHandlers() { ipcMain.handle('get-shortcuts', async () => { try { const data = await fs.readFile(SHORTCUTS_FILE, 'utf-8'); - return JSON.parse(data); + return parseJsonWithByteOrderMark(data); } catch { return null; } @@ -64,7 +65,7 @@ export function registerSettingsHandlers() { ipcMain.handle('get-recording-preferences', async () => { try { const content = await fs.readFile(RECORDINGS_SETTINGS_FILE, 'utf-8') - const parsed = JSON.parse(content) as Record + const parsed = parseJsonWithByteOrderMark>(content) return { success: true, microphoneEnabled: parsed.microphoneEnabled === true, @@ -81,7 +82,7 @@ export function registerSettingsHandlers() { let existing: Record = {} try { const content = await fs.readFile(RECORDINGS_SETTINGS_FILE, 'utf-8') - existing = JSON.parse(content) as Record + existing = parseJsonWithByteOrderMark>(content) } catch { // file doesn't exist yet } @@ -97,7 +98,7 @@ export function registerSettingsHandlers() { ipcMain.handle('get-countdown-delay', async () => { try { const content = await fs.readFile(COUNTDOWN_SETTINGS_FILE, 'utf-8') - const parsed = JSON.parse(content) as { delay?: number } + const parsed = parseJsonWithByteOrderMark<{ delay?: number }>(content) return { success: true, delay: parsed.delay ?? 3 } } catch { return { success: true, delay: 3 } diff --git a/electron/ipc/utils.ts b/electron/ipc/utils.ts index 5b7e040e..3f2efb06 100644 --- a/electron/ipc/utils.ts +++ b/electron/ipc/utils.ts @@ -1,15 +1,15 @@ -import { createRequire } from "node:module"; import fs from "node:fs/promises"; +import { createRequire } from "node:module"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { app } from "electron"; import { RECORDINGS_DIR } from "../appPaths"; -import { RECORDINGS_SETTINGS_FILE, AUTO_RECORDING_PREFIX } from "./constants"; +import { AUTO_RECORDING_PREFIX, RECORDINGS_SETTINGS_FILE } from "./constants"; import { approvedLocalReadPaths, customRecordingsDir, - setCustomRecordingsDir, recordingsDirLoaded, + setCustomRecordingsDir, setRecordingsDirLoaded, } from "./state"; @@ -49,6 +49,14 @@ export function normalizeVideoSourcePath(videoPath?: string | null): string | nu return trimmed; } +export function stripJsonByteOrderMark(content: string) { + return content.charCodeAt(0) === 0xfeff ? content.slice(1) : content; +} + +export function parseJsonWithByteOrderMark(content: string): T { + return JSON.parse(stripJsonByteOrderMark(content)) as T; +} + export function parseWindowId(sourceId?: string) { if (!sourceId) return null; const match = sourceId.match(/^window:(\d+)/); @@ -89,7 +97,7 @@ async function loadRecordingsDirectorySetting() { try { const content = await fs.readFile(RECORDINGS_SETTINGS_FILE, "utf-8"); - const parsed = JSON.parse(content) as { recordingsDir?: unknown }; + const parsed = parseJsonWithByteOrderMark<{ recordingsDir?: unknown }>(content); if (typeof parsed.recordingsDir === "string" && parsed.recordingsDir.trim()) { setCustomRecordingsDir(path.resolve(parsed.recordingsDir)); } diff --git a/electron/main.ts b/electron/main.ts index 7607913b..0bbb0be6 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -892,13 +892,19 @@ app.whenReady().then(async () => { registerExtensionIpcHandlers(); - if (IS_SMOKE_EXPORT) { + if (IS_SMOKE_EXPORT || process.env.RECORDLY_DEV_OPEN_RECORDING_INPUT) { await logSmokeExportGpuDiagnostics(); - const smokeSource = - process.env.RECORDLY_SMOKE_EXPORT_PROJECT ?? - process.env.RECORDLY_SMOKE_EXPORT_INPUT ?? - ""; - console.log(`[smoke-export] Starting editor smoke export for ${smokeSource}`); + if (IS_SMOKE_EXPORT) { + const smokeSource = + process.env.RECORDLY_SMOKE_EXPORT_PROJECT ?? + process.env.RECORDLY_SMOKE_EXPORT_INPUT ?? + ""; + console.log(`[smoke-export] Starting editor smoke export for ${smokeSource}`); + } else { + console.log( + `[dev-open-recording] Starting editor for ${process.env.RECORDLY_DEV_OPEN_RECORDING_INPUT}`, + ); + } createEditorWindowWrapper(); return; } diff --git a/electron/native/bin/win32-x64/cursor-monitor.exe b/electron/native/bin/win32-x64/cursor-monitor.exe index cd3e7611..ef71282d 100644 Binary files a/electron/native/bin/win32-x64/cursor-monitor.exe and b/electron/native/bin/win32-x64/cursor-monitor.exe differ diff --git a/electron/native/bin/win32-x64/helpers-manifest.json b/electron/native/bin/win32-x64/helpers-manifest.json index 0ae7bb77..6b3d48aa 100644 --- a/electron/native/bin/win32-x64/helpers-manifest.json +++ b/electron/native/bin/win32-x64/helpers-manifest.json @@ -5,17 +5,24 @@ "helpers": { "wgc-capture": { "binaryName": "wgc-capture.exe", - "binarySha256": "bb4c2aa4141e81e1a05b54bd49dbb114eb3a5ff4a666bf7d26525a9da585128b", + "binarySha256": "424c64fc9569e8f6add29130ee935dfcb4451c07c1036d3738c22d9b79ad39cd", "sourceDir": "electron/native/wgc-capture", - "sourceFingerprint": "5bb02a69049a02909d38188cae1dfdfb94fd49465b51ed6ab78a98092b7520cc", - "updatedAt": "2026-04-25T09:17:16.237Z" + "sourceFingerprint": "4fd7e2f5d0e804a8aaaaddcba3a3e02b42739c0148a0d38be24325ddbb659ec1", + "updatedAt": "2026-05-03T15:39:44.054Z" }, "cursor-monitor": { "binaryName": "cursor-monitor.exe", - "binarySha256": "b0732abc06998a40c3e95078465ad750a6169901944571c39cfd7996effe39c0", + "binarySha256": "6ae6d91103b6e891a851e8ea5791e1c1f9aaab700134c18bc4c46cfffd7fdd12", "sourceDir": "electron/native/cursor-monitor", "sourceFingerprint": "6ad1b8b50bb336f2a48937b06f5ec56d90b6ab4a3e56a4bca278cf67a5d3e52e", - "updatedAt": "2026-03-29T02:15:38.286Z" + "updatedAt": "2026-05-03T15:39:52.446Z" + }, + "recordly-gpu-export": { + "binaryName": "recordly-gpu-export.exe", + "binarySha256": "9b3d4dff520356e5db563cc3992d777f8bab8eaf6d0bb718f9c1d7d7da37fac5", + "sourceDir": "electron/native/gpu-export-probe", + "sourceFingerprint": "75bf080c4a5cbbcb1d42fb088a1545130c613e42d9e7681cc23f9151d1c8072b", + "updatedAt": "2026-05-03T15:39:47.938Z" } } } diff --git a/electron/native/bin/win32-x64/recordly-gpu-export.exe b/electron/native/bin/win32-x64/recordly-gpu-export.exe new file mode 100644 index 00000000..2a64a27d Binary files /dev/null and b/electron/native/bin/win32-x64/recordly-gpu-export.exe differ diff --git a/electron/native/bin/win32-x64/wgc-capture.exe b/electron/native/bin/win32-x64/wgc-capture.exe index d16a0724..9cc29a56 100644 Binary files a/electron/native/bin/win32-x64/wgc-capture.exe and b/electron/native/bin/win32-x64/wgc-capture.exe differ diff --git a/electron/native/gpu-export-probe/CMakeLists.txt b/electron/native/gpu-export-probe/CMakeLists.txt new file mode 100644 index 00000000..31a718a0 --- /dev/null +++ b/electron/native/gpu-export-probe/CMakeLists.txt @@ -0,0 +1,37 @@ +cmake_minimum_required(VERSION 3.20) +project(gpu-export-probe LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +add_executable(gpu-export-probe + src/main.cpp +) + +set(NVIDIA_VIDEO_SDK_SAMPLES_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../.tmp/video-sdk-samples/Samples") +set(NVIDIA_NVENC_SDK_DIR "${NVIDIA_VIDEO_SDK_SAMPLES_DIR}/NvCodec") +if(EXISTS "${NVIDIA_NVENC_SDK_DIR}/NvEncoder/NvEncoderD3D11.cpp") + target_sources(gpu-export-probe PRIVATE + "${NVIDIA_NVENC_SDK_DIR}/NvEncoder/NvEncoder.cpp" + "${NVIDIA_NVENC_SDK_DIR}/NvEncoder/NvEncoderD3D11.cpp" + ) + target_include_directories(gpu-export-probe PRIVATE + "${NVIDIA_VIDEO_SDK_SAMPLES_DIR}" + "${NVIDIA_NVENC_SDK_DIR}" + ) + target_compile_definitions(gpu-export-probe PRIVATE RECORDLY_GPU_EXPORT_ENABLE_NVENC_SDK=1) +endif() + +target_compile_options(gpu-export-probe PRIVATE /EHsc /W4 /utf-8) + +target_link_libraries(gpu-export-probe PRIVATE + d3d11 + d3dcompiler + dxgi + mfplat + mfreadwrite + mf + mfuuid + ole32 + windowscodecs +) diff --git a/electron/native/gpu-export-probe/src/main.cpp b/electron/native/gpu-export-probe/src/main.cpp new file mode 100644 index 00000000..3026a362 --- /dev/null +++ b/electron/native/gpu-export-probe/src/main.cpp @@ -0,0 +1,3155 @@ +#define NOMINMAX +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef RECORDLY_GPU_EXPORT_ENABLE_NVENC_SDK +#include "NvEncoder/NvEncoderD3D11.h" +#endif + +#pragma comment(lib, "d3d11.lib") +#pragma comment(lib, "d3dcompiler.lib") +#pragma comment(lib, "dxgi.lib") +#pragma comment(lib, "mfplat.lib") +#pragma comment(lib, "mfreadwrite.lib") +#pragma comment(lib, "mf.lib") +#pragma comment(lib, "mfuuid.lib") +#pragma comment(lib, "windowscodecs.lib") + +using Microsoft::WRL::ComPtr; + +namespace { + +struct Options { + std::wstring inputPath; + std::wstring outputPath = L"gpu-export-probe.mp4"; + UINT width = 1920; + UINT height = 1080; + UINT fps = 30; + double seconds = 60.0; + UINT bitrate = 12'000'000; + bool shaderComposite = false; + float radius = 32.0f; + float shadow = 36.0f; + float padding = 0.0f; + LONG contentLeft = -1; + LONG contentTop = -1; + LONG contentWidth = 0; + LONG contentHeight = 0; + float backgroundR = 0.035f; + float backgroundG = 0.035f; + float backgroundB = 0.045f; + std::wstring backgroundImagePath; + std::wstring webcamInputPath; + LONG webcamLeft = -1; + LONG webcamTop = -1; + LONG webcamSize = 0; + float webcamRadius = 18.0f; + float webcamShadow = 0.0f; + bool webcamMirror = false; + double webcamTimeOffsetMs = 0.0; + std::wstring cursorTelemetryPath; + std::wstring cursorAtlasPath; + std::wstring cursorAtlasMetadataPath; + float cursorSize = 84.0f; + std::wstring zoomTelemetryPath; + bool preferHighPerformanceAdapter = false; + int adapterIndex = -1; + bool fastEncoderTuning = false; + UINT surfacePoolSize = 4; + bool nvencSdk = false; +}; + +struct ShaderConstants { + float outputWidth; + float outputHeight; + float radius; + float shadowSize; + float contentLeft; + float contentTop; + float contentRight; + float contentBottom; + float backgroundR; + float backgroundG; + float backgroundB; + float backgroundA; + float shadowR; + float shadowG; + float shadowB; + float shadowA; + float backgroundImageEnabled; + float backgroundImageWidth; + float backgroundImageHeight; + float webcamEnabled; + float webcamLeft; + float webcamTop; + float webcamRight; + float webcamBottom; + float webcamRadius; + float webcamShadowSize; + float webcamShadowA; + float webcamMirror; + float cursorEnabled; + float cursorX; + float cursorY; + float cursorSize; + float cursorAtlasEnabled; + float cursorAtlasLeft; + float cursorAtlasTop; + float cursorAtlasRight; + float cursorAtlasBottom; + float cursorAtlasAnchorX; + float cursorAtlasAnchorY; + float cursorAtlasAspect; + float cursorBounceScale; + float cursorPadding0; + float cursorPadding1; + float cursorPadding2; + float zoomEnabled; + float zoomScale; + float zoomX; + float zoomY; +}; + +struct CursorSample { + double timeMs = 0.0; + float cx = 0.0f; + float cy = 0.0f; + int cursorTypeIndex = 0; + float bounceScale = 1.0f; +}; + +struct CursorAtlasEntry { + float x = 0.0f; + float y = 0.0f; + float width = 1.0f; + float height = 1.0f; + float anchorX = 0.0f; + float anchorY = 0.0f; + float aspectRatio = 1.0f; + bool valid = false; +}; + +struct ZoomSample { + double timeMs = 0.0; + float scale = 1.0f; + float x = 0.0f; + float y = 0.0f; +}; + +struct Timer { + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + + double elapsedMs() const { + const auto now = std::chrono::steady_clock::now(); + return std::chrono::duration(now - start).count(); + } +}; + +std::wstring getArgValue(const std::vector& args, const std::wstring& key) { + for (size_t i = 0; i + 1 < args.size(); ++i) { + if (args[i] == key) { + return args[i + 1]; + } + } + return L""; +} + +UINT parseUIntArg(const std::vector& args, const std::wstring& key, UINT fallback) { + const auto value = getArgValue(args, key); + if (value.empty()) { + return fallback; + } + + try { + return static_cast(std::stoul(value)); + } catch (...) { + return fallback; + } +} + +LONG parseLongArg(const std::vector& args, const std::wstring& key, LONG fallback) { + const auto value = getArgValue(args, key); + if (value.empty()) { + return fallback; + } + + try { + return static_cast(std::stol(value)); + } catch (...) { + return fallback; + } +} + +float parseFloatArg(const std::vector& args, const std::wstring& key, float fallback) { + const auto value = getArgValue(args, key); + if (value.empty()) { + return fallback; + } + + try { + return std::stof(value); + } catch (...) { + return fallback; + } +} + +double parseDoubleArg(const std::vector& args, const std::wstring& key, double fallback) { + const auto value = getArgValue(args, key); + if (value.empty()) { + return fallback; + } + + try { + return std::stod(value); + } catch (...) { + return fallback; + } +} + +bool hasArg(const std::vector& args, const std::wstring& key) { + return std::find(args.begin(), args.end(), key) != args.end(); +} + +bool parseHexColor(const std::wstring& value, float& red, float& green, float& blue) { + std::wstring trimmed = value; + trimmed.erase( + std::remove_if(trimmed.begin(), trimmed.end(), [](wchar_t ch) { + return ch == L' ' || ch == L'\t' || ch == L'\r' || ch == L'\n'; + }), + trimmed.end()); + if (!trimmed.empty() && trimmed[0] == L'#') { + trimmed.erase(trimmed.begin()); + } + if (trimmed.size() != 6) { + return false; + } + + try { + const auto number = std::stoul(trimmed, nullptr, 16); + red = static_cast((number >> 16) & 0xff) / 255.0f; + green = static_cast((number >> 8) & 0xff) / 255.0f; + blue = static_cast(number & 0xff) / 255.0f; + return true; + } catch (...) { + return false; + } +} + +Options parseOptions(int argc, wchar_t** argv) { + std::vector args; + args.reserve(static_cast(argc)); + for (int i = 0; i < argc; ++i) { + args.emplace_back(argv[i]); + } + + Options options; + const auto input = getArgValue(args, L"--input"); + if (!input.empty()) { + options.inputPath = input; + } + const auto output = getArgValue(args, L"--output"); + if (!output.empty()) { + options.outputPath = output; + } + options.width = parseUIntArg(args, L"--width", options.width); + options.height = parseUIntArg(args, L"--height", options.height); + options.fps = parseUIntArg(args, L"--fps", options.fps); + options.seconds = parseDoubleArg(args, L"--seconds", options.seconds); + options.bitrate = parseUIntArg(args, L"--bitrate", options.bitrate); + options.shaderComposite = hasArg(args, L"--shader-composite"); + options.radius = parseFloatArg(args, L"--radius", options.radius); + options.shadow = parseFloatArg(args, L"--shadow", options.shadow); + options.padding = parseFloatArg(args, L"--padding", options.padding); + options.contentLeft = parseLongArg(args, L"--content-left", options.contentLeft); + options.contentTop = parseLongArg(args, L"--content-top", options.contentTop); + options.contentWidth = parseLongArg(args, L"--content-width", options.contentWidth); + options.contentHeight = parseLongArg(args, L"--content-height", options.contentHeight); + const auto backgroundColor = getArgValue(args, L"--background-color"); + if (!backgroundColor.empty()) { + parseHexColor( + backgroundColor, + options.backgroundR, + options.backgroundG, + options.backgroundB); + } + const auto backgroundImage = getArgValue(args, L"--background-image"); + if (!backgroundImage.empty()) { + options.backgroundImagePath = backgroundImage; + } + const auto webcamInput = getArgValue(args, L"--webcam-input"); + if (!webcamInput.empty()) { + options.webcamInputPath = webcamInput; + } + options.webcamLeft = parseLongArg(args, L"--webcam-left", options.webcamLeft); + options.webcamTop = parseLongArg(args, L"--webcam-top", options.webcamTop); + options.webcamSize = parseLongArg(args, L"--webcam-size", options.webcamSize); + options.webcamRadius = parseFloatArg(args, L"--webcam-radius", options.webcamRadius); + options.webcamShadow = parseFloatArg(args, L"--webcam-shadow", options.webcamShadow); + options.webcamMirror = hasArg(args, L"--webcam-mirror"); + options.webcamTimeOffsetMs = parseDoubleArg( + args, + L"--webcam-time-offset-ms", + options.webcamTimeOffsetMs); + const auto cursorTelemetry = getArgValue(args, L"--cursor-telemetry"); + if (!cursorTelemetry.empty()) { + options.cursorTelemetryPath = cursorTelemetry; + } + const auto cursorAtlas = getArgValue(args, L"--cursor-atlas"); + if (!cursorAtlas.empty()) { + options.cursorAtlasPath = cursorAtlas; + } + const auto cursorAtlasMetadata = getArgValue(args, L"--cursor-atlas-metadata"); + if (!cursorAtlasMetadata.empty()) { + options.cursorAtlasMetadataPath = cursorAtlasMetadata; + } + options.cursorSize = parseFloatArg(args, L"--cursor-size", options.cursorSize); + const auto zoomTelemetry = getArgValue(args, L"--zoom-telemetry"); + if (!zoomTelemetry.empty()) { + options.zoomTelemetryPath = zoomTelemetry; + } + options.preferHighPerformanceAdapter = hasArg(args, L"--prefer-high-performance-adapter"); + options.adapterIndex = static_cast(parseLongArg(args, L"--adapter-index", options.adapterIndex)); + options.fastEncoderTuning = hasArg(args, L"--fast-encoder-tuning"); + options.nvencSdk = hasArg(args, L"--nvenc-sdk"); + options.surfacePoolSize = parseUIntArg(args, L"--surface-pool-size", options.surfacePoolSize); + options.width = std::max(2, options.width & ~1U); + options.height = std::max(2, options.height & ~1U); + options.fps = std::max(1, options.fps); + options.seconds = std::max(0.001, options.seconds); + options.surfacePoolSize = std::min(32, std::max(4, options.surfacePoolSize)); + options.radius = std::max(0.0f, options.radius); + options.shadow = std::max(0.0f, options.shadow); + options.webcamSize = std::max(0, options.webcamSize & ~1L); + options.webcamRadius = std::max(0.0f, options.webcamRadius); + options.webcamShadow = std::max(0.0f, options.webcamShadow); + options.cursorSize = std::max(0.0f, options.cursorSize); + if (options.padding > 1.0f) { + options.padding /= 100.0f; + } + options.padding = std::min(0.45f, std::max(0.0f, options.padding)); + return options; +} + +std::string hrToHex(HRESULT hr) { + std::ostringstream stream; + stream << "0x" << std::hex << static_cast(hr); + return stream.str(); +} + +bool succeeded(HRESULT hr, const char* label) { + if (SUCCEEDED(hr)) { + return true; + } + std::cerr << "ERROR: " << label << " failed: " << hrToHex(hr) << std::endl; + return false; +} + +DWORD firstVideoStreamIndex() { + return static_cast(MF_SOURCE_READER_FIRST_VIDEO_STREAM); +} + +class GpuProbe { +public: + bool initialize(const Options& options) { + const Timer initializeTimer; + options_ = options; + loadCursorTelemetry(); + loadZoomTelemetry(); + + { + const Timer timer; + const HRESULT coInit = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + coInitialized_ = SUCCEEDED(coInit); + initCoInitializeMs_ = timer.elapsedMs(); + if (!coInitialized_ && coInit != RPC_E_CHANGED_MODE) { + return succeeded(coInit, "CoInitializeEx"); + } + } + + { + const Timer timer; + const HRESULT hr = MFStartup(MF_VERSION); + initMfStartupMs_ = timer.elapsedMs(); + if (!succeeded(hr, "MFStartup")) { + return false; + } + mfStarted_ = true; + } + + { + const Timer timer; + if (!createD3DDevice()) { + return false; + } + initD3DDeviceMs_ = timer.elapsedMs(); + } + + initSourceReaderMs_ = 0.0; + if (!options_.inputPath.empty()) { + const Timer timer; + if (!createSourceReader()) { + return false; + } + initSourceReaderMs_ = timer.elapsedMs(); + } + + initWebcamReaderMs_ = 0.0; + if (hasWebcamOverlay()) { + const Timer timer; + if (!createWebcamSourceReader()) { + return false; + } + initWebcamReaderMs_ = timer.elapsedMs(); + } + + { + const Timer timer; + if (!createVideoProcessor()) { + return false; + } + initVideoProcessorMs_ = timer.elapsedMs(); + } + + { + const Timer timer; + if (!createTextures()) { + return false; + } + initTexturesMs_ = timer.elapsedMs(); + } + + initShaderPipelineMs_ = 0.0; + if (options_.shaderComposite) { + const Timer timer; + if (!createShaderPipeline()) { + return false; + } + initShaderPipelineMs_ = timer.elapsedMs(); + } + + { + const Timer timer; + if (options_.nvencSdk ? !createNvencSdkEncoder() : !createSinkWriter()) { + return false; + } + initSinkWriterMs_ = timer.elapsedMs(); + } + + initializeMs_ = initializeTimer.elapsedMs(); + return true; + } + + bool run() { + if (!options_.inputPath.empty()) { + return runSourceVideo(); + } + + const UINT frameCount = + static_cast(std::ceil(static_cast(options_.fps) * options_.seconds)); + const Timer totalTimer; + double clearMs = 0; + double processMs = 0; + double writeMs = 0; + + for (UINT frameIndex = 0; frameIndex < frameCount; ++frameIndex) { + const Timer clearTimer; + renderSyntheticFrame(frameIndex); + clearMs += clearTimer.elapsedMs(); + + const Timer processTimer; + if (!convertBgraToNv12(frameIndex)) { + return false; + } + processMs += processTimer.elapsedMs(); + + const Timer writeTimer; + if (!writeFrame(frameIndex)) { + return false; + } + writeMs += writeTimer.elapsedMs(); + emitProgress(frameIndex + 1, frameCount); + } + emitProgress(frameCount, frameCount, true); + + const Timer finalizeTimer; + const bool finalized = options_.nvencSdk ? finalizeNvencSdk() : SUCCEEDED(sinkWriter_->Finalize()); + const double finalizeMs = finalizeTimer.elapsedMs(); + if (!finalized) { + if (!options_.nvencSdk) { + std::cerr << "ERROR: IMFSinkWriter::Finalize failed" << std::endl; + } + return false; + } + + const double totalMs = totalTimer.elapsedMs(); + const double realtime = (options_.seconds * 1000.0) / totalMs; + std::cout + << "{" + << "\"success\":true," + << "\"width\":" << options_.width << "," + << "\"height\":" << options_.height << "," + << "\"fps\":" << options_.fps << "," + << "\"surfacePoolSize\":" << options_.surfacePoolSize << "," + << "\"adapterIndex\":" << selectedAdapterIndex_ << "," + << "\"adapterVendorId\":" << selectedAdapterVendorId_ << "," + << "\"adapterDeviceId\":" << selectedAdapterDeviceId_ << "," + << "\"adapterDedicatedVideoMemoryMB\":" << selectedAdapterDedicatedVideoMemoryMB_ << "," + << "\"seconds\":" << options_.seconds << "," + << "\"frames\":" << frameCount << "," + << "\"initializeMs\":" << initializeMs_ << "," + << "\"initCoInitializeMs\":" << initCoInitializeMs_ << "," + << "\"initMfStartupMs\":" << initMfStartupMs_ << "," + << "\"initD3DDeviceMs\":" << initD3DDeviceMs_ << "," + << "\"initSourceReaderMs\":" << initSourceReaderMs_ << "," + << "\"initWebcamReaderMs\":" << initWebcamReaderMs_ << "," + << "\"initVideoProcessorMs\":" << initVideoProcessorMs_ << "," + << "\"initTexturesMs\":" << initTexturesMs_ << "," + << "\"initShaderPipelineMs\":" << initShaderPipelineMs_ << "," + << "\"initSinkWriterMs\":" << initSinkWriterMs_ << "," + << "\"encoderBackend\":\"" << (options_.nvencSdk ? "nvenc-sdk-d3d11" : "media-foundation") << "\"," + << "\"encoderTuningApplied\":" << (encoderTuningApplied_ ? "true" : "false") << "," + << "\"nvencOutputBytes\":" << nvencOutputBytes_ << "," + << "\"totalMs\":" << totalMs << "," + << "\"clearMs\":" << clearMs << "," + << "\"videoProcessMs\":" << processMs << "," + << "\"writeSampleMs\":" << writeMs << "," + << "\"finalizeMs\":" << finalizeMs << "," + << "\"realtimeMultiplier\":" << realtime + << "}" << std::endl; + return true; + } + + void emitProgress(UINT currentFrame, UINT totalFrames, bool force = false) { + if (totalFrames == 0) { + return; + } + + const UINT cadence = std::max(1, static_cast(options_.fps)); + if (!force && currentFrame < totalFrames && (currentFrame % cadence) != 0) { + return; + } + + const double percentage = + std::min(100.0, (static_cast(currentFrame) / totalFrames) * 100.0); + std::cerr + << "PROGRESS {" + << "\"currentFrame\":" << currentFrame << "," + << "\"totalFrames\":" << totalFrames << "," + << "\"percentage\":" << percentage + << "}" << std::endl; + } + + ~GpuProbe() { + if (nvencOutputFile_) { + std::fclose(nvencOutputFile_); + nvencOutputFile_ = nullptr; + } +#ifdef RECORDLY_GPU_EXPORT_ENABLE_NVENC_SDK + nvencEncoder_.reset(); +#endif + sinkWriter_.Reset(); + sourceReader_.Reset(); + webcamReader_.Reset(); + pendingWebcamSample_.Reset(); + nv12Textures_.clear(); + nv12OutputViews_.clear(); + bgraNv12OutputViews_.clear(); + compositorConstants_.Reset(); + samplerState_.Reset(); + pixelShader_.Reset(); + vertexShader_.Reset(); + backgroundShaderResourceView_.Reset(); + webcamShaderResourceView_.Reset(); + webcamOutputView_.Reset(); + webcamTexture_.Reset(); + contentShaderResourceView_.Reset(); + contentOutputView_.Reset(); + contentTexture_.Reset(); + bgraInputView_.Reset(); + bgraRenderTargetView_.Reset(); + bgraTexture_.Reset(); + bgraVideoProcessor_.Reset(); + bgraVideoProcessorEnumerator_.Reset(); + webcamVideoProcessor_.Reset(); + webcamVideoProcessorEnumerator_.Reset(); + videoProcessor_.Reset(); + videoProcessorEnumerator_.Reset(); + videoContext_.Reset(); + videoDevice_.Reset(); + deviceContext_.Reset(); + device_.Reset(); + deviceManager_.Reset(); + if (mfStarted_) { + MFShutdown(); + } + if (coInitialized_) { + CoUninitialize(); + } + } + +private: + void captureSelectedAdapterInfo(IDXGIAdapter1* adapter) { + if (!adapter) { + return; + } + + DXGI_ADAPTER_DESC1 desc = {}; + if (FAILED(adapter->GetDesc1(&desc))) { + return; + } + selectedAdapterVendorId_ = desc.VendorId; + selectedAdapterDeviceId_ = desc.DeviceId; + selectedAdapterDedicatedVideoMemoryMB_ = + static_cast(desc.DedicatedVideoMemory / (1024 * 1024)); + } + + ComPtr selectHighPerformanceAdapter() { + ComPtr factory; + HRESULT hr = CreateDXGIFactory1(IID_PPV_ARGS(&factory)); + if (FAILED(hr)) { + return nullptr; + } + + ComPtr bestAdapter; + SIZE_T bestDedicatedMemory = 0; + for (UINT index = 0;; ++index) { + ComPtr adapter; + hr = factory->EnumAdapters1(index, &adapter); + if (hr == DXGI_ERROR_NOT_FOUND) { + break; + } + if (FAILED(hr)) { + continue; + } + + DXGI_ADAPTER_DESC1 desc = {}; + if (FAILED(adapter->GetDesc1(&desc))) { + continue; + } + if ((desc.Flags & DXGI_ADAPTER_FLAG_SOFTWARE) != 0) { + continue; + } + if (!bestAdapter || desc.DedicatedVideoMemory > bestDedicatedMemory) { + bestDedicatedMemory = desc.DedicatedVideoMemory; + bestAdapter = adapter; + } + } + + return bestAdapter; + } + + ComPtr selectAdapterByIndex(UINT requestedIndex) { + ComPtr factory; + HRESULT hr = CreateDXGIFactory1(IID_PPV_ARGS(&factory)); + if (FAILED(hr)) { + return nullptr; + } + + ComPtr adapter; + hr = factory->EnumAdapters1(requestedIndex, &adapter); + if (FAILED(hr)) { + return nullptr; + } + return adapter; + } + + bool createD3DDevice() { + const D3D_FEATURE_LEVEL levels[] = { + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL selectedLevel = D3D_FEATURE_LEVEL_11_0; + UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT; + ComPtr preferredAdapter; + if (options_.adapterIndex >= 0) { + preferredAdapter = selectAdapterByIndex(static_cast(options_.adapterIndex)); + if (!preferredAdapter) { + std::cerr << "ERROR: adapter index " << options_.adapterIndex << " was not found" << std::endl; + return false; + } + selectedAdapterIndex_ = options_.adapterIndex; + } else if (options_.preferHighPerformanceAdapter) { + preferredAdapter = selectHighPerformanceAdapter(); + } + + HRESULT hr = D3D11CreateDevice( + preferredAdapter.Get(), + preferredAdapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE, + nullptr, + flags, + levels, + ARRAYSIZE(levels), + D3D11_SDK_VERSION, + &device_, + &selectedLevel, + &deviceContext_); + if (!succeeded(hr, "D3D11CreateDevice")) { + return false; + } + + ComPtr dxgiDevice; + if (SUCCEEDED(device_.As(&dxgiDevice))) { + ComPtr adapter; + if (SUCCEEDED(dxgiDevice->GetAdapter(&adapter))) { + ComPtr adapter1; + if (SUCCEEDED(adapter.As(&adapter1))) { + captureSelectedAdapterInfo(adapter1.Get()); + } + } + } + + hr = device_.As(&videoDevice_); + if (!succeeded(hr, "Query ID3D11VideoDevice")) { + return false; + } + hr = deviceContext_.As(&videoContext_); + if (!succeeded(hr, "Query ID3D11VideoContext")) { + return false; + } + + UINT resetToken = 0; + hr = MFCreateDXGIDeviceManager(&resetToken, &deviceManager_); + if (!succeeded(hr, "MFCreateDXGIDeviceManager")) { + return false; + } + hr = deviceManager_->ResetDevice(device_.Get(), resetToken); + if (!succeeded(hr, "IMFDXGIDeviceManager::ResetDevice")) { + return false; + } + + return true; + } + + bool createVideoProcessor() { + D3D11_VIDEO_PROCESSOR_CONTENT_DESC desc = {}; + desc.InputFrameFormat = D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE; + desc.InputFrameRate.Numerator = options_.fps; + desc.InputFrameRate.Denominator = 1; + desc.InputWidth = sourceWidth_; + desc.InputHeight = sourceHeight_; + desc.OutputFrameRate.Numerator = options_.fps; + desc.OutputFrameRate.Denominator = 1; + desc.OutputWidth = options_.width; + desc.OutputHeight = options_.height; + desc.Usage = D3D11_VIDEO_USAGE_PLAYBACK_NORMAL; + + HRESULT hr = videoDevice_->CreateVideoProcessorEnumerator(&desc, &videoProcessorEnumerator_); + if (!succeeded(hr, "CreateVideoProcessorEnumerator")) { + return false; + } + hr = videoDevice_->CreateVideoProcessor(videoProcessorEnumerator_.Get(), 0, &videoProcessor_); + if (!succeeded(hr, "CreateVideoProcessor")) { + return false; + } + + RECT rect = { + 0, + 0, + static_cast(sourceWidth_), + static_cast(sourceHeight_), + }; + RECT outputRect = { + 0, + 0, + static_cast(options_.width), + static_cast(options_.height), + }; + RECT contentRect = getContentRect(); + videoContext_->VideoProcessorSetStreamSourceRect(videoProcessor_.Get(), 0, TRUE, &rect); + videoContext_->VideoProcessorSetStreamDestRect(videoProcessor_.Get(), 0, TRUE, &contentRect); + videoContext_->VideoProcessorSetOutputTargetRect(videoProcessor_.Get(), TRUE, &outputRect); + videoContext_->VideoProcessorSetStreamAutoProcessingMode(videoProcessor_.Get(), 0, FALSE); + D3D11_VIDEO_COLOR background = {}; + background.RGBA.A = 1.0f; + background.RGBA.R = 0.04f; + background.RGBA.G = 0.04f; + background.RGBA.B = 0.05f; + videoContext_->VideoProcessorSetOutputBackgroundColor( + videoProcessor_.Get(), + FALSE, + &background); + + D3D11_VIDEO_PROCESSOR_CONTENT_DESC bgraDesc = {}; + bgraDesc.InputFrameFormat = D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE; + bgraDesc.InputFrameRate.Numerator = options_.fps; + bgraDesc.InputFrameRate.Denominator = 1; + bgraDesc.InputWidth = options_.width; + bgraDesc.InputHeight = options_.height; + bgraDesc.OutputFrameRate.Numerator = options_.fps; + bgraDesc.OutputFrameRate.Denominator = 1; + bgraDesc.OutputWidth = options_.width; + bgraDesc.OutputHeight = options_.height; + bgraDesc.Usage = D3D11_VIDEO_USAGE_PLAYBACK_NORMAL; + + hr = videoDevice_->CreateVideoProcessorEnumerator( + &bgraDesc, + &bgraVideoProcessorEnumerator_); + if (!succeeded(hr, "Create BGRA video processor enumerator")) { + return false; + } + hr = videoDevice_->CreateVideoProcessor( + bgraVideoProcessorEnumerator_.Get(), + 0, + &bgraVideoProcessor_); + if (!succeeded(hr, "Create BGRA video processor")) { + return false; + } + + RECT bgraRect = { + 0, + 0, + static_cast(options_.width), + static_cast(options_.height), + }; + videoContext_->VideoProcessorSetStreamSourceRect( + bgraVideoProcessor_.Get(), + 0, + TRUE, + &bgraRect); + videoContext_->VideoProcessorSetStreamDestRect( + bgraVideoProcessor_.Get(), + 0, + TRUE, + &bgraRect); + videoContext_->VideoProcessorSetOutputTargetRect( + bgraVideoProcessor_.Get(), + TRUE, + &bgraRect); + videoContext_->VideoProcessorSetStreamAutoProcessingMode( + bgraVideoProcessor_.Get(), + 0, + FALSE); + + if (hasWebcamOverlay()) { + D3D11_VIDEO_PROCESSOR_CONTENT_DESC webcamDesc = {}; + webcamDesc.InputFrameFormat = D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE; + webcamDesc.InputFrameRate.Numerator = options_.fps; + webcamDesc.InputFrameRate.Denominator = 1; + webcamDesc.InputWidth = webcamWidth_; + webcamDesc.InputHeight = webcamHeight_; + webcamDesc.OutputFrameRate.Numerator = options_.fps; + webcamDesc.OutputFrameRate.Denominator = 1; + webcamDesc.OutputWidth = options_.width; + webcamDesc.OutputHeight = options_.height; + webcamDesc.Usage = D3D11_VIDEO_USAGE_PLAYBACK_NORMAL; + + hr = videoDevice_->CreateVideoProcessorEnumerator( + &webcamDesc, + &webcamVideoProcessorEnumerator_); + if (!succeeded(hr, "Create webcam video processor enumerator")) { + return false; + } + hr = videoDevice_->CreateVideoProcessor( + webcamVideoProcessorEnumerator_.Get(), + 0, + &webcamVideoProcessor_); + if (!succeeded(hr, "Create webcam video processor")) { + return false; + } + + RECT webcamSourceRect = { + 0, + 0, + static_cast(webcamWidth_), + static_cast(webcamHeight_), + }; + RECT webcamOutputRect = { + 0, + 0, + static_cast(options_.width), + static_cast(options_.height), + }; + RECT webcamDestRect = getWebcamRect(); + videoContext_->VideoProcessorSetStreamSourceRect( + webcamVideoProcessor_.Get(), + 0, + TRUE, + &webcamSourceRect); + videoContext_->VideoProcessorSetStreamDestRect( + webcamVideoProcessor_.Get(), + 0, + TRUE, + &webcamDestRect); + videoContext_->VideoProcessorSetOutputTargetRect( + webcamVideoProcessor_.Get(), + TRUE, + &webcamOutputRect); + videoContext_->VideoProcessorSetStreamAutoProcessingMode( + webcamVideoProcessor_.Get(), + 0, + FALSE); + } + return true; + } + + bool createTextures() { + HRESULT hr = S_OK; + if (options_.inputPath.empty() || options_.shaderComposite) { + D3D11_TEXTURE2D_DESC bgraDesc = {}; + bgraDesc.Width = options_.width; + bgraDesc.Height = options_.height; + bgraDesc.MipLevels = 1; + bgraDesc.ArraySize = 1; + bgraDesc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + bgraDesc.SampleDesc.Count = 1; + bgraDesc.Usage = D3D11_USAGE_DEFAULT; + bgraDesc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; + + hr = device_->CreateTexture2D(&bgraDesc, nullptr, &bgraTexture_); + if (!succeeded(hr, "Create BGRA texture")) { + return false; + } + hr = device_->CreateRenderTargetView(bgraTexture_.Get(), nullptr, &bgraRenderTargetView_); + if (!succeeded(hr, "Create BGRA render target view")) { + return false; + } + + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC inputViewDesc = {}; + inputViewDesc.FourCC = 0; + inputViewDesc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D; + inputViewDesc.Texture2D.MipSlice = 0; + inputViewDesc.Texture2D.ArraySlice = 0; + hr = videoDevice_->CreateVideoProcessorInputView( + bgraTexture_.Get(), + bgraVideoProcessorEnumerator_.Get(), + &inputViewDesc, + &bgraInputView_); + if (!succeeded(hr, "Create video processor input view")) { + return false; + } + } + + if (options_.shaderComposite) { + D3D11_TEXTURE2D_DESC contentDesc = {}; + contentDesc.Width = options_.width; + contentDesc.Height = options_.height; + contentDesc.MipLevels = 1; + contentDesc.ArraySize = 1; + contentDesc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + contentDesc.SampleDesc.Count = 1; + contentDesc.Usage = D3D11_USAGE_DEFAULT; + contentDesc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; + + hr = device_->CreateTexture2D(&contentDesc, nullptr, &contentTexture_); + if (!succeeded(hr, "Create compositor content texture")) { + return false; + } + + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC contentOutputDesc = {}; + contentOutputDesc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D; + contentOutputDesc.Texture2D.MipSlice = 0; + hr = videoDevice_->CreateVideoProcessorOutputView( + contentTexture_.Get(), + videoProcessorEnumerator_.Get(), + &contentOutputDesc, + &contentOutputView_); + if (!succeeded(hr, "Create compositor content output view")) { + return false; + } + + D3D11_SHADER_RESOURCE_VIEW_DESC srvDesc = {}; + srvDesc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + srvDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D; + srvDesc.Texture2D.MipLevels = 1; + hr = device_->CreateShaderResourceView( + contentTexture_.Get(), + &srvDesc, + &contentShaderResourceView_); + if (!succeeded(hr, "Create compositor content shader resource view")) { + return false; + } + + if (hasWebcamOverlay()) { + D3D11_TEXTURE2D_DESC webcamDesc = contentDesc; + hr = device_->CreateTexture2D(&webcamDesc, nullptr, &webcamTexture_); + if (!succeeded(hr, "Create compositor webcam texture")) { + return false; + } + + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC webcamOutputDesc = {}; + webcamOutputDesc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D; + webcamOutputDesc.Texture2D.MipSlice = 0; + hr = videoDevice_->CreateVideoProcessorOutputView( + webcamTexture_.Get(), + webcamVideoProcessorEnumerator_.Get(), + &webcamOutputDesc, + &webcamOutputView_); + if (!succeeded(hr, "Create compositor webcam output view")) { + return false; + } + + hr = device_->CreateShaderResourceView( + webcamTexture_.Get(), + &srvDesc, + &webcamShaderResourceView_); + if (!succeeded(hr, "Create compositor webcam shader resource view")) { + return false; + } + } + } + + D3D11_TEXTURE2D_DESC nv12Desc = {}; + nv12Desc.Width = options_.width; + nv12Desc.Height = options_.height; + nv12Desc.MipLevels = 1; + nv12Desc.ArraySize = 1; + nv12Desc.Format = DXGI_FORMAT_NV12; + nv12Desc.SampleDesc.Count = 1; + nv12Desc.Usage = D3D11_USAGE_DEFAULT; + nv12Desc.BindFlags = D3D11_BIND_RENDER_TARGET; + nv12Desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; + + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC outputViewDesc = {}; + outputViewDesc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D; + outputViewDesc.Texture2D.MipSlice = 0; + + const size_t surfaceCount = static_cast(options_.surfacePoolSize); + nv12Textures_.reserve(surfaceCount); + nv12OutputViews_.reserve(surfaceCount); + bgraNv12OutputViews_.reserve(surfaceCount); + + for (size_t index = 0; index < surfaceCount; ++index) { + ComPtr texture; + hr = device_->CreateTexture2D(&nv12Desc, nullptr, &texture); + if (!succeeded(hr, "Create NV12 texture")) { + return false; + } + + ComPtr outputView; + hr = videoDevice_->CreateVideoProcessorOutputView( + texture.Get(), + videoProcessorEnumerator_.Get(), + &outputViewDesc, + &outputView); + if (!succeeded(hr, "Create video processor output view")) { + return false; + } + + ComPtr bgraOutputView; + hr = videoDevice_->CreateVideoProcessorOutputView( + texture.Get(), + bgraVideoProcessorEnumerator_.Get(), + &outputViewDesc, + &bgraOutputView); + if (!succeeded(hr, "Create BGRA video processor output view")) { + return false; + } + + nv12Textures_.push_back(texture); + nv12OutputViews_.push_back(outputView); + bgraNv12OutputViews_.push_back(bgraOutputView); + } + + return true; + } + + bool compileShader( + const char* source, + const char* entryPoint, + const char* target, + ID3DBlob** bytecode) { + ComPtr errors; + const HRESULT hr = D3DCompile( + source, + std::strlen(source), + nullptr, + nullptr, + nullptr, + entryPoint, + target, + D3DCOMPILE_ENABLE_STRICTNESS, + 0, + bytecode, + &errors); + if (FAILED(hr)) { + if (errors) { + std::cerr + << "ERROR: D3DCompile " + << target + << " failed: " + << static_cast(errors->GetBufferPointer()) + << std::endl; + } + return succeeded(hr, "D3DCompile"); + } + return true; + } + + bool createBgraShaderResource( + UINT width, + UINT height, + const std::uint8_t* pixels, + UINT stride, + ComPtr& shaderResourceView) { + D3D11_TEXTURE2D_DESC desc = {}; + desc.Width = width; + desc.Height = height; + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + + D3D11_SUBRESOURCE_DATA data = {}; + data.pSysMem = pixels; + data.SysMemPitch = stride; + + ComPtr texture; + HRESULT hr = device_->CreateTexture2D(&desc, &data, &texture); + if (!succeeded(hr, "Create BGRA shader texture")) { + return false; + } + + D3D11_SHADER_RESOURCE_VIEW_DESC srvDesc = {}; + srvDesc.Format = desc.Format; + srvDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D; + srvDesc.Texture2D.MipLevels = 1; + hr = device_->CreateShaderResourceView( + texture.Get(), + &srvDesc, + &shaderResourceView); + return succeeded(hr, "Create BGRA shader resource view"); + } + + bool loadWicBgraShaderResource( + const std::wstring& imagePath, + ComPtr& shaderResourceView, + UINT& width, + UINT& height, + const char* label) { + ComPtr factory; + HRESULT hr = CoCreateInstance( + CLSID_WICImagingFactory, + nullptr, + CLSCTX_INPROC_SERVER, + IID_PPV_ARGS(&factory)); + if (!succeeded(hr, "Create WIC imaging factory")) { + return false; + } + + ComPtr decoder; + hr = factory->CreateDecoderFromFilename( + imagePath.c_str(), + nullptr, + GENERIC_READ, + WICDecodeMetadataCacheOnLoad, + &decoder); + if (!succeeded(hr, label)) { + return false; + } + + ComPtr frame; + hr = decoder->GetFrame(0, &frame); + if (!succeeded(hr, "Get WIC frame")) { + return false; + } + + ComPtr converter; + hr = factory->CreateFormatConverter(&converter); + if (!succeeded(hr, "Create WIC format converter")) { + return false; + } + hr = converter->Initialize( + frame.Get(), + GUID_WICPixelFormat32bppBGRA, + WICBitmapDitherTypeNone, + nullptr, + 0.0, + WICBitmapPaletteTypeCustom); + if (!succeeded(hr, "Initialize WIC format converter")) { + return false; + } + + hr = converter->GetSize(&width, &height); + if (!succeeded(hr, "Get WIC image size")) { + return false; + } + if (width == 0 || height == 0) { + std::cerr << "ERROR: WIC image has an invalid size." << std::endl; + return false; + } + + const UINT stride = width * 4; + std::vector pixels(static_cast(stride) * height); + hr = converter->CopyPixels( + nullptr, + stride, + static_cast(pixels.size()), + pixels.data()); + if (!succeeded(hr, "Copy WIC image pixels")) { + return false; + } + + return createBgraShaderResource( + width, + height, + pixels.data(), + stride, + shaderResourceView); + } + + bool createSolidBackgroundTexture() { + const std::uint8_t pixel[] = { + static_cast(std::round(options_.backgroundB * 255.0f)), + static_cast(std::round(options_.backgroundG * 255.0f)), + static_cast(std::round(options_.backgroundR * 255.0f)), + 255, + }; + backgroundImageWidth_ = 1; + backgroundImageHeight_ = 1; + hasBackgroundImage_ = false; + return createBgraShaderResource( + 1, + 1, + pixel, + 4, + backgroundShaderResourceView_); + } + + bool createBackgroundTexture() { + if (options_.backgroundImagePath.empty()) { + return createSolidBackgroundTexture(); + } + + if (!loadWicBgraShaderResource( + options_.backgroundImagePath, + backgroundShaderResourceView_, + backgroundImageWidth_, + backgroundImageHeight_, + "Create WIC background decoder")) { + return false; + } + + hasBackgroundImage_ = true; + return true; + } + + bool loadCursorAtlasMetadata() { + for (auto& entry : cursorAtlasEntries_) { + entry = CursorAtlasEntry{}; + } + if (options_.cursorAtlasMetadataPath.empty()) { + return false; + } + + FILE* file = nullptr; + if (_wfopen_s(&file, options_.cursorAtlasMetadataPath.c_str(), L"rb") != 0 || !file) { + std::cerr << "[gpu-export] Unable to open cursor atlas metadata file" << std::endl; + return false; + } + + char line[256]; + bool sawEntry = false; + while (std::fgets(line, sizeof(line), file)) { + int index = 0; + CursorAtlasEntry entry; + if (sscanf_s( + line, + "%d,%f,%f,%f,%f,%f,%f,%f", + &index, + &entry.x, + &entry.y, + &entry.width, + &entry.height, + &entry.anchorX, + &entry.anchorY, + &entry.aspectRatio) != 8) { + continue; + } + if ( + index < 0 || + index >= static_cast(cursorAtlasEntries_.size()) || + !std::isfinite(entry.x) || + !std::isfinite(entry.y) || + !std::isfinite(entry.width) || + !std::isfinite(entry.height) || + !std::isfinite(entry.anchorX) || + !std::isfinite(entry.anchorY) || + !std::isfinite(entry.aspectRatio) || + entry.width <= 0.0f || + entry.height <= 0.0f) { + continue; + } + entry.valid = true; + cursorAtlasEntries_[static_cast(index)] = entry; + sawEntry = true; + } + std::fclose(file); + return sawEntry; + } + + bool createCursorAtlasTexture() { + if (options_.cursorAtlasPath.empty() || options_.cursorAtlasMetadataPath.empty()) { + hasCursorAtlas_ = false; + return true; + } + + if (!loadCursorAtlasMetadata()) { + hasCursorAtlas_ = false; + return true; + } + + if (!loadWicBgraShaderResource( + options_.cursorAtlasPath, + cursorAtlasShaderResourceView_, + cursorAtlasWidth_, + cursorAtlasHeight_, + "Create WIC cursor atlas decoder")) { + hasCursorAtlas_ = false; + return true; + } + + hasCursorAtlas_ = true; + return true; + } + + bool createShaderPipeline() { + static const char* vertexShaderSource = R"( +struct VSOut { + float4 position : SV_POSITION; + float2 uv : TEXCOORD0; +}; + +VSOut main(uint vertexId : SV_VertexID) { + float2 positions[3] = { + float2(-1.0, -1.0), + float2(-1.0, 3.0), + float2( 3.0, -1.0) + }; + + VSOut output; + float2 position = positions[vertexId]; + output.position = float4(position, 0.0, 1.0); + output.uv = float2((position.x + 1.0) * 0.5, 1.0 - ((position.y + 1.0) * 0.5)); + return output; +} +)"; + + static const char* pixelShaderSource = R"( +cbuffer CompositorConstants : register(b0) { + float outputWidth; + float outputHeight; + float radius; + float shadowSize; + float contentLeft; + float contentTop; + float contentRight; + float contentBottom; + float backgroundR; + float backgroundG; + float backgroundB; + float backgroundA; + float shadowR; + float shadowG; + float shadowB; + float shadowA; + float backgroundImageEnabled; + float backgroundImageWidth; + float backgroundImageHeight; + float webcamEnabled; + float webcamLeft; + float webcamTop; + float webcamRight; + float webcamBottom; + float webcamRadius; + float webcamShadowSize; + float webcamShadowA; + float webcamMirror; + float cursorEnabled; + float cursorX; + float cursorY; + float cursorSize; + float cursorAtlasEnabled; + float cursorAtlasLeft; + float cursorAtlasTop; + float cursorAtlasRight; + float cursorAtlasBottom; + float cursorAtlasAnchorX; + float cursorAtlasAnchorY; + float cursorAtlasAspect; + float cursorBounceScale; + float cursorPadding0; + float cursorPadding1; + float cursorPadding2; + float zoomEnabled; + float zoomScale; + float zoomX; + float zoomY; +}; + +Texture2D contentTexture : register(t0); +Texture2D backgroundTexture : register(t1); +Texture2D webcamTexture : register(t2); +Texture2D cursorAtlasTexture : register(t3); +SamplerState linearSampler : register(s0); + +struct PSIn { + float4 position : SV_POSITION; + float2 uv : TEXCOORD0; +}; + +float roundedBoxDistance(float2 p, float2 halfSize, float cornerRadius) { + float2 q = abs(p) - halfSize + cornerRadius; + return length(max(q, 0.0)) + min(max(q.x, q.y), 0.0) - cornerRadius; +} + +float cross2(float2 a, float2 b) { + return a.x * b.y - a.y * b.x; +} + +float insideTriangle(float2 p, float2 a, float2 b, float2 c) { + float ab = cross2(b - a, p - a); + float bc = cross2(c - b, p - b); + float ca = cross2(a - c, p - c); + return (ab >= 0.0 && bc >= 0.0 && ca >= 0.0) || + (ab <= 0.0 && bc <= 0.0 && ca <= 0.0) ? 1.0 : 0.0; +} + +float cursorArrowMask(float2 p, float scale) { + float2 a = float2(0.0, 0.0) * scale; + float2 b = float2(0.0, 58.0) * scale; + float2 c = float2(15.0, 44.0) * scale; + float2 d = float2(25.0, 66.0) * scale; + float2 e = float2(37.0, 61.0) * scale; + float2 f = float2(27.0, 40.0) * scale; + float2 g = float2(45.0, 40.0) * scale; + + return max( + insideTriangle(p, a, b, c), + max( + insideTriangle(p, a, c, g), + max( + insideTriangle(p, c, d, e), + insideTriangle(p, c, e, f) + ) + ) + ); +} + +float sampleCursorAtlasAlpha(float2 cursorLocal, float cursorWidth, float cursorHeight) { + float2 cursorUv = float2( + cursorLocal.x / cursorWidth + cursorAtlasAnchorX, + cursorLocal.y / cursorHeight + cursorAtlasAnchorY + ); + if ( + cursorUv.x < 0.0 || cursorUv.x > 1.0 || + cursorUv.y < 0.0 || cursorUv.y > 1.0 + ) { + return 0.0; + } + + float2 atlasMin = float2(cursorAtlasLeft, cursorAtlasTop); + float2 atlasMax = float2(cursorAtlasRight, cursorAtlasBottom); + return cursorAtlasTexture.Sample(linearSampler, lerp(atlasMin, atlasMax, cursorUv)).a; +} + +float sampleCursorAtlasShadow(float2 cursorLocal, float cursorWidth, float cursorHeight) { + float2 shadowLocal = cursorLocal - float2(0.0, 2.0); + float alpha = sampleCursorAtlasAlpha(shadowLocal, cursorWidth, cursorHeight) * 0.20; + alpha += sampleCursorAtlasAlpha(shadowLocal - float2(1.5, 0.0), cursorWidth, cursorHeight) * 0.06; + alpha += sampleCursorAtlasAlpha(shadowLocal + float2(1.5, 0.0), cursorWidth, cursorHeight) * 0.06; + alpha += sampleCursorAtlasAlpha(shadowLocal - float2(0.0, 1.5), cursorWidth, cursorHeight) * 0.04; + alpha += sampleCursorAtlasAlpha(shadowLocal + float2(0.0, 1.5), cursorWidth, cursorHeight) * 0.04; + return saturate(alpha); +} + +float4 main(PSIn input) : SV_Target { + float2 outputSize = float2(outputWidth, outputHeight); + float2 pixel = input.uv * outputSize; + float safeZoomScale = max(zoomScale, 0.01); + float2 zoomOffset = float2(zoomX, zoomY); + float2 contentPixel = zoomEnabled > 0.5 + ? (pixel - zoomOffset) / safeZoomScale + : pixel; + float2 rectMin = float2(contentLeft, contentTop); + float2 rectMax = float2(contentRight, contentBottom); + float2 halfSize = max((rectMax - rectMin) * 0.5, float2(1.0, 1.0)); + float2 center = (rectMin + rectMax) * 0.5; + float distanceToRect = roundedBoxDistance(contentPixel - center, halfSize, radius); + + float contentAlpha = 1.0 - smoothstep(-0.75, 0.75, distanceToRect); + float outsideAlpha = smoothstep(-0.75, 0.75, distanceToRect); + float shadowAlpha = + (1.0 - smoothstep(0.0, max(shadowSize, 1.0), max(distanceToRect, 0.0))) * + outsideAlpha * + shadowA; + + float4 background = float4(backgroundR, backgroundG, backgroundB, backgroundA); + if (backgroundImageEnabled > 0.5) { + float2 backgroundUv = input.uv; + float outputAspect = outputWidth / outputHeight; + float backgroundAspect = backgroundImageWidth / backgroundImageHeight; + if (backgroundAspect > outputAspect) { + backgroundUv.x = 0.5 + ((backgroundUv.x - 0.5) * (outputAspect / backgroundAspect)); + } else { + backgroundUv.y = 0.5 + ((backgroundUv.y - 0.5) * (backgroundAspect / outputAspect)); + } + background = backgroundTexture.Sample(linearSampler, saturate(backgroundUv)); + } + float4 shadow = float4(shadowR, shadowG, shadowB, shadowAlpha); + float4 content = contentTexture.Sample(linearSampler, saturate(contentPixel / outputSize)); + + float3 withShadow = lerp(background.rgb, shadow.rgb, saturate(shadow.a)); + float3 rgb = lerp(withShadow, content.rgb, saturate(contentAlpha)); + if (webcamEnabled > 0.5) { + float2 webcamMin = float2(webcamLeft, webcamTop); + float2 webcamMax = float2(webcamRight, webcamBottom); + float webcamInfluence = max(webcamShadowSize, 1.0) + 2.0; + bool nearWebcam = + pixel.x >= webcamMin.x - webcamInfluence && + pixel.x <= webcamMax.x + webcamInfluence && + pixel.y >= webcamMin.y - webcamInfluence && + pixel.y <= webcamMax.y + webcamInfluence; + if (nearWebcam) { + float2 webcamHalfSize = max((webcamMax - webcamMin) * 0.5, float2(1.0, 1.0)); + float2 webcamCenter = (webcamMin + webcamMax) * 0.5; + float webcamDistance = + roundedBoxDistance(pixel - webcamCenter, webcamHalfSize, webcamRadius); + float webcamAlpha = 1.0 - smoothstep(-0.75, 0.75, webcamDistance); + float webcamOutsideAlpha = smoothstep(-0.75, 0.75, webcamDistance); + float webcamShadowAlpha = + (1.0 - smoothstep(0.0, max(webcamShadowSize, 1.0), max(webcamDistance, 0.0))) * + webcamOutsideAlpha * + webcamShadowA; + rgb = lerp(rgb, shadow.rgb, saturate(webcamShadowAlpha)); + + if (webcamAlpha > 0.001) { + float2 webcamUv = input.uv; + if (webcamMirror > 0.5) { + float mirroredX = webcamLeft + (webcamRight - pixel.x); + webcamUv.x = mirroredX / outputWidth; + } + float4 webcam = webcamTexture.Sample(linearSampler, saturate(webcamUv)); + rgb = lerp(rgb, webcam.rgb, saturate(webcamAlpha)); + } + } + } + if (cursorEnabled > 0.5) { + float safeBounceScale = max(cursorBounceScale, 0.1); + float cursorHeight = max(cursorSize, 1.0) * safeBounceScale; + float cursorWidth = cursorHeight * max(cursorAtlasAspect, 0.01); + float2 cursorPixel = float2(cursorX, cursorY); + if (zoomEnabled > 0.5) { + cursorPixel = cursorPixel * safeZoomScale + zoomOffset; + } + float2 cursorLocal = pixel - cursorPixel; + if (cursorAtlasEnabled > 0.5) { + float shadowAlpha = sampleCursorAtlasShadow(cursorLocal, cursorWidth, cursorHeight); + rgb = lerp(rgb, float3(0.0, 0.0, 0.0), shadowAlpha); + + float2 cursorUv = float2( + cursorLocal.x / cursorWidth + cursorAtlasAnchorX, + cursorLocal.y / cursorHeight + cursorAtlasAnchorY + ); + if ( + cursorUv.x >= 0.0 && cursorUv.x <= 1.0 && + cursorUv.y >= 0.0 && cursorUv.y <= 1.0 + ) { + float2 atlasMin = float2(cursorAtlasLeft, cursorAtlasTop); + float2 atlasMax = float2(cursorAtlasRight, cursorAtlasBottom); + float4 cursorSample = cursorAtlasTexture.Sample( + linearSampler, + lerp(atlasMin, atlasMax, cursorUv) + ); + rgb = lerp(rgb, cursorSample.rgb, saturate(cursorSample.a)); + } + } else { + float scale = max(cursorSize, 1.0) / 72.0; + float cursorExtent = max(cursorSize, 1.0) * safeBounceScale * 1.15 + 8.0; + if (abs(cursorLocal.x) <= cursorExtent && abs(cursorLocal.y) <= cursorExtent) { + float shadowMask = cursorArrowMask(cursorLocal - float2(3.0, 3.0), scale); + rgb = lerp(rgb, float3(0.0, 0.0, 0.0), shadowMask * 0.35); + float outlineMask = cursorArrowMask(cursorLocal / 1.08, scale * 1.08); + rgb = lerp(rgb, float3(0.0, 0.0, 0.0), outlineMask * 0.75); + float cursorMask = cursorArrowMask(cursorLocal, scale); + rgb = lerp(rgb, float3(1.0, 1.0, 1.0), cursorMask * 0.95); + } + } + } + return float4(rgb, 1.0); +} +)"; + + ComPtr vertexBytecode; + if (!compileShader(vertexShaderSource, "main", "vs_5_0", &vertexBytecode)) { + return false; + } + HRESULT hr = device_->CreateVertexShader( + vertexBytecode->GetBufferPointer(), + vertexBytecode->GetBufferSize(), + nullptr, + &vertexShader_); + if (!succeeded(hr, "CreateVertexShader")) { + return false; + } + + ComPtr pixelBytecode; + if (!compileShader(pixelShaderSource, "main", "ps_5_0", &pixelBytecode)) { + return false; + } + hr = device_->CreatePixelShader( + pixelBytecode->GetBufferPointer(), + pixelBytecode->GetBufferSize(), + nullptr, + &pixelShader_); + if (!succeeded(hr, "CreatePixelShader")) { + return false; + } + + D3D11_BUFFER_DESC constantDesc = {}; + constantDesc.ByteWidth = sizeof(ShaderConstants); + constantDesc.Usage = D3D11_USAGE_DEFAULT; + constantDesc.BindFlags = D3D11_BIND_CONSTANT_BUFFER; + hr = device_->CreateBuffer(&constantDesc, nullptr, &compositorConstants_); + if (!succeeded(hr, "Create compositor constant buffer")) { + return false; + } + + D3D11_SAMPLER_DESC samplerDesc = {}; + samplerDesc.Filter = D3D11_FILTER_MIN_MAG_MIP_LINEAR; + samplerDesc.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP; + samplerDesc.AddressV = D3D11_TEXTURE_ADDRESS_CLAMP; + samplerDesc.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP; + samplerDesc.MaxLOD = D3D11_FLOAT32_MAX; + hr = device_->CreateSamplerState(&samplerDesc, &samplerState_); + if (!succeeded(hr, "Create compositor sampler state")) { + return false; + } + return createBackgroundTexture() && createCursorAtlasTexture(); + } + + bool createSinkWriter() { + ComPtr attributes; + HRESULT hr = MFCreateAttributes(&attributes, 4); + if (!succeeded(hr, "MFCreateAttributes")) { + return false; + } + attributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE); + attributes->SetUINT32(MF_SINK_WRITER_DISABLE_THROTTLING, TRUE); + attributes->SetUnknown(MF_SINK_WRITER_D3D_MANAGER, deviceManager_.Get()); + + hr = MFCreateSinkWriterFromURL(options_.outputPath.c_str(), nullptr, attributes.Get(), &sinkWriter_); + if (!succeeded(hr, "MFCreateSinkWriterFromURL")) { + return false; + } + + ComPtr outputType; + hr = MFCreateMediaType(&outputType); + if (!succeeded(hr, "MFCreateMediaType output")) { + return false; + } + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + outputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); + outputType->SetUINT32(MF_MT_AVG_BITRATE, options_.bitrate); + outputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + MFSetAttributeSize(outputType.Get(), MF_MT_FRAME_SIZE, options_.width, options_.height); + MFSetAttributeRatio(outputType.Get(), MF_MT_FRAME_RATE, options_.fps, 1); + MFSetAttributeRatio(outputType.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + + hr = sinkWriter_->AddStream(outputType.Get(), &streamIndex_); + if (!succeeded(hr, "IMFSinkWriter::AddStream")) { + return false; + } + + ComPtr inputType; + hr = MFCreateMediaType(&inputType); + if (!succeeded(hr, "MFCreateMediaType input")) { + return false; + } + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); + inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + inputType->SetUINT32(MF_MT_DEFAULT_STRIDE, options_.width); + MFSetAttributeSize(inputType.Get(), MF_MT_FRAME_SIZE, options_.width, options_.height); + MFSetAttributeRatio(inputType.Get(), MF_MT_FRAME_RATE, options_.fps, 1); + MFSetAttributeRatio(inputType.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + + ComPtr encoderAttributes; + if (options_.fastEncoderTuning && SUCCEEDED(MFCreateAttributes(&encoderAttributes, 8))) { + encoderAttributes->SetUINT32(CODECAPI_AVLowLatencyMode, TRUE); + encoderAttributes->SetUINT32(CODECAPI_AVEncCommonQualityVsSpeed, 0); + encoderAttributes->SetUINT32( + CODECAPI_AVEncCommonRateControlMode, + eAVEncCommonRateControlMode_LowDelayVBR); + encoderAttributes->SetUINT32(CODECAPI_AVEncCommonMeanBitRate, options_.bitrate); + encoderAttributes->SetUINT32( + CODECAPI_AVEncCommonMaxBitRate, + static_cast(std::min( + 0xffffffffu, + static_cast(options_.bitrate) * 3 / 2))); + encoderAttributes->SetUINT32(CODECAPI_AVEncMPVDefaultBPictureCount, 0); + encoderAttributes->SetUINT32(CODECAPI_AVEncH264CABACEnable, FALSE); + } + + hr = sinkWriter_->SetInputMediaType(streamIndex_, inputType.Get(), encoderAttributes.Get()); + encoderTuningApplied_ = SUCCEEDED(hr) && encoderAttributes; + if (FAILED(hr) && encoderAttributes) { + hr = sinkWriter_->SetInputMediaType(streamIndex_, inputType.Get(), nullptr); + encoderTuningApplied_ = false; + } + if (!succeeded(hr, "IMFSinkWriter::SetInputMediaType")) { + return false; + } + hr = sinkWriter_->BeginWriting(); + return succeeded(hr, "IMFSinkWriter::BeginWriting"); + } + + bool createNvencSdkEncoder() { +#ifdef RECORDLY_GPU_EXPORT_ENABLE_NVENC_SDK + try { + nvencEncoder_ = std::make_unique( + device_.Get(), + options_.width, + options_.height, + NV_ENC_BUFFER_FORMAT_NV12); + + NV_ENC_INITIALIZE_PARAMS initializeParams = {NV_ENC_INITIALIZE_PARAMS_VER}; + NV_ENC_CONFIG encodeConfig = {NV_ENC_CONFIG_VER}; + initializeParams.encodeConfig = &encodeConfig; + nvencEncoder_->CreateDefaultEncoderParams( + &initializeParams, + NV_ENC_CODEC_H264_GUID, + NV_ENC_PRESET_HP_GUID); + + initializeParams.frameRateNum = options_.fps; + initializeParams.frameRateDen = 1; + initializeParams.enableEncodeAsync = 1; + encodeConfig.profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID; + encodeConfig.gopLength = options_.fps * 2; + encodeConfig.frameIntervalP = 1; + encodeConfig.rcParams.rateControlMode = NV_ENC_PARAMS_RC_VBR; + encodeConfig.rcParams.averageBitRate = options_.bitrate; + encodeConfig.rcParams.maxBitRate = static_cast(std::min( + 0xffffffffu, + static_cast(options_.bitrate) * 3 / 2)); + encodeConfig.rcParams.vbvBufferSize = options_.bitrate; + encodeConfig.rcParams.vbvInitialDelay = options_.bitrate; + encodeConfig.encodeCodecConfig.h264Config.idrPeriod = encodeConfig.gopLength; + + nvencEncoder_->CreateEncoder(&initializeParams); + + if (_wfopen_s(&nvencOutputFile_, options_.outputPath.c_str(), L"wb") != 0 || !nvencOutputFile_) { + std::cerr << "[gpu-export] Failed to open NVENC SDK output" << std::endl; + return false; + } + encoderTuningApplied_ = true; + return true; + } catch (const std::exception& error) { + std::cerr << "[gpu-export] NVENC SDK init failed: " << error.what() << std::endl; + return false; + } +#else + std::cerr << "[gpu-export] --nvenc-sdk requested, but this build was not compiled with NVENC SDK support" + << std::endl; + return false; +#endif + } + + bool createSourceReaderForPath( + const std::wstring& path, + ComPtr& reader, + UINT& width, + UINT& height, + const char* label) { + ComPtr attributes; + HRESULT hr = MFCreateAttributes(&attributes, 4); + if (!succeeded(hr, "MFCreateAttributes source reader")) { + return false; + } + const bool useD3DSourceReader = !options_.preferHighPerformanceAdapter || options_.nvencSdk; + if (useD3DSourceReader) { + attributes->SetUnknown(MF_SOURCE_READER_D3D_MANAGER, deviceManager_.Get()); + } + attributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE); + attributes->SetUINT32( + MF_SOURCE_READER_DISABLE_DXVA, + useD3DSourceReader ? FALSE : TRUE); + if (options_.preferHighPerformanceAdapter && !useD3DSourceReader) { + attributes->SetUINT32(MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, TRUE); + } + + hr = MFCreateSourceReaderFromURL(path.c_str(), attributes.Get(), &reader); + if (!succeeded(hr, label)) { + return false; + } + + ComPtr mediaType; + hr = MFCreateMediaType(&mediaType); + if (!succeeded(hr, "MFCreateMediaType source output")) { + return false; + } + mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + mediaType->SetGUID( + MF_MT_SUBTYPE, + (options_.preferHighPerformanceAdapter && !options_.nvencSdk) ? MFVideoFormat_RGB32 : MFVideoFormat_NV12); + hr = reader->SetCurrentMediaType( + firstVideoStreamIndex(), + nullptr, + mediaType.Get()); + if (!succeeded(hr, "IMFSourceReader::SetCurrentMediaType")) { + return false; + } + + ComPtr currentType; + hr = reader->GetCurrentMediaType(firstVideoStreamIndex(), ¤tType); + if (!succeeded(hr, "IMFSourceReader::GetCurrentMediaType")) { + return false; + } + + UINT32 detectedWidth = 0; + UINT32 detectedHeight = 0; + hr = MFGetAttributeSize(currentType.Get(), MF_MT_FRAME_SIZE, &detectedWidth, &detectedHeight); + if (!succeeded(hr, "MFGetAttributeSize source frame")) { + return false; + } + width = std::max(2, detectedWidth & ~1U); + height = std::max(2, detectedHeight & ~1U); + return true; + } + + bool createSourceReader() { + return createSourceReaderForPath( + options_.inputPath, + sourceReader_, + sourceWidth_, + sourceHeight_, + "MFCreateSourceReaderFromURL input"); + } + + bool createWebcamSourceReader() { + return createSourceReaderForPath( + options_.webcamInputPath, + webcamReader_, + webcamWidth_, + webcamHeight_, + "MFCreateSourceReaderFromURL webcam"); + } + + bool hasWebcamOverlay() const { + return !options_.webcamInputPath.empty() && + options_.webcamLeft >= 0 && + options_.webcamTop >= 0 && + options_.webcamSize >= 2; + } + + bool hasCursorOverlay() const { + return !cursorSamples_.empty() && options_.cursorSize > 0.0f; + } + + bool hasZoomOverlay() const { + return !zoomSamples_.empty(); + } + + void loadCursorTelemetry() { + cursorSamples_.clear(); + if (options_.cursorTelemetryPath.empty()) { + return; + } + + FILE* file = nullptr; + if (_wfopen_s(&file, options_.cursorTelemetryPath.c_str(), L"rb") != 0 || !file) { + std::cerr << "[gpu-export] Unable to open cursor telemetry file" << std::endl; + return; + } + + char line[256]; + while (std::fgets(line, sizeof(line), file)) { + double timeMs = 0.0; + float cx = 0.0f; + float cy = 0.0f; + int cursorTypeIndex = 0; + float bounceScale = 1.0f; + const int parsed = sscanf_s( + line, + "%lf,%f,%f,%d,%f", + &timeMs, + &cx, + &cy, + &cursorTypeIndex, + &bounceScale); + if (parsed < 3) { + continue; + } + if (!std::isfinite(timeMs) || !std::isfinite(cx) || !std::isfinite(cy)) { + continue; + } + cursorSamples_.push_back(CursorSample{ + std::max(0.0, timeMs), + std::min(1.0f, std::max(0.0f, cx)), + std::min(1.0f, std::max(0.0f, cy)), + std::min(8, std::max(0, cursorTypeIndex)), + std::isfinite(bounceScale) + ? std::min(2.0f, std::max(0.1f, bounceScale)) + : 1.0f, + }); + } + std::fclose(file); + + std::sort(cursorSamples_.begin(), cursorSamples_.end(), [](const auto& left, const auto& right) { + return left.timeMs < right.timeMs; + }); + } + + CursorSample getCursorSampleAt(double timeMs) const { + if (cursorSamples_.empty()) { + return {}; + } + if (timeMs <= cursorSamples_.front().timeMs) { + return cursorSamples_.front(); + } + if (timeMs >= cursorSamples_.back().timeMs) { + return cursorSamples_.back(); + } + + const auto upper = std::upper_bound( + cursorSamples_.begin(), + cursorSamples_.end(), + timeMs, + [](double value, const CursorSample& sample) { + return value < sample.timeMs; + }); + const auto& b = *upper; + const auto& a = *(upper - 1); + const double span = std::max(1.0, b.timeMs - a.timeMs); + const float t = static_cast((timeMs - a.timeMs) / span); + return CursorSample{ + timeMs, + a.cx + (b.cx - a.cx) * t, + a.cy + (b.cy - a.cy) * t, + a.cursorTypeIndex, + a.bounceScale + (b.bounceScale - a.bounceScale) * t, + }; + } + + const CursorAtlasEntry* getCursorAtlasEntry(int cursorTypeIndex) const { + if (!hasCursorAtlas_ || cursorAtlasWidth_ == 0 || cursorAtlasHeight_ == 0) { + return nullptr; + } + + const size_t index = static_cast( + std::min(8, std::max(0, cursorTypeIndex))); + if (index >= cursorAtlasEntries_.size() || !cursorAtlasEntries_[index].valid) { + return nullptr; + } + + return &cursorAtlasEntries_[index]; + } + + void loadZoomTelemetry() { + zoomSamples_.clear(); + if (options_.zoomTelemetryPath.empty()) { + return; + } + + FILE* file = nullptr; + if (_wfopen_s(&file, options_.zoomTelemetryPath.c_str(), L"rb") != 0 || !file) { + std::cerr << "[gpu-export] Unable to open zoom telemetry file" << std::endl; + return; + } + + char line[256]; + while (std::fgets(line, sizeof(line), file)) { + double timeMs = 0.0; + float scale = 1.0f; + float x = 0.0f; + float y = 0.0f; + if (sscanf_s(line, "%lf,%f,%f,%f", &timeMs, &scale, &x, &y) != 4) { + continue; + } + if (!std::isfinite(timeMs) || !std::isfinite(scale) || !std::isfinite(x) || !std::isfinite(y)) { + continue; + } + zoomSamples_.push_back(ZoomSample{ + std::max(0.0, timeMs), + std::max(0.01f, scale), + x, + y, + }); + } + std::fclose(file); + + std::sort(zoomSamples_.begin(), zoomSamples_.end(), [](const auto& left, const auto& right) { + return left.timeMs < right.timeMs; + }); + } + + ZoomSample getZoomSampleAt(double timeMs) const { + if (zoomSamples_.empty()) { + return {}; + } + if (timeMs <= zoomSamples_.front().timeMs) { + return zoomSamples_.front(); + } + if (timeMs >= zoomSamples_.back().timeMs) { + return zoomSamples_.back(); + } + + const auto upper = std::upper_bound( + zoomSamples_.begin(), + zoomSamples_.end(), + timeMs, + [](double value, const ZoomSample& sample) { + return value < sample.timeMs; + }); + const auto& b = *upper; + const auto& a = *(upper - 1); + const double span = std::max(1.0, b.timeMs - a.timeMs); + const float t = static_cast((timeMs - a.timeMs) / span); + return ZoomSample{ + timeMs, + a.scale + (b.scale - a.scale) * t, + a.x + (b.x - a.x) * t, + a.y + (b.y - a.y) * t, + }; + } + + RECT getWebcamRect() const { + if (!hasWebcamOverlay()) { + return {0, 0, 0, 0}; + } + + const LONG left = std::min( + std::max(0, options_.webcamLeft), + static_cast(options_.width) - 2); + const LONG top = std::min( + std::max(0, options_.webcamTop), + static_cast(options_.height) - 2); + const LONG size = std::min( + options_.webcamSize & ~1L, + std::min( + static_cast(options_.width) - left, + static_cast(options_.height) - top)); + const LONG safeSize = std::max(2, size); + return {left, top, left + safeSize, top + safeSize}; + } + + RECT getContentRect() const { + if ( + options_.contentLeft >= 0 && + options_.contentTop >= 0 && + options_.contentWidth >= 2 && + options_.contentHeight >= 2 + ) { + const LONG left = std::min( + std::max(0, options_.contentLeft), + static_cast(options_.width) - 2); + const LONG top = std::min( + std::max(0, options_.contentTop), + static_cast(options_.height) - 2); + const LONG width = std::min( + options_.contentWidth & ~1L, + static_cast(options_.width) - left); + const LONG height = std::min( + options_.contentHeight & ~1L, + static_cast(options_.height) - top); + return { + left, + top, + left + std::max(2, width), + top + std::max(2, height), + }; + } + + const double availableWidth = + static_cast(options_.width) * (1.0 - (2.0 * options_.padding)); + const double availableHeight = + static_cast(options_.height) * (1.0 - (2.0 * options_.padding)); + const double scale = std::min( + availableWidth / static_cast(sourceWidth_), + availableHeight / static_cast(sourceHeight_)); + const LONG contentWidth = static_cast( + std::max(2, static_cast(sourceWidth_ * scale) & ~1U)); + const LONG contentHeight = static_cast( + std::max(2, static_cast(sourceHeight_ * scale) & ~1U)); + const LONG x = (static_cast(options_.width) - contentWidth) / 2; + const LONG y = (static_cast(options_.height) - contentHeight) / 2; + return {x, y, x + contentWidth, y + contentHeight}; + } + + void renderSyntheticFrame(UINT frameIndex) { + const float t = static_cast(frameIndex % options_.fps) / static_cast(options_.fps); + const float color[4] = { + 0.05f + 0.45f * t, + 0.18f, + 0.42f + 0.3f * (1.0f - t), + 1.0f, + }; + deviceContext_->ClearRenderTargetView(bgraRenderTargetView_.Get(), color); + } + + bool convertBgraToNv12(UINT frameIndex) { + const size_t surfaceIndex = static_cast(frameIndex) % nv12OutputViews_.size(); + D3D11_VIDEO_PROCESSOR_STREAM stream = {}; + stream.Enable = TRUE; + stream.OutputIndex = 0; + stream.InputFrameOrField = 0; + stream.PastFrames = 0; + stream.FutureFrames = 0; + stream.pInputSurface = bgraInputView_.Get(); + + const HRESULT hr = videoContext_->VideoProcessorBlt( + bgraVideoProcessor_.Get(), + bgraNv12OutputViews_[surfaceIndex].Get(), + frameIndex, + 1, + &stream); + if (!succeeded(hr, "VideoProcessorBlt")) { + return false; + } + return true; + } + + bool convertSourceTextureToNv12( + ID3D11Texture2D* texture, + UINT subresourceIndex, + UINT frameIndex) { + ComPtr inputView; + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC inputViewDesc = {}; + inputViewDesc.FourCC = 0; + inputViewDesc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D; + inputViewDesc.Texture2D.MipSlice = 0; + inputViewDesc.Texture2D.ArraySlice = subresourceIndex; + + HRESULT hr = videoDevice_->CreateVideoProcessorInputView( + texture, + videoProcessorEnumerator_.Get(), + &inputViewDesc, + &inputView); + if (!succeeded(hr, "Create source video processor input view")) { + return false; + } + + const size_t surfaceIndex = static_cast(frameIndex) % nv12OutputViews_.size(); + D3D11_VIDEO_PROCESSOR_STREAM stream = {}; + stream.Enable = TRUE; + stream.OutputIndex = 0; + stream.InputFrameOrField = 0; + stream.pInputSurface = inputView.Get(); + + hr = videoContext_->VideoProcessorBlt( + videoProcessor_.Get(), + nv12OutputViews_[surfaceIndex].Get(), + frameIndex, + 1, + &stream); + if (!succeeded(hr, "VideoProcessorBlt source")) { + return false; + } + return true; + } + + bool convertSourceTextureToBgra( + ID3D11Texture2D* texture, + UINT subresourceIndex, + UINT frameIndex) { + ComPtr inputView; + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC inputViewDesc = {}; + inputViewDesc.FourCC = 0; + inputViewDesc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D; + inputViewDesc.Texture2D.MipSlice = 0; + inputViewDesc.Texture2D.ArraySlice = subresourceIndex; + + HRESULT hr = videoDevice_->CreateVideoProcessorInputView( + texture, + videoProcessorEnumerator_.Get(), + &inputViewDesc, + &inputView); + if (!succeeded(hr, "Create source BGRA video processor input view")) { + return false; + } + + D3D11_VIDEO_PROCESSOR_STREAM stream = {}; + stream.Enable = TRUE; + stream.OutputIndex = 0; + stream.InputFrameOrField = 0; + stream.pInputSurface = inputView.Get(); + + hr = videoContext_->VideoProcessorBlt( + videoProcessor_.Get(), + contentOutputView_.Get(), + frameIndex, + 1, + &stream); + if (!succeeded(hr, "VideoProcessorBlt source BGRA")) { + return false; + } + return true; + } + + bool ensureNv12UploadTexture( + ComPtr& texture, + UINT width, + UINT height, + const char* label) { + if (texture) { + return true; + } + + D3D11_TEXTURE2D_DESC desc = {}; + desc.Width = width; + desc.Height = height; + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_NV12; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_DECODER | D3D11_BIND_SHADER_RESOURCE; + + const HRESULT hr = device_->CreateTexture2D(&desc, nullptr, &texture); + return succeeded(hr, label); + } + + bool uploadNv12BufferToTexture( + IMFMediaBuffer* buffer, + ComPtr& texture, + UINT width, + UINT height, + const char* label) { + if (!ensureNv12UploadTexture(texture, width, height, label)) { + return false; + } + + const DWORD expectedLength = width * height * 3 / 2; + ComPtr buffer2D; + HRESULT hr = buffer->QueryInterface(IID_PPV_ARGS(&buffer2D)); + if (SUCCEEDED(hr)) { + DWORD contiguousLength = 0; + hr = buffer2D->GetContiguousLength(&contiguousLength); + if (SUCCEEDED(hr) && contiguousLength >= expectedLength) { + uploadPaddedScratch_.resize(contiguousLength); + hr = buffer2D->ContiguousCopyTo(uploadPaddedScratch_.data(), contiguousLength); + if (!succeeded(hr, "IMF2DBuffer::ContiguousCopyTo")) { + return false; + } + + const UINT nv12Rows = height + (height / 2); + const UINT sourcePitch = + contiguousLength % nv12Rows == 0 + ? contiguousLength / nv12Rows + : width; + if (sourcePitch < width) { + std::cerr << "ERROR: Unsupported contiguous NV12 pitch." << std::endl; + return false; + } + + uploadScratch_.resize(expectedLength); + const BYTE* yPlane = uploadPaddedScratch_.data(); + const BYTE* uvPlane = uploadPaddedScratch_.data() + (sourcePitch * height); + BYTE* yOut = uploadScratch_.data(); + BYTE* uvOut = uploadScratch_.data() + (width * height); + for (UINT row = 0; row < height; row += 1) { + std::memcpy(yOut + (row * width), yPlane + (row * sourcePitch), width); + } + for (UINT row = 0; row < height / 2; row += 1) { + std::memcpy(uvOut + (row * width), uvPlane + (row * sourcePitch), width); + } + + deviceContext_->UpdateSubresource( + texture.Get(), + 0, + nullptr, + uploadScratch_.data(), + width, + expectedLength); + return true; + } + + BYTE* scanline0 = nullptr; + LONG pitch = 0; + hr = buffer2D->Lock2D(&scanline0, &pitch); + if (!succeeded(hr, "IMF2DBuffer::Lock2D")) { + return false; + } + if (pitch <= 0 || static_cast(pitch) < width) { + buffer2D->Unlock2D(); + std::cerr << "ERROR: Unsupported NV12 pitch." << std::endl; + return false; + } + + uploadScratch_.resize(expectedLength); + const BYTE* yPlane = scanline0; + const BYTE* uvPlane = scanline0 + (pitch * height); + BYTE* yOut = uploadScratch_.data(); + BYTE* uvOut = uploadScratch_.data() + (width * height); + for (UINT row = 0; row < height; row += 1) { + std::memcpy(yOut + (row * width), yPlane + (row * pitch), width); + } + for (UINT row = 0; row < height / 2; row += 1) { + std::memcpy(uvOut + (row * width), uvPlane + (row * pitch), width); + } + buffer2D->Unlock2D(); + + deviceContext_->UpdateSubresource( + texture.Get(), + 0, + nullptr, + uploadScratch_.data(), + width, + expectedLength); + return true; + } + + BYTE* data = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + hr = buffer->Lock(&data, &maxLength, ¤tLength); + if (!succeeded(hr, "IMFMediaBuffer::Lock")) { + return false; + } + if (currentLength < expectedLength) { + buffer->Unlock(); + std::cerr << "ERROR: NV12 buffer shorter than expected." << std::endl; + return false; + } + deviceContext_->UpdateSubresource(texture.Get(), 0, nullptr, data, width, expectedLength); + buffer->Unlock(); + return true; + } + + bool ensureBgraUploadTexture( + ComPtr& texture, + UINT width, + UINT height, + const char* label) { + if (texture) { + return true; + } + + D3D11_TEXTURE2D_DESC desc = {}; + desc.Width = width; + desc.Height = height; + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; + + const HRESULT hr = device_->CreateTexture2D(&desc, nullptr, &texture); + return succeeded(hr, label); + } + + bool uploadBgraBufferToTexture( + IMFMediaBuffer* buffer, + ComPtr& texture, + UINT width, + UINT height, + const char* label) { + if (!ensureBgraUploadTexture(texture, width, height, label)) { + return false; + } + + const DWORD rowBytes = width * 4; + const DWORD expectedLength = rowBytes * height; + ComPtr buffer2D; + HRESULT hr = buffer->QueryInterface(IID_PPV_ARGS(&buffer2D)); + if (SUCCEEDED(hr)) { + BYTE* scanline0 = nullptr; + LONG pitch = 0; + hr = buffer2D->Lock2D(&scanline0, &pitch); + if (!succeeded(hr, "IMF2DBuffer::Lock2D BGRA")) { + return false; + } + + const LONG absPitch = pitch < 0 ? -pitch : pitch; + if (absPitch < static_cast(rowBytes)) { + buffer2D->Unlock2D(); + std::cerr << "ERROR: Unsupported BGRA pitch." << std::endl; + return false; + } + + uploadScratch_.resize(expectedLength); + for (UINT row = 0; row < height; row += 1) { + const BYTE* sourceRow = pitch > 0 + ? scanline0 + (row * pitch) + : scanline0 + ((height - 1 - row) * absPitch); + std::memcpy(uploadScratch_.data() + (row * rowBytes), sourceRow, rowBytes); + } + buffer2D->Unlock2D(); + + deviceContext_->UpdateSubresource( + texture.Get(), + 0, + nullptr, + uploadScratch_.data(), + rowBytes, + expectedLength); + return true; + } + + BYTE* data = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + hr = buffer->Lock(&data, &maxLength, ¤tLength); + if (!succeeded(hr, "IMFMediaBuffer::Lock BGRA")) { + return false; + } + if (currentLength < expectedLength) { + buffer->Unlock(); + std::cerr << "ERROR: BGRA buffer shorter than expected." << std::endl; + return false; + } + deviceContext_->UpdateSubresource( + texture.Get(), + 0, + nullptr, + data, + rowBytes, + expectedLength); + buffer->Unlock(); + return true; + } + + bool convertWebcamTextureToBgra( + ID3D11Texture2D* texture, + UINT subresourceIndex, + UINT frameIndex) { + ComPtr inputView; + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC inputViewDesc = {}; + inputViewDesc.FourCC = 0; + inputViewDesc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D; + inputViewDesc.Texture2D.MipSlice = 0; + inputViewDesc.Texture2D.ArraySlice = subresourceIndex; + + HRESULT hr = videoDevice_->CreateVideoProcessorInputView( + texture, + webcamVideoProcessorEnumerator_.Get(), + &inputViewDesc, + &inputView); + if (!succeeded(hr, "Create webcam video processor input view")) { + return false; + } + + D3D11_VIDEO_PROCESSOR_STREAM stream = {}; + stream.Enable = TRUE; + stream.OutputIndex = 0; + stream.InputFrameOrField = 0; + stream.pInputSurface = inputView.Get(); + + hr = videoContext_->VideoProcessorBlt( + webcamVideoProcessor_.Get(), + webcamOutputView_.Get(), + frameIndex, + 1, + &stream); + if (!succeeded(hr, "VideoProcessorBlt webcam BGRA")) { + return false; + } + webcamFrameReady_ = true; + return true; + } + + bool convertWebcamSampleToBgra( + IMFSample* sample, + UINT frameIndex) { + ComPtr buffer; + HRESULT hr = sample->GetBufferByIndex(0, &buffer); + if (!succeeded(hr, "IMFSample::GetBufferByIndex webcam")) { + return false; + } + + ComPtr dxgiBuffer; + hr = buffer.As(&dxgiBuffer); + if (FAILED(hr)) { + if (options_.preferHighPerformanceAdapter) { + if (!uploadBgraBufferToTexture( + buffer.Get(), + webcamUploadBgraTexture_, + webcamWidth_, + webcamHeight_, + "Create webcam BGRA upload texture")) { + return false; + } + return convertWebcamTextureToBgra(webcamUploadBgraTexture_.Get(), 0, frameIndex); + } else { + if (!uploadNv12BufferToTexture( + buffer.Get(), + webcamUploadTexture_, + webcamWidth_, + webcamHeight_, + "Create webcam NV12 upload texture")) { + return false; + } + return convertWebcamTextureToBgra(webcamUploadTexture_.Get(), 0, frameIndex); + } + } + + ComPtr webcamTexture; + hr = dxgiBuffer->GetResource(IID_PPV_ARGS(&webcamTexture)); + if (!succeeded(hr, "IMFDXGIBuffer::GetResource webcam")) { + return false; + } + UINT subresourceIndex = 0; + dxgiBuffer->GetSubresourceIndex(&subresourceIndex); + return convertWebcamTextureToBgra(webcamTexture.Get(), subresourceIndex, frameIndex); + } + + bool readWebcamFrameForTimestamp(LONGLONG outputTimestamp, UINT frameIndex) { + if (!hasWebcamOverlay() || webcamEnded_) { + return true; + } + + const LONGLONG targetTimestamp = std::max( + 0, + outputTimestamp - static_cast(options_.webcamTimeOffsetMs * 10'000.0)); + + if (pendingWebcamSample_) { + const LONGLONG pendingTimestamp = std::max( + 0, + pendingWebcamTimestamp_ - webcamFirstTimestamp_); + if (pendingTimestamp > targetTimestamp && webcamFrameReady_) { + return true; + } + if (!convertWebcamSampleToBgra(pendingWebcamSample_.Get(), frameIndex)) { + return false; + } + pendingWebcamSample_.Reset(); + if (pendingTimestamp >= targetTimestamp) { + return true; + } + } + + while (true) { + DWORD streamIndex = 0; + DWORD flags = 0; + LONGLONG timestamp = 0; + ComPtr sample; + + const HRESULT hr = webcamReader_->ReadSample( + firstVideoStreamIndex(), + 0, + &streamIndex, + &flags, + ×tamp, + &sample); + if (!succeeded(hr, "IMFSourceReader::ReadSample webcam")) { + return false; + } + if (flags & MF_SOURCE_READERF_ENDOFSTREAM) { + webcamEnded_ = true; + return true; + } + if (!sample) { + return true; + } + if (webcamFirstTimestamp_ < 0) { + webcamFirstTimestamp_ = timestamp; + } + + const LONGLONG adjustedTimestamp = std::max( + 0, + timestamp - webcamFirstTimestamp_); + if (adjustedTimestamp > targetTimestamp && webcamFrameReady_) { + pendingWebcamSample_ = sample; + pendingWebcamTimestamp_ = timestamp; + return true; + } + if (!convertWebcamSampleToBgra(sample.Get(), frameIndex)) { + return false; + } + if (adjustedTimestamp >= targetTimestamp) { + return true; + } + } + } + + bool renderShaderComposite(LONGLONG outputTimestamp) { + const RECT contentRect = getContentRect(); + const RECT webcamRect = getWebcamRect(); + const bool webcamEnabled = hasWebcamOverlay() && webcamFrameReady_; + const bool cursorEnabled = hasCursorOverlay(); + const bool zoomEnabled = hasZoomOverlay(); + const ZoomSample zoom = zoomEnabled + ? getZoomSampleAt(static_cast(outputTimestamp) / 10'000.0) + : ZoomSample{}; + const CursorSample cursor = cursorEnabled + ? getCursorSampleAt(static_cast(outputTimestamp) / 10'000.0) + : CursorSample{}; + const CursorAtlasEntry* cursorAtlasEntry = cursorEnabled + ? getCursorAtlasEntry(cursor.cursorTypeIndex) + : nullptr; + const bool cursorAtlasEnabled = cursorAtlasEntry != nullptr; + const float cursorX = cursorEnabled + ? static_cast(contentRect.left) + + cursor.cx * static_cast(contentRect.right - contentRect.left) + : 0.0f; + const float cursorY = cursorEnabled + ? static_cast(contentRect.top) + + cursor.cy * static_cast(contentRect.bottom - contentRect.top) + : 0.0f; + const ShaderConstants constants = { + static_cast(options_.width), + static_cast(options_.height), + std::min(options_.radius, static_cast( + std::min(contentRect.right - contentRect.left, contentRect.bottom - contentRect.top)) * 0.5f), + options_.shadow, + static_cast(contentRect.left), + static_cast(contentRect.top), + static_cast(contentRect.right), + static_cast(contentRect.bottom), + options_.backgroundR, + options_.backgroundG, + options_.backgroundB, + 1.0f, + 0.0f, + 0.0f, + 0.0f, + 0.42f, + hasBackgroundImage_ ? 1.0f : 0.0f, + static_cast(backgroundImageWidth_), + static_cast(backgroundImageHeight_), + webcamEnabled ? 1.0f : 0.0f, + static_cast(webcamRect.left), + static_cast(webcamRect.top), + static_cast(webcamRect.right), + static_cast(webcamRect.bottom), + webcamEnabled ? std::min(options_.webcamRadius, static_cast( + std::min(webcamRect.right - webcamRect.left, webcamRect.bottom - webcamRect.top)) * 0.5f) : 0.0f, + webcamEnabled ? options_.webcamShadow : 0.0f, + webcamEnabled ? 0.42f : 0.0f, + options_.webcamMirror ? 1.0f : 0.0f, + cursorEnabled ? 1.0f : 0.0f, + cursorX, + cursorY, + options_.cursorSize, + cursorAtlasEnabled ? 1.0f : 0.0f, + cursorAtlasEnabled + ? cursorAtlasEntry->x / static_cast(cursorAtlasWidth_) + : 0.0f, + cursorAtlasEnabled + ? cursorAtlasEntry->y / static_cast(cursorAtlasHeight_) + : 0.0f, + cursorAtlasEnabled + ? (cursorAtlasEntry->x + cursorAtlasEntry->width) / + static_cast(cursorAtlasWidth_) + : 1.0f, + cursorAtlasEnabled + ? (cursorAtlasEntry->y + cursorAtlasEntry->height) / + static_cast(cursorAtlasHeight_) + : 1.0f, + cursorAtlasEnabled ? cursorAtlasEntry->anchorX : 0.0f, + cursorAtlasEnabled ? cursorAtlasEntry->anchorY : 0.0f, + cursorAtlasEnabled ? cursorAtlasEntry->aspectRatio : 1.0f, + cursorEnabled ? cursor.bounceScale : 1.0f, + 0.0f, + 0.0f, + 0.0f, + zoomEnabled ? 1.0f : 0.0f, + zoomEnabled ? zoom.scale : 1.0f, + zoomEnabled ? zoom.x : 0.0f, + zoomEnabled ? zoom.y : 0.0f, + }; + + deviceContext_->UpdateSubresource(compositorConstants_.Get(), 0, nullptr, &constants, 0, 0); + + const float clearColor[4] = {constants.backgroundR, constants.backgroundG, constants.backgroundB, 1.0f}; + deviceContext_->ClearRenderTargetView(bgraRenderTargetView_.Get(), clearColor); + + D3D11_VIEWPORT viewport = {}; + viewport.Width = static_cast(options_.width); + viewport.Height = static_cast(options_.height); + viewport.MinDepth = 0.0f; + viewport.MaxDepth = 1.0f; + + ID3D11RenderTargetView* renderTargets[] = {bgraRenderTargetView_.Get()}; + deviceContext_->OMSetRenderTargets(1, renderTargets, nullptr); + deviceContext_->RSSetViewports(1, &viewport); + deviceContext_->IASetInputLayout(nullptr); + deviceContext_->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST); + deviceContext_->VSSetShader(vertexShader_.Get(), nullptr, 0); + deviceContext_->PSSetShader(pixelShader_.Get(), nullptr, 0); + ID3D11Buffer* constantBuffers[] = {compositorConstants_.Get()}; + deviceContext_->PSSetConstantBuffers(0, 1, constantBuffers); + ID3D11ShaderResourceView* shaderResources[] = { + contentShaderResourceView_.Get(), + backgroundShaderResourceView_.Get(), + webcamShaderResourceView_ ? webcamShaderResourceView_.Get() : contentShaderResourceView_.Get(), + cursorAtlasShaderResourceView_ + ? cursorAtlasShaderResourceView_.Get() + : backgroundShaderResourceView_.Get(), + }; + deviceContext_->PSSetShaderResources(0, 4, shaderResources); + ID3D11SamplerState* samplers[] = {samplerState_.Get()}; + deviceContext_->PSSetSamplers(0, 1, samplers); + deviceContext_->Draw(3, 0); + + ID3D11ShaderResourceView* nullResources[] = {nullptr, nullptr, nullptr, nullptr}; + deviceContext_->PSSetShaderResources(0, 4, nullResources); + return true; + } + + bool writeNvencSdkPackets(const std::vector>& packets) { + if (!nvencOutputFile_) { + return false; + } + for (const auto& packet : packets) { + if (packet.empty()) { + continue; + } + const size_t written = std::fwrite(packet.data(), 1, packet.size(), nvencOutputFile_); + if (written != packet.size()) { + std::cerr << "[gpu-export] Failed to write NVENC SDK packet" << std::endl; + return false; + } + nvencOutputBytes_ += packet.size(); + } + return true; + } + + bool writeNvencSdkFrame(UINT frameIndex) { +#ifdef RECORDLY_GPU_EXPORT_ENABLE_NVENC_SDK + if (!nvencEncoder_) { + std::cerr << "[gpu-export] NVENC SDK encoder is not initialized" << std::endl; + return false; + } + try { + const size_t surfaceIndex = static_cast(frameIndex) % nv12Textures_.size(); + const NvEncInputFrame* inputFrame = nvencEncoder_->GetNextInputFrame(); + auto* encoderTexture = reinterpret_cast(inputFrame->inputPtr); + deviceContext_->CopyResource(encoderTexture, nv12Textures_[surfaceIndex].Get()); + deviceContext_->Flush(); + + std::vector> packets; + nvencEncoder_->EncodeFrame(packets); + return writeNvencSdkPackets(packets); + } catch (const std::exception& error) { + std::cerr << "[gpu-export] NVENC SDK encode failed: " << error.what() << std::endl; + return false; + } +#else + (void)frameIndex; + return false; +#endif + } + + bool finalizeNvencSdk() { +#ifdef RECORDLY_GPU_EXPORT_ENABLE_NVENC_SDK + if (!nvencEncoder_) { + return true; + } + try { + std::vector> packets; + nvencEncoder_->EndEncode(packets); + if (!writeNvencSdkPackets(packets)) { + return false; + } + nvencEncoder_->DestroyEncoder(); + nvencEncoder_.reset(); + if (nvencOutputFile_) { + std::fclose(nvencOutputFile_); + nvencOutputFile_ = nullptr; + } + return true; + } catch (const std::exception& error) { + std::cerr << "[gpu-export] NVENC SDK finalize failed: " << error.what() << std::endl; + return false; + } +#else + return false; +#endif + } + + bool writeFrame( + UINT frameIndex, + LONGLONG sampleTimeOverride = -1, + LONGLONG sampleDurationOverride = -1) { + if (options_.nvencSdk) { + return writeNvencSdkFrame(frameIndex); + } + + const size_t surfaceIndex = static_cast(frameIndex) % nv12Textures_.size(); + ComPtr buffer; + HRESULT hr = MFCreateDXGISurfaceBuffer( + __uuidof(ID3D11Texture2D), + nv12Textures_[surfaceIndex].Get(), + 0, + FALSE, + &buffer); + if (!succeeded(hr, "MFCreateDXGISurfaceBuffer")) { + return false; + } + const DWORD nv12ByteLength = options_.width * options_.height * 3 / 2; + hr = buffer->SetCurrentLength(nv12ByteLength); + if (!succeeded(hr, "IMFMediaBuffer::SetCurrentLength")) { + return false; + } + + ComPtr sample; + hr = MFCreateSample(&sample); + if (!succeeded(hr, "MFCreateSample")) { + return false; + } + hr = sample->AddBuffer(buffer.Get()); + if (!succeeded(hr, "IMFSample::AddBuffer")) { + return false; + } + + const LONGLONG defaultSampleTime = + static_cast(frameIndex) * 10'000'000LL / options_.fps; + const LONGLONG defaultSampleDuration = 10'000'000LL / options_.fps; + const LONGLONG sampleTime = + sampleTimeOverride >= 0 ? sampleTimeOverride : defaultSampleTime; + const LONGLONG sampleDuration = + sampleDurationOverride > 0 ? sampleDurationOverride : defaultSampleDuration; + sample->SetSampleTime(sampleTime); + sample->SetSampleDuration(sampleDuration); + + hr = sinkWriter_->WriteSample(streamIndex_, sample.Get()); + return succeeded(hr, "IMFSinkWriter::WriteSample"); + } + + bool runSourceVideo() { + const UINT maxFrames = std::max( + static_cast(std::ceil(static_cast(options_.fps) * options_.seconds * 4.0)), + static_cast(std::ceil(options_.seconds * 240.0))); + const Timer totalTimer; + double readMs = 0; + double processMs = 0; + double writeMs = 0; + UINT frameIndex = 0; + bool sawDxgiSurface = false; + LONGLONG firstSourceTimestamp = -1; + LONGLONG lastOutputTimestamp = 0; + const LONGLONG maxOutputTimestamp = + static_cast(options_.seconds * 10'000'000.0); + const LONGLONG outputFrameDuration = 10'000'000LL / options_.fps; + LONGLONG nextOutputTimestamp = 0; + const UINT expectedOutputFrames = std::max( + 1, + static_cast(std::ceil(options_.seconds * static_cast(options_.fps)))); + + while (frameIndex < maxFrames && frameIndex < expectedOutputFrames) { + DWORD streamIndex = 0; + DWORD flags = 0; + LONGLONG timestamp = 0; + ComPtr sample; + + const Timer readTimer; + HRESULT hr = sourceReader_->ReadSample( + firstVideoStreamIndex(), + 0, + &streamIndex, + &flags, + ×tamp, + &sample); + readMs += readTimer.elapsedMs(); + if (!succeeded(hr, "IMFSourceReader::ReadSample")) { + return false; + } + if (flags & MF_SOURCE_READERF_ENDOFSTREAM) { + break; + } + if (!sample) { + continue; + } + if (firstSourceTimestamp < 0) { + firstSourceTimestamp = timestamp; + } + const LONGLONG outputTimestamp = std::max(0, timestamp - firstSourceTimestamp); + if (outputTimestamp >= maxOutputTimestamp) { + break; + } + const LONGLONG sampleWindowEnd = outputTimestamp + (outputFrameDuration / 2); + if (sampleWindowEnd < nextOutputTimestamp) { + continue; + } + + ComPtr buffer; + hr = sample->GetBufferByIndex(0, &buffer); + if (!succeeded(hr, "IMFSample::GetBufferByIndex")) { + return false; + } + + ComPtr dxgiBuffer; + hr = buffer.As(&dxgiBuffer); + if (FAILED(hr)) { + ID3D11Texture2D* uploadedSourceTexture = nullptr; + if (options_.preferHighPerformanceAdapter) { + if (!uploadBgraBufferToTexture( + buffer.Get(), + sourceUploadBgraTexture_, + sourceWidth_, + sourceHeight_, + "Create source BGRA upload texture")) { + return false; + } + uploadedSourceTexture = sourceUploadBgraTexture_.Get(); + } else { + if (!uploadNv12BufferToTexture( + buffer.Get(), + sourceUploadTexture_, + sourceWidth_, + sourceHeight_, + "Create source NV12 upload texture")) { + return false; + } + uploadedSourceTexture = sourceUploadTexture_.Get(); + } + + while ( + frameIndex < maxFrames && + frameIndex < expectedOutputFrames && + nextOutputTimestamp <= sampleWindowEnd && + nextOutputTimestamp < maxOutputTimestamp + ) { + const Timer processTimer; + if (options_.shaderComposite) { + if (!convertSourceTextureToBgra(uploadedSourceTexture, 0, frameIndex)) { + return false; + } + if (!readWebcamFrameForTimestamp(nextOutputTimestamp, frameIndex)) { + return false; + } + if (!renderShaderComposite(nextOutputTimestamp)) { + return false; + } + if (!convertBgraToNv12(frameIndex)) { + return false; + } + } else if (!convertSourceTextureToNv12(uploadedSourceTexture, 0, frameIndex)) { + return false; + } + processMs += processTimer.elapsedMs(); + + const Timer writeTimer; + lastOutputTimestamp = nextOutputTimestamp; + if (!writeFrame(frameIndex, nextOutputTimestamp, outputFrameDuration)) { + return false; + } + writeMs += writeTimer.elapsedMs(); + frameIndex++; + nextOutputTimestamp += outputFrameDuration; + emitProgress(std::min(frameIndex, expectedOutputFrames), expectedOutputFrames); + } + + continue; + } + sawDxgiSurface = true; + + ComPtr sourceTexture; + hr = dxgiBuffer->GetResource(IID_PPV_ARGS(&sourceTexture)); + if (!succeeded(hr, "IMFDXGIBuffer::GetResource")) { + return false; + } + UINT subresourceIndex = 0; + dxgiBuffer->GetSubresourceIndex(&subresourceIndex); + + while ( + frameIndex < maxFrames && + frameIndex < expectedOutputFrames && + nextOutputTimestamp <= sampleWindowEnd && + nextOutputTimestamp < maxOutputTimestamp + ) { + const Timer processTimer; + if (options_.shaderComposite) { + if (!convertSourceTextureToBgra(sourceTexture.Get(), subresourceIndex, frameIndex)) { + return false; + } + if (!readWebcamFrameForTimestamp(nextOutputTimestamp, frameIndex)) { + return false; + } + if (!renderShaderComposite(nextOutputTimestamp)) { + return false; + } + if (!convertBgraToNv12(frameIndex)) { + return false; + } + } else { + if (!convertSourceTextureToNv12(sourceTexture.Get(), subresourceIndex, frameIndex)) { + return false; + } + } + processMs += processTimer.elapsedMs(); + + const Timer writeTimer; + lastOutputTimestamp = nextOutputTimestamp; + if (!writeFrame(frameIndex, nextOutputTimestamp, outputFrameDuration)) { + return false; + } + writeMs += writeTimer.elapsedMs(); + ++frameIndex; + emitProgress(std::min(frameIndex, expectedOutputFrames), expectedOutputFrames); + nextOutputTimestamp += outputFrameDuration; + } + } + emitProgress(std::min(frameIndex, expectedOutputFrames), expectedOutputFrames, true); + + const Timer finalizeTimer; + const bool finalized = options_.nvencSdk ? finalizeNvencSdk() : SUCCEEDED(sinkWriter_->Finalize()); + const double finalizeMs = finalizeTimer.elapsedMs(); + if (!finalized) { + if (!options_.nvencSdk) { + std::cerr << "ERROR: IMFSinkWriter::Finalize failed" << std::endl; + } + return false; + } + + const double totalMs = totalTimer.elapsedMs(); + const double mediaMs = + (static_cast(lastOutputTimestamp) / 10'000.0) + + (1000.0 / static_cast(options_.fps)); + const double realtime = mediaMs / totalMs; + std::cout + << "{" + << "\"success\":true," + << "\"mode\":\"source-video\"," + << "\"shaderComposite\":" << (options_.shaderComposite ? "true" : "false") << "," + << "\"webcamOverlay\":" << (hasWebcamOverlay() ? "true" : "false") << "," + << "\"cursorOverlay\":" << (hasCursorOverlay() ? "true" : "false") << "," + << "\"cursorAtlas\":" << (hasCursorAtlas_ ? "true" : "false") << "," + << "\"zoomOverlay\":" << (hasZoomOverlay() ? "true" : "false") << "," + << "\"gpuDecodeSurface\":" << (sawDxgiSurface ? "true" : "false") << "," + << "\"sourceWidth\":" << sourceWidth_ << "," + << "\"sourceHeight\":" << sourceHeight_ << "," + << "\"width\":" << options_.width << "," + << "\"height\":" << options_.height << "," + << "\"fps\":" << options_.fps << "," + << "\"surfacePoolSize\":" << options_.surfacePoolSize << "," + << "\"adapterIndex\":" << selectedAdapterIndex_ << "," + << "\"adapterVendorId\":" << selectedAdapterVendorId_ << "," + << "\"adapterDeviceId\":" << selectedAdapterDeviceId_ << "," + << "\"adapterDedicatedVideoMemoryMB\":" << selectedAdapterDedicatedVideoMemoryMB_ << "," + << "\"frames\":" << frameIndex << "," + << "\"mediaMs\":" << mediaMs << "," + << "\"initializeMs\":" << initializeMs_ << "," + << "\"initCoInitializeMs\":" << initCoInitializeMs_ << "," + << "\"initMfStartupMs\":" << initMfStartupMs_ << "," + << "\"initD3DDeviceMs\":" << initD3DDeviceMs_ << "," + << "\"initSourceReaderMs\":" << initSourceReaderMs_ << "," + << "\"initWebcamReaderMs\":" << initWebcamReaderMs_ << "," + << "\"initVideoProcessorMs\":" << initVideoProcessorMs_ << "," + << "\"initTexturesMs\":" << initTexturesMs_ << "," + << "\"initShaderPipelineMs\":" << initShaderPipelineMs_ << "," + << "\"initSinkWriterMs\":" << initSinkWriterMs_ << "," + << "\"encoderBackend\":\"" << (options_.nvencSdk ? "nvenc-sdk-d3d11" : "media-foundation") << "\"," + << "\"encoderTuningApplied\":" << (encoderTuningApplied_ ? "true" : "false") << "," + << "\"nvencOutputBytes\":" << nvencOutputBytes_ << "," + << "\"totalMs\":" << totalMs << "," + << "\"readMs\":" << readMs << "," + << "\"videoProcessMs\":" << processMs << "," + << "\"writeSampleMs\":" << writeMs << "," + << "\"finalizeMs\":" << finalizeMs << "," + << "\"realtimeMultiplier\":" << realtime + << "}" << std::endl; + return true; + } + + Options options_; + UINT sourceWidth_ = 1920; + UINT sourceHeight_ = 1080; + double initializeMs_ = 0.0; + double initCoInitializeMs_ = 0.0; + double initMfStartupMs_ = 0.0; + double initD3DDeviceMs_ = 0.0; + double initSourceReaderMs_ = 0.0; + double initWebcamReaderMs_ = 0.0; + double initVideoProcessorMs_ = 0.0; + double initTexturesMs_ = 0.0; + double initShaderPipelineMs_ = 0.0; + double initSinkWriterMs_ = 0.0; + UINT selectedAdapterVendorId_ = 0; + UINT selectedAdapterDeviceId_ = 0; + UINT64 selectedAdapterDedicatedVideoMemoryMB_ = 0; + int selectedAdapterIndex_ = -1; + bool coInitialized_ = false; + bool mfStarted_ = false; + DWORD streamIndex_ = 0; + ComPtr device_; + ComPtr deviceContext_; + ComPtr videoDevice_; + ComPtr videoContext_; + ComPtr deviceManager_; + ComPtr sourceReader_; + ComPtr webcamReader_; + ComPtr videoProcessorEnumerator_; + ComPtr videoProcessor_; + ComPtr bgraVideoProcessorEnumerator_; + ComPtr bgraVideoProcessor_; + ComPtr webcamVideoProcessorEnumerator_; + ComPtr webcamVideoProcessor_; + ComPtr bgraTexture_; + ComPtr bgraRenderTargetView_; + ComPtr bgraInputView_; + ComPtr contentTexture_; + ComPtr contentOutputView_; + ComPtr contentShaderResourceView_; + ComPtr webcamTexture_; + ComPtr webcamOutputView_; + ComPtr webcamShaderResourceView_; + ComPtr backgroundShaderResourceView_; + ComPtr cursorAtlasShaderResourceView_; + ComPtr vertexShader_; + ComPtr pixelShader_; + ComPtr samplerState_; + ComPtr compositorConstants_; + UINT backgroundImageWidth_ = 1; + UINT backgroundImageHeight_ = 1; + bool hasBackgroundImage_ = false; + UINT cursorAtlasWidth_ = 1; + UINT cursorAtlasHeight_ = 1; + bool hasCursorAtlas_ = false; + std::array cursorAtlasEntries_; + UINT webcamWidth_ = 640; + UINT webcamHeight_ = 480; + bool webcamFrameReady_ = false; + bool webcamEnded_ = false; + LONGLONG webcamFirstTimestamp_ = -1; + LONGLONG pendingWebcamTimestamp_ = 0; + ComPtr pendingWebcamSample_; + std::vector cursorSamples_; + std::vector zoomSamples_; + ComPtr sourceUploadTexture_; + ComPtr webcamUploadTexture_; + ComPtr sourceUploadBgraTexture_; + ComPtr webcamUploadBgraTexture_; + std::vector uploadScratch_; + std::vector uploadPaddedScratch_; + std::vector> nv12Textures_; + std::vector> nv12OutputViews_; + std::vector> bgraNv12OutputViews_; + ComPtr sinkWriter_; +#ifdef RECORDLY_GPU_EXPORT_ENABLE_NVENC_SDK + std::unique_ptr nvencEncoder_; +#endif + FILE* nvencOutputFile_ = nullptr; + uint64_t nvencOutputBytes_ = 0; + bool encoderTuningApplied_ = false; +}; + +} // namespace + +int wmain(int argc, wchar_t** argv) { + const Options options = parseOptions(argc, argv); + GpuProbe probe; + if (!probe.initialize(options)) { + std::cerr << "{\"success\":false,\"phase\":\"initialize\"}" << std::endl; + return 1; + } + if (!probe.run()) { + std::cerr << "{\"success\":false,\"phase\":\"run\"}" << std::endl; + return 1; + } + return 0; +} diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index 7fca95b8..24fbc015 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -183,6 +183,8 @@ static void writeCompanionAudioTimingMetadata( } metadataFile << "{\"startDelayMs\":" << startDelayMs; + metadataFile << ",\"capturedDurationMs\":" << capture.capturedDurationMs(); + metadataFile << ",\"dataBytes\":" << capture.totalDataBytes(); const uint32_t discontinuityCount = capture.dataDiscontinuityCount(); if (discontinuityCount > 0) { metadataFile << ",\"dataDiscontinuityCount\":" << discontinuityCount; diff --git a/electron/native/wgc-capture/src/wasapi_loopback.cpp b/electron/native/wgc-capture/src/wasapi_loopback.cpp index e4008def..547423e3 100644 --- a/electron/native/wgc-capture/src/wasapi_loopback.cpp +++ b/electron/native/wgc-capture/src/wasapi_loopback.cpp @@ -2,9 +2,41 @@ #include #include #include +#include #pragma comment(lib, "ole32.lib") +namespace { +constexpr int64_t kHundredNanosecondsPerSecond = 10000000; +constexpr uint64_t kSilenceWriteChunkFrames = 4096; + +bool isFloatFormat(const WAVEFORMATEX* format) { + if (!format) return false; + if (format->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) return true; + if (format->wFormatTag != WAVE_FORMAT_EXTENSIBLE) return false; + return reinterpret_cast(format)->SubFormat == + KSDATAFORMAT_SUBTYPE_IEEE_FLOAT; +} + +bool isPcmFormat(const WAVEFORMATEX* format) { + if (!format) return false; + if (format->wFormatTag == WAVE_FORMAT_PCM) return true; + if (format->wFormatTag != WAVE_FORMAT_EXTENSIBLE) return false; + return reinterpret_cast(format)->SubFormat == + KSDATAFORMAT_SUBTYPE_PCM; +} + +int16_t pcm24ToInt16(const BYTE* sample) { + int32_t value = static_cast(sample[0]) | + (static_cast(sample[1]) << 8) | + (static_cast(sample[2]) << 16); + if ((value & 0x800000) != 0) { + value |= ~0xFFFFFF; + } + return static_cast(value >> 8); +} +} + static const CLSID CLSID_MMDeviceEnumerator_ = __uuidof(MMDeviceEnumerator); static const IID IID_IMMDeviceEnumerator_ = __uuidof(IMMDeviceEnumerator); static const IID IID_IAudioClient_ = __uuidof(IAudioClient); @@ -137,6 +169,7 @@ bool WasapiCapture::start() { } totalDataBytes_ = 0; + framesWritten_ = 0; firstPacketQpcHns_ = -1; dataDiscontinuityCount_ = 0; timestampErrorCount_ = 0; @@ -177,7 +210,10 @@ void WasapiCapture::stop() { if (outputFile_ != INVALID_HANDLE_VALUE) { SetFilePointer(outputFile_, 0, nullptr, FILE_BEGIN); - writeWavHeader(outputFile_, totalDataBytes_); + const uint64_t dataBytes = totalDataBytes_.load(); + const DWORD wavDataBytes = + static_cast(std::min(dataBytes, 0xFFFFFFFFu)); + writeWavHeader(outputFile_, wavDataBytes); CloseHandle(outputFile_); outputFile_ = INVALID_HANDLE_VALUE; } @@ -217,6 +253,39 @@ bool WasapiCapture::writeWavHeader(HANDLE file, DWORD dataSize) { return true; } +void WasapiCapture::writePcmFrames(const int16_t* samples, UINT32 frameCount, WORD channels) { + if (!samples || frameCount == 0 || channels == 0 || outputFile_ == INVALID_HANDLE_VALUE) { + return; + } + + const DWORD bytesToWrite = frameCount * channels * sizeof(int16_t); + DWORD written = 0; + WriteFile(outputFile_, samples, bytesToWrite, &written, nullptr); + totalDataBytes_.fetch_add(written); + framesWritten_.fetch_add(written / (channels * sizeof(int16_t))); +} + +void WasapiCapture::writeSilenceFrames(uint64_t frameCount, WORD channels) { + if (frameCount == 0 || channels == 0 || outputFile_ == INVALID_HANDLE_VALUE) { + return; + } + + std::vector silence(static_cast(kSilenceWriteChunkFrames * channels), 0); + while (frameCount > 0) { + const uint64_t chunkFrames = std::min(frameCount, kSilenceWriteChunkFrames); + writePcmFrames(silence.data(), static_cast(chunkFrames), channels); + frameCount -= chunkFrames; + } +} + +uint64_t WasapiCapture::capturedDurationMs() const { + if (!mixFormat_ || mixFormat_->nSamplesPerSec == 0) { + return 0; + } + + return (framesWritten_.load() * 1000) / mixFormat_->nSamplesPerSec; +} + void WasapiCapture::captureThread() { // COM must be initialized on every thread that uses COM objects. // The main thread calls winrt::init_apartment(MTA) but that only covers @@ -225,9 +294,12 @@ void WasapiCapture::captureThread() { CoInitializeEx(nullptr, COINIT_MULTITHREADED); WORD channels = static_cast(mixFormat_->nChannels); - bool isFloat = (mixFormat_->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) || - (mixFormat_->wFormatTag == WAVE_FORMAT_EXTENSIBLE && - reinterpret_cast(mixFormat_)->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT); + const bool isFloat = isFloatFormat(mixFormat_); + const bool isPcm = isPcmFormat(mixFormat_); + const WORD bitsPerSample = mixFormat_->wBitsPerSample; + const WORD sourceBlockAlign = mixFormat_->nBlockAlign; + const WORD sourceBytesPerSample = + channels > 0 ? static_cast(sourceBlockAlign / channels) : 0; std::vector pcmBuffer; @@ -273,38 +345,96 @@ void WasapiCapture::captureThread() { if ((flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) != 0) { timestampErrorCount_.fetch_add(1); } - if ( + const bool hasReliableTimestamp = numFrames > 0 && qpcPosition > 0 && - (flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) == 0 - ) { + (flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) == 0; + if (hasReliableTimestamp) { int64_t expected = -1; firstPacketQpcHns_.compare_exchange_strong( expected, static_cast(qpcPosition)); + + const int64_t firstPacketQpcHns = firstPacketQpcHns_.load(); + if ( + firstPacketQpcHns >= 0 && + static_cast(qpcPosition) > firstPacketQpcHns + ) { + const int64_t elapsedHns = + static_cast(qpcPosition) - firstPacketQpcHns; + const uint64_t expectedStartFrame = + (static_cast(elapsedHns) * mixFormat_->nSamplesPerSec + + kHundredNanosecondsPerSecond / 2) / + kHundredNanosecondsPerSecond; + const uint64_t writtenFrames = framesWritten_.load(); + const uint64_t gapThresholdFrames = mixFormat_->nSamplesPerSec / 100; + + if (expectedStartFrame > writtenFrames + gapThresholdFrames) { + writeSilenceFrames(expectedStartFrame - writtenFrames, channels); + } + } } UINT32 totalSamples = numFrames * channels; if (flags & AUDCLNT_BUFFERFLAGS_SILENT) { pcmBuffer.assign(totalSamples, 0); - } else if (isFloat) { + } else if (isFloat && bitsPerSample == 32 && sourceBytesPerSample >= 4) { pcmBuffer.resize(totalSamples); const float* src = reinterpret_cast(data); for (UINT32 i = 0; i < totalSamples; i++) { pcmBuffer[i] = floatToInt16(src[i]); } - } else { + } else if (isPcm && bitsPerSample == 16 && sourceBytesPerSample >= 2) { pcmBuffer.resize(totalSamples); - std::memcpy(pcmBuffer.data(), data, totalSamples * sizeof(int16_t)); + if (sourceBytesPerSample == sizeof(int16_t)) { + std::memcpy(pcmBuffer.data(), data, totalSamples * sizeof(int16_t)); + } else { + for (UINT32 frame = 0; frame < numFrames; frame++) { + const BYTE* frameData = data + frame * sourceBlockAlign; + for (WORD channel = 0; channel < channels; channel++) { + const BYTE* sample = frameData + channel * sourceBytesPerSample; + pcmBuffer[frame * channels + channel] = + *reinterpret_cast(sample); + } + } + } + } else if (isPcm && bitsPerSample == 24 && sourceBytesPerSample >= 3) { + pcmBuffer.resize(totalSamples); + for (UINT32 frame = 0; frame < numFrames; frame++) { + const BYTE* frameData = data + frame * sourceBlockAlign; + for (WORD channel = 0; channel < channels; channel++) { + const BYTE* sample = frameData + channel * sourceBytesPerSample; + pcmBuffer[frame * channels + channel] = pcm24ToInt16(sample); + } + } + } else if (isPcm && bitsPerSample == 32 && sourceBytesPerSample >= 4) { + pcmBuffer.resize(totalSamples); + for (UINT32 frame = 0; frame < numFrames; frame++) { + const BYTE* frameData = data + frame * sourceBlockAlign; + for (WORD channel = 0; channel < channels; channel++) { + const BYTE* sample = frameData + channel * sourceBytesPerSample; + const int32_t value = *reinterpret_cast(sample); + pcmBuffer[frame * channels + channel] = static_cast(value >> 16); + } + } + } else if (isPcm && bitsPerSample == 8 && sourceBytesPerSample >= 1) { + pcmBuffer.resize(totalSamples); + for (UINT32 frame = 0; frame < numFrames; frame++) { + const BYTE* frameData = data + frame * sourceBlockAlign; + for (WORD channel = 0; channel < channels; channel++) { + const BYTE value = *(frameData + channel * sourceBytesPerSample); + pcmBuffer[frame * channels + channel] = + static_cast((static_cast(value) - 128) << 8); + } + } + } else { + pcmBuffer.assign(totalSamples, 0); } captureClient_->ReleaseBuffer(numFrames); - DWORD bytesToWrite = totalSamples * sizeof(int16_t); - DWORD written; - WriteFile(outputFile_, pcmBuffer.data(), bytesToWrite, &written, nullptr); - totalDataBytes_ += written; + writePcmFrames(pcmBuffer.data(), numFrames, channels); hr = captureClient_->GetNextPacketSize(&packetLength); if (FAILED(hr)) { diff --git a/electron/native/wgc-capture/src/wasapi_loopback.h b/electron/native/wgc-capture/src/wasapi_loopback.h index 1792a863..b925f0d3 100644 --- a/electron/native/wgc-capture/src/wasapi_loopback.h +++ b/electron/native/wgc-capture/src/wasapi_loopback.h @@ -20,6 +20,8 @@ public: bool resume(); void stop(); int64_t firstPacketQpcHns() const { return firstPacketQpcHns_.load(); } + uint64_t capturedDurationMs() const; + uint64_t totalDataBytes() const { return totalDataBytes_.load(); } uint32_t dataDiscontinuityCount() const { return dataDiscontinuityCount_.load(); } uint32_t timestampErrorCount() const { return timestampErrorCount_.load(); } @@ -27,6 +29,8 @@ private: bool initializeCommon(); void captureThread(); bool writeWavHeader(HANDLE file, DWORD dataSize); + void writePcmFrames(const int16_t* samples, UINT32 frameCount, WORD channels); + void writeSilenceFrames(uint64_t frameCount, WORD channels); IMMDevice* findCaptureDeviceByName(const std::wstring& name); std::string outputPath_; @@ -34,7 +38,8 @@ private: std::atomic capturing_{false}; std::atomic paused_{false}; HANDLE outputFile_ = INVALID_HANDLE_VALUE; - DWORD totalDataBytes_ = 0; + std::atomic totalDataBytes_{0}; + std::atomic framesWritten_{0}; IMMDeviceEnumerator* enumerator_ = nullptr; IMMDevice* device_ = nullptr; diff --git a/electron/preload.ts b/electron/preload.ts index c9e464f2..69f2ef8b 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -10,6 +10,84 @@ type NativeVideoAudioMuxMetrics = { tempEditedAudioBytes?: number; muxedVideoBytes?: number; }; +type WindowsGpuExportSummary = { + success?: boolean; + width?: number; + height?: number; + fps?: number; + seconds?: number; + mediaMs?: number; + frames?: number; + cursorOverlay?: boolean; + zoomOverlay?: boolean; + adapterVendorId?: number; + adapterDeviceId?: number; + adapterDedicatedVideoMemoryMB?: number; + initializeMs?: number; + initCoInitializeMs?: number; + initMfStartupMs?: number; + initD3DDeviceMs?: number; + initSourceReaderMs?: number; + initWebcamReaderMs?: number; + initVideoProcessorMs?: number; + initTexturesMs?: number; + initShaderPipelineMs?: number; + initSinkWriterMs?: number; + totalMs?: number; + readMs?: number; + clearMs?: number; + videoProcessMs?: number; + writeSampleMs?: number; + finalizeMs?: number; + realtimeMultiplier?: number; +}; +type NativeStaticLayoutChunkMetric = { + index: number; + startSec: number; + durationSec: number; + backend: + | "cuda-overlay" + | "cuda-scale-cpu-pad" + | "cuda-static-composite" + | "nvidia-cuda-compositor" + | "windows-d3d11-compositor"; + elapsedMs: number; + outputBytes: number; + fallbackReason?: string; + windowsGpuSummary?: WindowsGpuExportSummary; +}; +type NativeStaticLayoutMetrics = NativeVideoAudioMuxMetrics & { + chunkCount: number; + chunkDurationSec: number; + chunkExecMs: number; + concatExecMs?: number; + staticAssetExecMs?: number; + fallbackChunkCount: number; + videoOnlyBytes?: number; + chunks: NativeStaticLayoutChunkMetric[]; +}; +type NativeStaticLayoutProgress = { + sessionId?: string; + backend?: NativeStaticLayoutChunkMetric["backend"]; + elapsedMs?: number; + averageFps?: number; + currentFrame: number; + totalFrames: number; + percentage: number; +}; +type NativeVideoMetadataProbe = { + width: number; + height: number; + duration: number; + mediaStartTime?: number; + streamStartTime?: number; + streamDuration?: number; + frameRate: number; + codec: string; + hasAudio: boolean; + audioCodec?: string; + audioSampleRate?: number; +}; const nativeVideoExportWriteRequests = new Map< number, @@ -114,6 +192,93 @@ contextBridge.exposeInMainWorld("electronAPI", { generateWallpaperThumbnail: (filePath: string) => { return ipcRenderer.invoke("generate-wallpaper-thumbnail", filePath); }, + probeNativeVideoMetadata: (filePath: string) => { + return ipcRenderer.invoke("probe-native-video-metadata", filePath) as Promise<{ + success: boolean; + metadata?: NativeVideoMetadataProbe; + error?: string; + }>; + }, + nativeStaticLayoutExport: (options: { + sessionId?: string; + inputPath: string; + width: number; + height: number; + frameRate: number; + bitrate: number; + encodingMode: "fast" | "balanced" | "quality"; + durationSec: number; + contentWidth: number; + contentHeight: number; + offsetX: number; + offsetY: number; + backgroundColor: string; + backgroundImagePath?: string | null; + borderRadius?: number; + shadowIntensity?: number; + webcamInputPath?: string | null; + webcamLeft?: number; + webcamTop?: number; + webcamSize?: number; + webcamRadius?: number; + webcamShadowIntensity?: number; + webcamMirror?: boolean; + webcamTimeOffsetMs?: number; + cursorTelemetry?: Array<{ + timeMs: number; + cx: number; + cy: number; + cursorTypeIndex?: number; + bounceScale?: number; + }>; + cursorSize?: number; + cursorAtlasPngDataUrl?: string | null; + cursorAtlasEntries?: Array<{ + index: number; + x: number; + y: number; + width: number; + height: number; + anchorX: number; + anchorY: number; + aspectRatio: number; + }>; + zoomTelemetry?: Array<{ timeMs: number; scale: number; x: number; y: number }>; + chunkDurationSec?: number; + experimentalWindowsGpuCompositor?: boolean; + audioOptions?: { + audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; + audioSourcePath?: string | null; + audioSourceSampleRate?: number; + outputDurationSec?: number; + trimSegments?: Array<{ startMs: number; endMs: number }>; + editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; + editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; + editedAudioData?: ArrayBuffer; + editedAudioMimeType?: string | null; + }; + }) => { + return ipcRenderer.invoke("native-static-layout-export", options) as Promise<{ + success: boolean; + tempPath?: string; + encoderName?: string; + error?: string; + metrics?: NativeStaticLayoutMetrics; + }>; + }, + nativeStaticLayoutExportCancel: (sessionId: string) => { + return ipcRenderer.invoke("native-static-layout-export-cancel", sessionId) as Promise<{ + success: boolean; + }>; + }, + onNativeStaticLayoutExportProgress: ( + callback: (progress: NativeStaticLayoutProgress) => void, + ) => { + const listener = (_event: Electron.IpcRendererEvent, payload: NativeStaticLayoutProgress) => + callback(payload); + ipcRenderer.on("native-static-layout-export-progress", listener); + return () => ipcRenderer.removeListener("native-static-layout-export-progress", listener); + }, nativeVideoExportStart: (options: { width: number; height: number; @@ -147,6 +312,7 @@ contextBridge.exposeInMainWorld("electronAPI", { audioMode?: "none" | "copy-source" | "trim-source" | "edited-track"; audioSourcePath?: string | null; audioSourceSampleRate?: number; + outputDurationSec?: number; trimSegments?: Array<{ startMs: number; endMs: number }>; editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback"; editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>; diff --git a/electron/windows.ts b/electron/windows.ts index 6f51f742..30cf610d 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -48,6 +48,13 @@ function getEditorWindowQuery(): Record { windowType: "editor", }; + if (process.env.RECORDLY_DEV_OPEN_RECORDING_INPUT) { + query.devOpenInput = process.env.RECORDLY_DEV_OPEN_RECORDING_INPUT; + } + if (process.env.RECORDLY_DEV_OPEN_RECORDING_WEBCAM) { + query.devOpenWebcam = process.env.RECORDLY_DEV_OPEN_RECORDING_WEBCAM; + } + if (process.env.RECORDLY_SMOKE_EXPORT === "1") { query.smokeExport = "1"; if (process.env.RECORDLY_SMOKE_EXPORT_INPUT) { @@ -80,6 +87,9 @@ function getEditorWindowQuery(): Record { if (process.env.RECORDLY_SMOKE_EXPORT_BACKEND) { query.smokeBackendPreference = process.env.RECORDLY_SMOKE_EXPORT_BACKEND; } + if (process.env.RECORDLY_SMOKE_EXPORT_RENDER_BACKEND) { + query.smokeRenderBackend = process.env.RECORDLY_SMOKE_EXPORT_RENDER_BACKEND; + } if (process.env.RECORDLY_SMOKE_EXPORT_MAX_ENCODE_QUEUE) { query.smokeMaxEncodeQueue = process.env.RECORDLY_SMOKE_EXPORT_MAX_ENCODE_QUEUE; } diff --git a/package.json b/package.json index dc1c95b9..aee7e919 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,8 @@ "rebuild:native": "node ./node_modules/@electron/rebuild/lib/cli.js --force --only uiohook-napi", "build:native-helpers": "node scripts/build-native-helpers.mjs", "build:whisper-runtime": "node scripts/build-whisper-runtime.mjs", - "build:platform-native-helpers": "npm run build:native-helpers && npm run build:windows-capture && npm run build:cursor-monitor && npm run build:whisper-runtime", + "build:platform-native-helpers": "npm run build:native-helpers && npm run build:windows-capture && npm run build:windows-gpu-export && npm run build:cursor-monitor && npm run build:whisper-runtime", + "build:windows-gpu-export": "node scripts/build-windows-gpu-export.mjs", "build:windows-capture": "node scripts/build-windows-capture.mjs", "build:cursor-monitor": "node scripts/build-cursor-monitor.mjs", "build:mac": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && electron-builder --mac", diff --git a/scripts/benchmark-export-queues.mjs b/scripts/benchmark-export-queues.mjs index 92bd72ee..7344ebe2 100644 --- a/scripts/benchmark-export-queues.mjs +++ b/scripts/benchmark-export-queues.mjs @@ -11,7 +11,7 @@ import ffmpegStatic from "ffmpeg-static"; const execFileAsync = promisify(execFile); const __dirname = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(__dirname, ".."); -const mainEntry = path.join(repoRoot, "dist-electron", "main.js"); +const mainEntry = path.join(repoRoot, "dist-electron", "main.cjs"); const rendererEntry = path.join(repoRoot, "dist", "index.html"); const width = parseEvenInteger(process.env.RECORDLY_BENCH_EXPORT_WIDTH ?? "1280", "Width"); @@ -28,9 +28,13 @@ const timeoutMs = parsePositiveInteger( const runsPerVariant = parsePositiveInteger(process.env.RECORDLY_BENCH_EXPORT_RUNS ?? "2", "Runs"); const useNativeExport = process.env.RECORDLY_BENCH_EXPORT_USE_NATIVE === "1"; const useWebcamOverlay = process.env.RECORDLY_BENCH_EXPORT_ENABLE_WEBCAM === "1"; +const providedInputPath = process.env.RECORDLY_BENCH_EXPORT_INPUT ?? null; +const providedWebcamInputPath = process.env.RECORDLY_BENCH_EXPORT_WEBCAM_INPUT ?? null; +const keepTempArtifacts = process.env.RECORDLY_BENCH_EXPORT_KEEP_TEMP === "1"; const exportEncodingMode = parseExportEncodingMode( process.env.RECORDLY_BENCH_EXPORT_ENCODING_MODE ?? null, ); +const exportQuality = parseExportQuality(process.env.RECORDLY_BENCH_EXPORT_QUALITY ?? null); const exportShadowIntensity = parseExportShadowIntensity( process.env.RECORDLY_BENCH_EXPORT_SHADOW_INTENSITY ?? null, ); @@ -49,6 +53,9 @@ const webcamSize = parseExportWebcamSize(process.env.RECORDLY_BENCH_EXPORT_WEBCA const MODERN_BACKEND_SWEEP = ["auto", "webcodecs", "breeze"]; const exportPipeline = parseExportPipeline(process.env.RECORDLY_BENCH_EXPORT_PIPELINE ?? null); const exportBackend = parseExportBackend(process.env.RECORDLY_BENCH_EXPORT_BACKEND ?? null); +const exportRenderBackend = parseRenderBackend( + process.env.RECORDLY_BENCH_EXPORT_RENDER_BACKEND ?? null, +); const exportBackendList = parseExportBackendList( process.env.RECORDLY_BENCH_EXPORT_BACKENDS ?? null, ); @@ -113,6 +120,18 @@ function parseExportBackend(rawValue) { throw new Error("RECORDLY_BENCH_EXPORT_BACKEND must be 'auto', 'webcodecs', or 'breeze'"); } +function parseRenderBackend(rawValue) { + if (rawValue === null || rawValue === "") { + return null; + } + + if (rawValue === "webgl" || rawValue === "webgpu") { + return rawValue; + } + + throw new Error("RECORDLY_BENCH_EXPORT_RENDER_BACKEND must be 'webgl' or 'webgpu'"); +} + function parseExportBackendList(rawValue) { if (rawValue === null || rawValue === "") { return null; @@ -177,6 +196,18 @@ function parseExportEncodingMode(rawValue) { throw new Error("RECORDLY_BENCH_EXPORT_ENCODING_MODE must be 'fast', 'balanced', or 'quality'"); } +function parseExportQuality(rawValue) { + if (rawValue === null || rawValue === "") { + return null; + } + + if (rawValue === "medium" || rawValue === "good" || rawValue === "high" || rawValue === "source") { + return rawValue; + } + + throw new Error("RECORDLY_BENCH_EXPORT_QUALITY must be 'medium', 'good', 'high', or 'source'"); +} + function parseExportShadowIntensity(rawValue) { if (rawValue === null || rawValue === "") { return null; @@ -477,8 +508,10 @@ function buildRequestedConfigRows(benchmarkRequests) { { key: "Runs per variant", value: runsPerVariant }, { key: "Pipeline", value: exportPipeline ?? "default" }, { key: "Requested backends", value: benchmarkRequests.map((request) => request.label) }, + { key: "Render backend", value: exportRenderBackend ?? "default" }, { key: "Backend sweep", value: formatBoolean(benchmarkRequests.length > 1) }, { key: "Encoding mode", value: exportEncodingMode ?? "default" }, + { key: "Quality", value: exportQuality ?? "default" }, { key: "Shadow intensity", value: exportShadowIntensity ?? "default" }, { key: "Webcam enabled", value: formatBoolean(useWebcamOverlay) }, { key: "Experimental native override", value: formatBoolean(useNativeExport) }, @@ -665,6 +698,7 @@ async function runVariant( ...(exportEncodingMode ? { RECORDLY_SMOKE_EXPORT_ENCODING_MODE: exportEncodingMode } : {}), + ...(exportQuality ? { RECORDLY_SMOKE_EXPORT_QUALITY: exportQuality } : {}), ...(exportShadowIntensity !== null ? { RECORDLY_SMOKE_EXPORT_SHADOW_INTENSITY: String(exportShadowIntensity) } : {}), @@ -681,6 +715,9 @@ async function runVariant( ...(benchmarkRequest.backend ? { RECORDLY_SMOKE_EXPORT_BACKEND: benchmarkRequest.backend } : {}), + ...(exportRenderBackend + ? { RECORDLY_SMOKE_EXPORT_RENDER_BACKEND: exportRenderBackend } + : {}), ...(typeof variant.maxEncodeQueue === "number" ? { RECORDLY_SMOKE_EXPORT_MAX_ENCODE_QUEUE: String(variant.maxEncodeQueue) } : {}), @@ -869,28 +906,45 @@ async function main() { requestedPipeline: exportPipeline, requestedBackend: exportBackend, requestedBackends: benchmarkRequests.map((request) => request.label), + requestedRenderBackend: exportRenderBackend, backendSweepEnabled: benchmarkRequests.length > 1, requestedEncodingMode: exportEncodingMode, + requestedQuality: exportQuality, requestedShadowIntensity: exportShadowIntensity, webcamEnabled: useWebcamOverlay, + providedInput: providedInputPath, + providedWebcamInput: providedWebcamInputPath, + keepTempArtifacts, requestedWebcamShadowIntensity: webcamShadowIntensity, requestedWebcamSize: webcamSize, }), ); printRequestedConfigTable(benchmarkRequests); - console.log(`[benchmark-export-queues] Generating fixture video: ${inputPath}`); - await createFixtureVideo(ffmpegStatic, inputPath); + if (providedInputPath) { + console.log(`[benchmark-export-queues] Using provided input video: ${providedInputPath}`); + await fs.copyFile(providedInputPath, inputPath); + } else { + console.log(`[benchmark-export-queues] Generating fixture video: ${inputPath}`); + await createFixtureVideo(ffmpegStatic, inputPath); + } if (webcamInputPath) { - console.log( - `[benchmark-export-queues] Generating webcam fixture video: ${webcamInputPath}`, - ); - await createFixtureVideo(ffmpegStatic, webcamInputPath, { - fixtureWidth: webcamWidth, - fixtureHeight: webcamHeight, - includeAudio: false, - videoFilter: `testsrc=size=${webcamWidth}x${webcamHeight}:rate=${frameRate}`, - }); + if (providedWebcamInputPath) { + console.log( + `[benchmark-export-queues] Using provided webcam video: ${providedWebcamInputPath}`, + ); + await fs.copyFile(providedWebcamInputPath, webcamInputPath); + } else { + console.log( + `[benchmark-export-queues] Generating webcam fixture video: ${webcamInputPath}`, + ); + await createFixtureVideo(ffmpegStatic, webcamInputPath, { + fixtureWidth: webcamWidth, + fixtureHeight: webcamHeight, + includeAudio: false, + videoFilter: `testsrc=size=${webcamWidth}x${webcamHeight}:rate=${frameRate}`, + }); + } } const benchmarkResults = []; @@ -968,7 +1022,11 @@ async function main() { ); } } finally { - await fs.rm(tempDir, { recursive: true, force: true }); + if (keepTempArtifacts) { + console.log(`[benchmark-export-queues] Preserved temp artifacts: ${tempDir}`); + } else { + await fs.rm(tempDir, { recursive: true, force: true }); + } } } diff --git a/scripts/build-windows-gpu-export.mjs b/scripts/build-windows-gpu-export.mjs new file mode 100644 index 00000000..d981bfa6 --- /dev/null +++ b/scripts/build-windows-gpu-export.mjs @@ -0,0 +1,166 @@ +import { execSync } from "node:child_process"; +import { copyFileSync, existsSync, mkdirSync, rmSync } from "node:fs"; +import path from "node:path"; + +import { + formatNativeHelperManifestWarning, + updateNativeHelperManifest, + verifyNativeHelperManifest, +} from "./native-helper-manifest.mjs"; + +const projectRoot = process.cwd(); +const sourceDir = path.join(projectRoot, "electron", "native", "gpu-export-probe"); +const buildDir = path.join(sourceDir, "build"); +const bundledDir = path.join( + projectRoot, + "electron", + "native", + "bin", + process.arch === "arm64" ? "win32-arm64" : "win32-x64", +); +const bundledExePath = path.join(bundledDir, "recordly-gpu-export.exe"); +const helperId = "recordly-gpu-export"; +const generatorArch = process.arch === "arm64" ? "ARM64" : "x64"; + +if (process.platform !== "win32") { + console.log("[build-windows-gpu-export] Skipping Windows GPU export helper build."); + process.exit(0); +} + +if (!existsSync(path.join(sourceDir, "CMakeLists.txt"))) { + console.error("[build-windows-gpu-export] CMakeLists.txt not found at", sourceDir); + process.exit(1); +} + +function findCmake() { + try { + execSync("cmake --version", { stdio: "pipe" }); + return "cmake"; + } catch { + // Continue probing common Windows install locations. + } + + const standaloneCmakePaths = [ + path.join("C:", "Program Files", "CMake", "bin", "cmake.exe"), + path.join("C:", "Program Files (x86)", "CMake", "bin", "cmake.exe"), + ]; + for (const cmakePath of standaloneCmakePaths) { + if (existsSync(cmakePath)) { + return `"${cmakePath}"`; + } + } + + const vsRoots = [ + path.join("C:", "Program Files", "Microsoft Visual Studio"), + path.join("C:", "Program Files (x86)", "Microsoft Visual Studio"), + ]; + const vsEditions = ["Community", "Professional", "Enterprise", "BuildTools"]; + const vsVersions = ["2022", "2019"]; + for (const root of vsRoots) { + for (const version of vsVersions) { + for (const edition of vsEditions) { + const cmakePath = path.join( + root, + version, + edition, + "Common7", + "IDE", + "CommonExtensions", + "Microsoft", + "CMake", + "CMake", + "bin", + "cmake.exe", + ); + if (existsSync(cmakePath)) { + return `"${cmakePath}"`; + } + } + } + } + + return null; +} + +const cmake = findCmake(); +if (!cmake) { + if (existsSync(bundledExePath)) { + const verification = verifyNativeHelperManifest({ + projectRoot, + helperId, + sourceDir, + binaryPath: bundledExePath, + binaryName: "recordly-gpu-export.exe", + }); + if (!verification.ok) { + console.warn(formatNativeHelperManifestWarning("build-windows-gpu-export", verification)); + } + console.log(`[build-windows-gpu-export] Using bundled helper: ${bundledExePath}`); + process.exit(0); + } + + console.error( + "[build-windows-gpu-export] CMake not found. Install Visual Studio with C++ CMake tools or standalone CMake.", + ); + process.exit(1); +} + +mkdirSync(buildDir, { recursive: true }); + +function clearCmakeCache() { + rmSync(path.join(buildDir, "CMakeCache.txt"), { force: true }); + rmSync(path.join(buildDir, "CMakeFiles"), { recursive: true, force: true }); +} + +console.log("[build-windows-gpu-export] Configuring CMake..."); +try { + clearCmakeCache(); + execSync(`${cmake} .. -G "Visual Studio 17 2022" -A ${generatorArch}`, { + cwd: buildDir, + stdio: "inherit", + timeout: 120000, + }); +} catch { + console.log("[build-windows-gpu-export] VS 2022 generator not found, trying VS 2019..."); + try { + clearCmakeCache(); + execSync(`${cmake} .. -G "Visual Studio 16 2019" -A ${generatorArch}`, { + cwd: buildDir, + stdio: "inherit", + timeout: 120000, + }); + } catch (error) { + console.error("[build-windows-gpu-export] CMake configure failed:", error.message); + process.exit(1); + } +} + +console.log("[build-windows-gpu-export] Building Windows GPU export helper..."); +try { + execSync(`${cmake} --build . --config Release`, { + cwd: buildDir, + stdio: "inherit", + timeout: 300000, + }); +} catch (error) { + console.error("[build-windows-gpu-export] Build failed:", error.message); + process.exit(1); +} + +const exePath = path.join(buildDir, "Release", "gpu-export-probe.exe"); +if (!existsSync(exePath)) { + console.error("[build-windows-gpu-export] Expected exe not found at", exePath); + process.exit(1); +} + +mkdirSync(bundledDir, { recursive: true }); +copyFileSync(exePath, bundledExePath); +console.log(`[build-windows-gpu-export] Staged bundled helper: ${bundledExePath}`); +const manifestPath = updateNativeHelperManifest({ + projectRoot, + helperId, + sourceDir, + binaryPath: bundledExePath, + binaryName: "recordly-gpu-export.exe", +}); +console.log(`[build-windows-gpu-export] Updated helper manifest: ${manifestPath}`); diff --git a/scripts/smoke-packaged-binaries.mjs b/scripts/smoke-packaged-binaries.mjs index edefeebc..745f787b 100644 --- a/scripts/smoke-packaged-binaries.mjs +++ b/scripts/smoke-packaged-binaries.mjs @@ -166,6 +166,11 @@ function getExpectedNativeHelperFiles(archTag) { label: "Windows cursor monitor helper", executable: true, }, + { + name: "recordly-gpu-export.exe", + label: "Windows GPU export helper", + executable: true, + }, { name: "helpers-manifest.json", label: "Windows helper manifest" }, { name: "whisper-cli.exe", label: "Whisper CLI runtime", executable: true }, { name: "whisper-runtime.json", label: "Whisper runtime manifest" }, diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 13c3440a..9f5b6d15 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -54,6 +54,7 @@ import { type ExportPipelineModel, type ExportProgress, type ExportQuality, + type ExportRenderBackend, type ExportSettings, FrameRenderer, GIF_SIZE_PRESETS, @@ -66,6 +67,7 @@ import { type SupportedMp4Dimensions, VideoExporter, } from "@/lib/exporter"; +import { getMp4ExportBitrate, getSourceQualityBitrate } from "@/lib/exporter/exportBitrate"; import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource"; import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback"; import { @@ -230,6 +232,7 @@ type SmokeExportConfig = { webcamSize?: number; pipelineModel?: ExportPipelineModel; backendPreference?: ExportBackendPreference; + renderBackend?: ExportRenderBackend; maxEncodeQueue?: number; maxDecodeQueue?: number; maxPendingFrames?: number; @@ -245,6 +248,11 @@ type SaveProjectOptions = { captureThumbnail?: boolean; }; +type DevOpenRecordingConfig = { + inputPath: string | null; + webcamInputPath: string | null; +}; + async function writeSmokeExportReport( outputPath: string | null, report: Record, @@ -268,22 +276,11 @@ async function writeSmokeExportReport( } } +const SMOKE_EXPORT_READY_TIMEOUT_MS = 30_000; const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30; const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error"; const PROJECT_AUTOSAVE_DELAY_MS = 1000; -function getEncodingModeBitrateMultiplier(encodingMode: ExportEncodingMode): number { - switch (encodingMode) { - case "fast": - return 0.1; - case "quality": - return 0.9; - case "balanced": - default: - return 0.5; - } -} - function summarizeErrorMessage(message: string): string { const firstLine = message .split(/\r?\n/) @@ -328,6 +325,10 @@ function parseSmokeExportFps(value: string | null): ExportMp4FrameRate | undefin return isValidMp4FrameRate(parsed) ? parsed : undefined; } +function parseSmokeRenderBackend(value: string | null): ExportRenderBackend | undefined { + return value === "webgl" || value === "webgpu" ? value : undefined; +} + function getSmokeExportConfig(search: string): SmokeExportConfig { const params = new URLSearchParams(search); const enabled = params.get("smokeExport") === "1"; @@ -369,6 +370,7 @@ function getSmokeExportConfig(search: string): SmokeExportConfig { : enabled && params.get("smokeBackendPreference") === "breeze" ? "breeze" : undefined, + renderBackend: enabled ? parseSmokeRenderBackend(params.get("smokeRenderBackend")) : undefined, maxEncodeQueue: enabled ? parseSmokeExportNumber(params.get("smokeMaxEncodeQueue")) : undefined, @@ -384,6 +386,14 @@ function getSmokeExportConfig(search: string): SmokeExportConfig { }; } +function getDevOpenRecordingConfig(search: string): DevOpenRecordingConfig { + const params = new URLSearchParams(search); + return { + inputPath: params.get("devOpenInput"), + webcamInputPath: params.get("devOpenWebcam"), + }; +} + function isComparableObject(value: unknown): value is Record { return typeof value === "object" && value !== null; } @@ -496,17 +506,6 @@ function calculateMp4ExportDimensions( }; } -function getSourceQualityBitrate(width: number, height: number): number { - const totalPixels = width * height; - if (totalPixels > 2560 * 1440) { - return 80_000_000; - } - if (totalPixels > 1920 * 1080) { - return 50_000_000; - } - return 30_000_000; -} - function getErrorMessage(error: unknown): string { if (error instanceof Error) { return error.message; @@ -525,6 +524,13 @@ export default function VideoEditor() { () => getSmokeExportConfig(typeof window === "undefined" ? "" : window.location.search), [], ); + const devOpenRecordingConfig = useMemo( + () => + getDevOpenRecordingConfig( + typeof window === "undefined" ? "" : window.location.search, + ), + [], + ); const [appPlatform, setAppPlatform] = useState( typeof navigator !== "undefined" && /Mac/i.test(navigator.platform) ? "darwin" : "", ); @@ -721,6 +727,7 @@ export default function VideoEditor() { const smokeExportStartedRef = useRef(false); const projectAutosaveTimeoutRef = useRef(null); const projectSaveQueueRef = useRef>(Promise.resolve()); + const smokeExportReadyStateRef = useRef>({}); const [historyVersion, setHistoryVersion] = useState(0); const timelineRef = useRef(null); @@ -2161,6 +2168,29 @@ export default function VideoEditor() { return; } + if (!smokeExportConfig.enabled && devOpenRecordingConfig.inputPath) { + const sourcePath = fromFileUrl(devOpenRecordingConfig.inputPath); + const sourceVideoUrl = await resolveVideoUrl(sourcePath); + const webcamSourcePath = devOpenRecordingConfig.webcamInputPath + ? fromFileUrl(devOpenRecordingConfig.webcamInputPath) + : null; + setVideoSourcePath(sourcePath); + setVideoPath(sourceVideoUrl); + setCurrentProjectPath(null); + setLastSavedSnapshot(null); + pendingFreshRecordingAutoZoomPathRef.current = autoApplyFreshRecordingAutoZooms + ? sourceVideoUrl + : null; + setWebcam((prev) => ({ + ...prev, + enabled: Boolean(webcamSourcePath), + sourcePath: webcamSourcePath, + timeOffsetMs: DEFAULT_WEBCAM_TIME_OFFSET_MS, + })); + setError(null); + return; + } + if (smokeExportConfig.enabled) { if (!smokeExportConfig.inputPath) { setError("Smoke export input path is missing."); @@ -2275,6 +2305,8 @@ export default function VideoEditor() { }, [ applyLoadedProject, autoApplyFreshRecordingAutoZooms, + devOpenRecordingConfig.inputPath, + devOpenRecordingConfig.webcamInputPath, initialEditorPreferences, smokeExportConfig.enabled, smokeExportConfig.inputPath, @@ -4429,9 +4461,14 @@ export default function VideoEditor() { ? (smokeExportConfig.pipelineModel ?? (smokeExportConfig.useNativeExport ? "modern" : "legacy")) : (settings.pipelineModel ?? exportPipelineModel); + const useExperimentalNativeExport = + pipelineModel === "modern" && + (smokeExportConfig.enabled ? smokeExportConfig.useNativeExport : true); const backendPreference = pipelineModel === "legacy" ? "webcodecs" + : useExperimentalNativeExport + ? "auto" : smokeExportConfig.enabled ? (smokeExportConfig.backendPreference ?? (smokeExportConfig.useNativeExport ? "breeze" : "webcodecs")) @@ -4444,33 +4481,14 @@ export default function VideoEditor() { supportedSourceDimensions.height, quality, ); - let bitrate: number; - - if (quality === "source") { - // Calculate visually lossless bitrate matching screen recording optimization - const totalPixels = exportWidth * exportHeight; - bitrate = 30_000_000; - if (totalPixels > 1920 * 1080 && totalPixels <= 2560 * 1440) { - bitrate = 50_000_000; - } else if (totalPixels > 2560 * 1440) { - bitrate = 80_000_000; - } - } else { - // Adjust bitrate for lower resolutions - const totalPixels = exportWidth * exportHeight; - if (totalPixels <= 1280 * 720) { - bitrate = 10_000_000; - } else if (totalPixels <= 1920 * 1080) { - bitrate = 20_000_000; - } else { - bitrate = 30_000_000; - } - } - - bitrate = Math.max( - 2_000_000, - Math.round(bitrate * getEncodingModeBitrateMultiplier(encodingMode)), - ); + const bitrate = getMp4ExportBitrate({ + width: exportWidth, + height: exportHeight, + frameRate: selectedMp4FrameRate, + quality, + encodingMode, + useModernNativeStaticLayout: useExperimentalNativeExport, + }); const exporterConfig = { videoUrl: videoPath, @@ -4481,7 +4499,8 @@ export default function VideoEditor() { codec: DEFAULT_MP4_CODEC, encodingMode, preferredEncoderPath: supportedSourceDimensions.encoderPath, - experimentalNativeExport: smokeExportConfig.useNativeExport, + preferredRenderBackend: smokeExportConfig.renderBackend, + experimentalNativeExport: useExperimentalNativeExport, maxEncodeQueue: smokeExportConfig.maxEncodeQueue, maxDecodeQueue: smokeExportConfig.maxDecodeQueue, maxPendingFrames: smokeExportConfig.maxPendingFrames, @@ -4787,6 +4806,7 @@ export default function VideoEditor() { remountPreview, showExportSuccessToast, smokeExportConfig.backendPreference, + smokeExportConfig.renderBackend, smokeExportConfig.enabled, smokeExportConfig.useNativeExport, smokeExportConfig.maxDecodeQueue, @@ -4803,6 +4823,48 @@ export default function VideoEditor() { ], ); + useEffect(() => { + smokeExportReadyStateRef.current = { + cursorTelemetrySourcePath, + duration, + hasVideoPath: Boolean(videoPath), + isPreviewReady, + loading, + projectPath: smokeExportConfig.projectPath ?? null, + videoSourcePath, + }; + }, [ + cursorTelemetrySourcePath, + duration, + isPreviewReady, + loading, + smokeExportConfig.projectPath, + videoPath, + videoSourcePath, + ]); + + useEffect(() => { + if (!smokeExportConfig.enabled) { + return; + } + + const timeoutId = window.setTimeout(() => { + if (smokeExportStartedRef.current) { + return; + } + + smokeExportStartedRef.current = true; + void writeSmokeExportReport(smokeExportConfig.outputPath, { + success: false, + phase: "ready", + error: `Smoke export did not become ready within ${SMOKE_EXPORT_READY_TIMEOUT_MS}ms.`, + readyState: smokeExportReadyStateRef.current, + }).finally(() => window.close()); + }, SMOKE_EXPORT_READY_TIMEOUT_MS); + + return () => window.clearTimeout(timeoutId); + }, [smokeExportConfig.enabled, smokeExportConfig.outputPath]); + useEffect(() => { if (!smokeExportConfig.enabled || smokeExportStartedRef.current) { return; @@ -4811,11 +4873,16 @@ export default function VideoEditor() { if (error) { smokeExportStartedRef.current = true; console.error(`[smoke-export] ${error}`); - window.close(); + void writeSmokeExportReport(smokeExportConfig.outputPath, { + success: false, + phase: "load", + error, + readyState: smokeExportReadyStateRef.current, + }).finally(() => window.close()); return; } - if (!videoPath || loading) { + if (!videoPath || loading || !isPreviewReady || duration <= 0) { return; } @@ -4841,9 +4908,12 @@ export default function VideoEditor() { cursorTelemetrySourcePath, error, handleExport, + isPreviewReady, loading, + duration, smokeExportConfig.enabled, smokeExportConfig.encodingMode, + smokeExportConfig.outputPath, smokeExportConfig.projectPath, videoPath, videoSourcePath, @@ -5055,6 +5125,8 @@ export default function VideoEditor() { exportFormat === "mp4" && exportPipelineModel === "modern" && (isExporting || exportProgress !== null); + const shouldSuspendPreviewRendering = + isExporting && exportFormat === "mp4" && exportPipelineModel === "modern"; const isLegacyExportInProgress = exportFormat === "mp4" && exportPipelineModel === "legacy" && @@ -5095,6 +5167,9 @@ export default function VideoEditor() { return encoderName ? `${pathLabel} (${encoderName})` : pathLabel; }, [exportProgress]); + const exportNativeSkipLabel = exportProgress?.nativeStaticLayoutSkipReason + ? `Native skipped: ${exportProgress.nativeStaticLayoutSkipReason}` + : null; const exportPercentLabel = exportProgress ? isExportSaving ? t("editor.exportStatus.saving", "Opening save dialog...") @@ -5463,6 +5538,11 @@ export default function VideoEditor() { Path: {exportRuntimeLabel}

) : null} + {exportNativeSkipLabel ? ( +

+ {exportNativeSkipLabel} +

+ ) : null} ) : exportError ? (
@@ -5960,6 +6040,7 @@ export default function VideoEditor() { } cursorSway={cursorSway} volume={shouldMutePreviewVideo ? 0 : previewVolume} + suspendRendering={shouldSuspendPreviewRendering} />
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 99362abb..3a618d18 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -159,6 +159,56 @@ function createPlaybackAnimationState(): PlaybackAnimationState { }; } +type PixiPreviewBackend = "webgpu" | "webgl"; +type PixiRendererAttempt = { + backend: PixiPreviewBackend; + message: string; +}; +const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000; + +function isCanvasRenderer(application: Application): boolean { + const rendererName = application?.renderer?.constructor?.name?.toLowerCase(); + return Boolean(rendererName && (rendererName.includes("canvasrenderer") || rendererName.includes("canvas"))); +} + +function toRendererErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error ?? "Unknown renderer init error"); +} + +function isRendererUnavailableError(error: unknown): boolean { + const message = toRendererErrorMessage(error).toLowerCase(); + return message.includes("canvasrenderer is not yet implemented") || message.includes("no available renderer"); +} + +function summarizeRendererAttempts(attempts: readonly PixiRendererAttempt[]): string { + const details = attempts.map((attempt) => `${attempt.backend}: ${attempt.message}`).join(" | "); + return `No supported Pixi preview renderer was available. Attempted: ${details}`; +} + +type PixiInitOptions = Parameters[0]; + +async function initApplicationWithTimeout( + app: Application, + options: PixiInitOptions, + backend: PixiPreviewBackend, +): Promise { + const timeoutErrorMessage = `Initialization timed out after ${PIXI_RENDERER_INIT_TIMEOUT_MS}ms for ${backend} renderer`; + let timeoutId: ReturnType | undefined; + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject(new Error(timeoutErrorMessage)); + }, PIXI_RENDERER_INIT_TIMEOUT_MS); + }); + + try { + await Promise.race([app.init(options), timeoutPromise]); + } finally { + if (timeoutId !== undefined) { + clearTimeout(timeoutId); + } + } +} + function getCursorPositionAtTime( telemetry: CursorTelemetryPoint[], timeMs: number, @@ -273,6 +323,7 @@ interface VideoPlaybackProps { cursorClickBounceDuration?: number; cursorSway?: number; volume?: number; + suspendRendering?: boolean; } type CaptionEditSession = { @@ -349,6 +400,7 @@ const VideoPlayback = forwardRef( cursorClickBounceDuration = DEFAULT_CURSOR_CLICK_BOUNCE_DURATION, cursorSway = DEFAULT_CURSOR_SWAY, volume = 1, + suspendRendering = false, }, ref, ) => { @@ -364,6 +416,10 @@ const VideoPlayback = forwardRef( const timeUpdateAnimationRef = useRef(null); const [pixiReady, setPixiReady] = useState(false); const [videoReady, setVideoReady] = useState(false); + const [pixiRendererError, setPixiRendererError] = useState(null); + const [pixiRendererBackend, setPixiRendererBackend] = useState( + null, + ); const overlayRef = useRef(null); const focusIndicatorRef = useRef(null); const webcamVideoRef = useRef(null); @@ -404,6 +460,7 @@ const VideoPlayback = forwardRef( const frameContainerRef = useRef(null); const frameIdRef = useRef(frame); const isPlayingRef = useRef(isPlaying); + const suspendRenderingRef = useRef(suspendRendering); const isSeekingRef = useRef(false); const allowPlaybackRef = useRef(false); const lockedVideoDimensionsRef = useRef<{ @@ -451,6 +508,75 @@ const VideoPlayback = forwardRef( createCursorFollowCameraState(), ); + const initializePixiRenderer = useCallback( + async (container: HTMLDivElement): Promise<{ + app: Application; + backend: PixiPreviewBackend; + }> => { + const backendOrder: PixiPreviewBackend[] = ["webgl", "webgpu"]; + const attempts: PixiRendererAttempt[] = []; + + for (const backend of backendOrder) { + if ( + backend === "webgpu" && + !(typeof navigator !== "undefined" && "gpu" in navigator) + ) { + attempts.push({ + backend, + message: "WebGPU runtime is unavailable in this browser.", + }); + continue; + } + + const rendererApp = new Application(); + const initStarted = typeof performance === "undefined" ? Date.now() : performance.now(); + try { + await initApplicationWithTimeout( + rendererApp, + { + width: container.clientWidth, + height: container.clientHeight, + backgroundAlpha: 0, + antialias: true, + failIfMajorPerformanceCaveat: false, + resolution: window.devicePixelRatio || 1, + autoDensity: true, + preference: backend, + autoStart: true, + sharedTicker: false, + }, + backend, + ); + const elapsed = Math.round( + (typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted, + ); + if (isCanvasRenderer(rendererApp)) { + throw new Error( + `Renderer initialized with unsupported fallback backend after ${elapsed}ms: ${rendererApp.renderer.constructor?.name ?? "unknown"}`, + ); + } + return { app: rendererApp, backend }; + } catch (error) { + const elapsed = Math.round( + (typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted, + ); + attempts.push({ backend, message: `${toRendererErrorMessage(error)} (after ${elapsed}ms)` }); + const statusMessage = isRendererUnavailableError(error) + ? "renderer backend unavailable in this runtime" + : "renderer init failed"; + console.warn( + `[VideoPlayback] Failed to init ${backend} renderer (${statusMessage}) after ${elapsed}ms; trying fallback.`, + error, + ); + rendererApp.destroy(true); + } + } + + throw new Error(summarizeRendererAttempts(attempts)); + }, + [], + ); + const activeCaptionLayout = useMemo(() => { if ( !autoCaptionSettings?.enabled || @@ -864,7 +990,7 @@ const VideoPlayback = forwardRef( const maskRect = result.maskRect; const insets = frameData.screenInsets; if (insets) { - // Frame is larger than screen area — compute full frame size from insets + // Frame is larger than screen area - compute full frame size from insets const screenW = maskRect.width; const screenH = maskRect.height; const frameW = screenW / (1 - insets.left - insets.right); @@ -1171,6 +1297,54 @@ const VideoPlayback = forwardRef( } }, [isPlaying]); + useEffect(() => { + suspendRenderingRef.current = suspendRendering; + const app = appRef.current; + if (!app?.ticker) { + return; + } + + if (suspendRendering) { + bgVideoRef.current?.pause(); + webcamVideoRef.current?.pause(); + layoutVideoContentRef.current?.(); + const videoTextureSource = videoSpriteRef.current?.texture?.source as + | { update?: () => void } + | undefined; + videoTextureSource?.update?.(); + app.render(); + return; + } + + app.ticker.start(); + const video = videoRef.current; + if (video) { + const targetTime = clampMediaTimeToDuration( + currentTimeRef.current / 1000, + Number.isFinite(video.duration) ? video.duration : null, + ); + if (Math.abs(video.currentTime - targetTime) > 0.001) { + try { + video.currentTime = targetTime; + } catch { + // no-op + } + } + } + layoutVideoContentRef.current?.(); + const videoTextureSource = videoSpriteRef.current?.texture?.source as + | { update?: () => void } + | undefined; + videoTextureSource?.update?.(); + requestAnimationFrame(() => { + appRef.current?.render(); + }); + if (isPlayingRef.current) { + bgVideoRef.current?.play().catch(() => undefined); + webcamVideoRef.current?.play().catch(() => undefined); + } + }, [pixiReady, suspendRendering]); + // Keep video wallpapers locked to the same source timestamp as the main clip. useEffect(() => { const bgVideo = bgVideoRef.current; @@ -1519,19 +1693,12 @@ const VideoPlayback = forwardRef( error, ); } + setPixiRendererError(null); + setPixiRendererBackend(null); - app = new Application(); - - await app.init({ - width: container.clientWidth, - height: container.clientHeight, - backgroundAlpha: 0, - antialias: true, - failIfMajorPerformanceCaveat: false, - resolution: window.devicePixelRatio || 1, - autoDensity: true, - preference: "webgl", - }); + const result = await initializePixiRenderer(container); + app = result.app; + setPixiRendererBackend(result.backend); app.ticker.maxFPS = 60; @@ -1557,7 +1724,7 @@ const VideoPlayback = forwardRef( videoContainerRef.current = videoContainer; cameraContainer.addChild(videoContainer); - // Device frame overlay container — sits above video but below cursor + // Device frame overlay container - sits above video but below cursor const frameContainer = new Container(); frameContainerRef.current = frameContainer; cameraContainer.addChild(frameContainer); @@ -1586,7 +1753,12 @@ const VideoPlayback = forwardRef( setPixiReady(true); })().catch((error) => { + const errorMessage = + error instanceof Error + ? error.message + : "Failed to initialize preview renderer"; console.error("Failed to initialize preview renderer:", error); + setPixiRendererError(errorMessage); onError( error instanceof Error ? error.message @@ -1597,6 +1769,8 @@ const VideoPlayback = forwardRef( return () => { mounted = false; setPixiReady(false); + setPixiRendererError(null); + setPixiRendererBackend(null); if (cursorOverlayRef.current) { cursorOverlayRef.current.destroy(); cursorOverlayRef.current = null; @@ -1616,7 +1790,7 @@ const VideoPlayback = forwardRef( cursorContainerRef.current = null; videoSpriteRef.current = null; }; - }, [onError]); + }, [initializePixiRenderer, onError]); useEffect(() => { const video = videoRef.current; @@ -1675,7 +1849,7 @@ const VideoPlayback = forwardRef( blurFilter.resolution = app.renderer.resolution; blurFilter.blur = 0; const motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0); - // Don't attach filters by default — the filter pipeline forces the video + // Don't attach filters by default - the filter pipeline forces the video // through an intermediate RenderTexture at renderer resolution, downsampling // the native video and destroying detail. Filters are attached conditionally // in the ticker only when zoom motion blur is actually active. @@ -1783,6 +1957,10 @@ const VideoPlayback = forwardRef( }; const ticker = () => { + if (suspendRenderingRef.current) { + return; + } + const { region, strength, blendedScale, transition } = findDominantRegion( zoomRegionsRef.current, currentTimeRef.current, @@ -2407,6 +2585,10 @@ const VideoPlayback = forwardRef( : resolvedWallpaperKind === "video" ? {} : { background: resolvedWallpaper || "" }; + const fallbackVideoClassName = pixiRendererError + ? "absolute inset-0 h-full w-full object-cover" + : "pointer-events-none absolute left-0 top-0 h-px w-px opacity-0"; + const hasRendererFallback = Boolean(pixiRendererError); const nativeAspectRatio = (() => { const locked = lockedVideoDimensionsRef.current; @@ -2464,9 +2646,18 @@ const VideoPlayback = forwardRef( filter: showShadow && shadowIntensity > 0 ? `drop-shadow(0 ${shadowIntensity * 12}px ${shadowIntensity * 48}px rgba(0,0,0,${shadowIntensity * 0.7})) drop-shadow(0 ${shadowIntensity * 4}px ${shadowIntensity * 16}px rgba(0,0,0,${shadowIntensity * 0.5})) drop-shadow(0 ${shadowIntensity * 2}px ${shadowIntensity * 8}px rgba(0,0,0,${shadowIntensity * 0.3}))` - : "none", + : "none", }} /> + {hasRendererFallback && ( +
+
+ {`Pixi renderer unavailable on this environment (${pixiRendererBackend ?? "unknown"}).`} +
+ Fallback to 2D native preview so you can continue working while the GPU path is unavailable. +
+
+ )} {/* Canvas overlay for extension cursor effects (drawn via Canvas 2D API) */} (