Add native GPU static layout export path

This commit is contained in:
wiiiii123
2026-05-03 22:43:23 +07:00
parent 816116de26
commit 9c81006e5c
62 changed files with 9819 additions and 328 deletions
+10 -6
View File
@@ -34,9 +34,13 @@ vite.config.js
vite.config.d.ts
# Native capture build artifacts
electron/native/wgc-capture/build/
electron/native/cursor-monitor/build/
# Local debug helpers
tmp-*.ps1
.tmp-*.ps1
electron/native/wgc-capture/build/
electron/native/cursor-monitor/build/
electron/native/gpu-export-probe/build/
electron/native/bin/*/whisper-*
electron/native/bin/*/whisper-runtime.json
# Local debug helpers
tmp-*.ps1
.tmp-*.ps1
gpu-export-probe.mp4
+5 -4
View File
@@ -17,10 +17,11 @@
"output": "release"
},
"files": [
"dist",
"dist-electron",
"electron/native",
"!*.png",
"dist",
"dist-electron",
"electron/native",
"!electron/native/**/build/**",
"!*.png",
"!preview*.png",
"!*.md",
"!README.md",
+181
View File
@@ -82,6 +82,110 @@ interface RendererFfmpegAudioMuxMetrics {
muxedVideoBytes?: number;
}
interface RendererWindowsGpuExportSummary {
success?: boolean;
width?: number;
height?: number;
fps?: number;
seconds?: number;
mediaMs?: number;
frames?: number;
gpuDecodeSurface?: boolean;
webcamOverlay?: boolean;
cursorOverlay?: boolean;
zoomOverlay?: boolean;
surfacePoolSize?: number;
adapterIndex?: number;
adapterVendorId?: number;
adapterDeviceId?: number;
adapterDedicatedVideoMemoryMB?: number;
encoderBackend?: string;
encoderTuningApplied?: boolean;
nvencOutputBytes?: number;
initializeMs?: number;
initCoInitializeMs?: number;
initMfStartupMs?: number;
initD3DDeviceMs?: number;
initSourceReaderMs?: number;
initWebcamReaderMs?: number;
initVideoProcessorMs?: number;
initTexturesMs?: number;
initShaderPipelineMs?: number;
initSinkWriterMs?: number;
totalMs?: number;
readMs?: number;
clearMs?: number;
videoProcessMs?: number;
writeSampleMs?: number;
finalizeMs?: number;
realtimeMultiplier?: number;
}
interface RendererNativeStaticLayoutChunkMetric {
index: number;
startSec: number;
durationSec: number;
backend:
| "cuda-overlay"
| "cuda-scale-cpu-pad"
| "cuda-static-composite"
| "nvidia-cuda-compositor"
| "windows-d3d11-compositor";
elapsedMs: number;
outputBytes: number;
fallbackReason?: string;
windowsGpuSummary?: RendererWindowsGpuExportSummary;
}
interface RendererNativeStaticLayoutMetrics extends RendererFfmpegAudioMuxMetrics {
chunkCount: number;
chunkDurationSec: number;
chunkExecMs: number;
concatExecMs?: number;
staticAssetExecMs?: number;
fallbackChunkCount: number;
videoOnlyBytes?: number;
chunks: RendererNativeStaticLayoutChunkMetric[];
}
interface RendererNativeStaticLayoutProgress {
sessionId?: string;
backend?: RendererNativeStaticLayoutChunkMetric["backend"];
elapsedMs?: number;
averageFps?: number;
instantFps?: number;
intervalMs?: number;
intervalFrames?: number;
intervalDecodeWallMs?: number;
intervalEncodeMs?: number;
intervalPipelineWaitMs?: number;
intervalCompositeMs?: number;
intervalNvencMs?: number;
intervalPacketWriteMs?: number;
intervalWebcamDecodeMs?: number;
intervalWebcamCopyMs?: number;
intervalRoiCompositeFrames?: number;
intervalMonolithicCompositeFrames?: number;
intervalCopyCompositeFrames?: number;
currentFrame: number;
totalFrames: number;
percentage: number;
}
interface RendererNativeVideoMetadataProbe {
width: number;
height: number;
duration: number;
mediaStartTime?: number;
streamStartTime?: number;
streamDuration?: number;
frameRate: number;
codec: string;
hasAudio: boolean;
audioCodec?: string;
audioSampleRate?: number;
}
interface Window {
electronAPI: {
hudOverlaySetIgnoreMouse: (ignore: boolean) => void;
@@ -193,6 +297,82 @@ interface Window {
generateWallpaperThumbnail: (
filePath: string,
) => Promise<{ success: boolean; data?: Uint8Array; error?: string }>;
probeNativeVideoMetadata: (filePath: string) => Promise<{
success: boolean;
metadata?: RendererNativeVideoMetadataProbe;
error?: string;
}>;
nativeStaticLayoutExport: (options: {
sessionId?: string;
inputPath: string;
width: number;
height: number;
frameRate: number;
bitrate: number;
encodingMode: "fast" | "balanced" | "quality";
durationSec: number;
contentWidth: number;
contentHeight: number;
offsetX: number;
offsetY: number;
backgroundColor: string;
backgroundImagePath?: string | null;
borderRadius?: number;
shadowIntensity?: number;
webcamInputPath?: string | null;
webcamLeft?: number;
webcamTop?: number;
webcamSize?: number;
webcamRadius?: number;
webcamShadowIntensity?: number;
webcamMirror?: boolean;
webcamTimeOffsetMs?: number;
cursorTelemetry?: Array<{
timeMs: number;
cx: number;
cy: number;
cursorTypeIndex?: number;
bounceScale?: number;
}>;
cursorSize?: number;
cursorAtlasPngDataUrl?: string | null;
cursorAtlasEntries?: Array<{
index: number;
x: number;
y: number;
width: number;
height: number;
anchorX: number;
anchorY: number;
aspectRatio: number;
}>;
zoomTelemetry?: Array<{ timeMs: number; scale: number; x: number; y: number }>;
chunkDurationSec?: number;
experimentalWindowsGpuCompositor?: boolean;
audioOptions?: {
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
audioSourcePath?: string | null;
audioSourceSampleRate?: number;
outputDurationSec?: number;
trimSegments?: Array<{ startMs: number; endMs: number }>;
editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
editedAudioData?: ArrayBuffer;
editedAudioMimeType?: string | null;
};
}) => Promise<{
success: boolean;
tempPath?: string;
encoderName?: string;
error?: string;
metrics?: RendererNativeStaticLayoutMetrics;
}>;
nativeStaticLayoutExportCancel: (sessionId: string) => Promise<{
success: boolean;
}>;
onNativeStaticLayoutExportProgress: (
callback: (progress: RendererNativeStaticLayoutProgress) => void,
) => () => void;
nativeVideoExportStart: (options: {
width: number;
height: number;
@@ -216,6 +396,7 @@ interface Window {
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
audioSourcePath?: string | null;
audioSourceSampleRate?: number;
outputDurationSec?: number;
trimSegments?: Array<{ startMs: number; endMs: number }>;
editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
+252
View File
@@ -0,0 +1,252 @@
import { describe, expect, it, vi } from "vitest";
vi.mock("electron", () => ({
app: {
getAppPath: () => process.cwd(),
getPath: () => process.env.TEMP ?? process.cwd(),
isPackaged: false,
},
}));
vi.mock("../ffmpeg/binary", () => ({
getFfmpegBinaryPath: () => "ffmpeg",
}));
import {
buildNativeVideoAudioMuxArgs,
normalizeNativeStaticLayoutBackground,
parseFfmpegDurationSeconds,
parseFfmpegFrameRate,
parseNativeVideoMetadataProbeOutput,
parseNvidiaCudaExportSummary,
parseWindowsGpuExportProgressLine,
parseWindowsGpuExportSummary,
} from "./native-video";
describe("normalizeNativeStaticLayoutBackground", () => {
it("falls back to a solid background when the configured image file is missing", async () => {
const normalized = await normalizeNativeStaticLayoutBackground({
inputPath: "input.mp4",
width: 1920,
height: 1080,
frameRate: 30,
bitrate: 8_000_000,
encodingMode: "quality",
durationSec: 10,
contentWidth: 1600,
contentHeight: 900,
offsetX: 160,
offsetY: 90,
backgroundColor: "#101010",
backgroundImagePath: "Z:\\recordly-missing-wallpaper\\midnight-8.jpg",
});
expect(normalized.backgroundImagePath).toBeNull();
expect(normalized.backgroundColor).toBe("#ffffff");
});
});
describe("buildNativeVideoAudioMuxArgs", () => {
it("stream-copies source audio and preserves the requested video duration", () => {
const args = buildNativeVideoAudioMuxArgs("video.mp4", "source.mp4", "out.mp4", {
audioMode: "copy-source",
outputDurationSec: 60,
});
expect(args).toEqual(
expect.arrayContaining([
"-map",
"0:v:0",
"-map",
"1:a:0",
"-c:v",
"copy",
"-c:a",
"copy",
"-t",
"60.000",
]),
);
expect(args).not.toContain("-shortest");
});
it("does not shorten copy-source muxes when no explicit duration is available", () => {
const args = buildNativeVideoAudioMuxArgs("video.mp4", "source.mp4", "out.mp4", {
audioMode: "copy-source",
});
expect(args).toEqual(expect.arrayContaining(["-c:a", "copy"]));
expect(args).not.toContain("-shortest");
});
it("keeps filtered audio on the AAC encode path", () => {
const args = buildNativeVideoAudioMuxArgs("video.mp4", "source.mp4", "out.mp4", {
audioMode: "trim-source",
trimSegments: [{ startMs: 0, endMs: 1_000 }],
outputDurationSec: 1,
});
expect(args).toEqual(expect.arrayContaining(["-filter_complex"]));
expect(args).toEqual(expect.arrayContaining(["-c:a", "aac", "-b:a", "192k"]));
});
});
describe("parseWindowsGpuExportSummary", () => {
it("returns the last JSON summary from helper stdout", () => {
const summary = parseWindowsGpuExportSummary(
[
"initializing",
'{"success":true,"frames":30,"totalMs":1000,"realtimeMultiplier":2}',
"cleanup",
'{"success":true,"frames":60,"surfacePoolSize":12,"readMs":12.5,"videoProcessMs":30,"writeSampleMs":40,"finalizeMs":5,"realtimeMultiplier":4}',
].join("\n"),
);
expect(summary).toEqual({
success: true,
frames: 60,
surfacePoolSize: 12,
readMs: 12.5,
videoProcessMs: 30,
writeSampleMs: 40,
finalizeMs: 5,
realtimeMultiplier: 4,
});
});
it("returns null when helper stdout has no valid JSON summary", () => {
expect(parseWindowsGpuExportSummary("initializing\nnot-json")).toBeNull();
expect(parseWindowsGpuExportSummary("")).toBeNull();
});
});
describe("parseNvidiaCudaExportSummary", () => {
it("parses the pretty JSON summary emitted by the CUDA lab wrapper", () => {
const summary = parseNvidiaCudaExportSummary(
[
"preflight",
JSON.stringify(
{
success: true,
fps: 30,
durationSec: 10,
targetFrames: 300,
timingsMs: { nativeEncode: 920, mux: 45, endToEnd: 1400 },
nativeSummary: { success: true, frames: 300, fps: 326.1 },
},
null,
2,
),
].join("\n"),
);
expect(summary?.success).toBe(true);
expect(summary?.targetFrames).toBe(300);
expect(summary?.timingsMs?.nativeEncode).toBe(920);
expect(summary?.nativeSummary?.fps).toBe(326.1);
});
it("returns null when the wrapper output has no JSON object", () => {
expect(parseNvidiaCudaExportSummary("native probe failed before summary")).toBeNull();
});
});
describe("parseWindowsGpuExportProgressLine", () => {
it("parses bounded helper progress lines", () => {
expect(
parseWindowsGpuExportProgressLine(
'PROGRESS {"currentFrame":30,"totalFrames":60,"percentage":50,"averageFps":240.5,"instantFps":180.25,"intervalMs":166.4,"intervalFrames":30,"intervalEncodeMs":120.2,"intervalPipelineWaitMs":46.2,"intervalMonolithicCompositeFrames":0}',
),
).toEqual({
currentFrame: 30,
totalFrames: 60,
percentage: 50,
averageFps: 240.5,
instantFps: 180.25,
intervalMs: 166.4,
intervalFrames: 30,
intervalEncodeMs: 120.2,
intervalPipelineWaitMs: 46.2,
intervalMonolithicCompositeFrames: 0,
});
});
it("ignores non-progress or malformed helper stderr", () => {
expect(parseWindowsGpuExportProgressLine("warning: encoder selected")).toBeNull();
expect(parseWindowsGpuExportProgressLine("PROGRESS not-json")).toBeNull();
expect(
parseWindowsGpuExportProgressLine(
'PROGRESS {"currentFrame":1,"totalFrames":0,"percentage":999}',
),
).toBeNull();
});
});
describe("parseNativeVideoMetadataProbeOutput", () => {
it("parses FFmpeg input metadata with video and audio streams", () => {
const metadata = parseNativeVideoMetadataProbeOutput(`
Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'recording.mp4':
Metadata:
major_brand : isom
Duration: 00:06:04.25, start: 0.000000, bitrate: 3938 kb/s
Stream #0:0[0x1](und): Video: h264 (High) (avc1 / 0x31637661), yuv420p(progressive), 1920x1080, 3720 kb/s, 46.05 fps, 60 tbr, 90k tbn (default)
Stream #0:1[0x2](und): Audio: aac (LC) (mp4a / 0x6134706D), 48000 Hz, stereo, fltp, 192 kb/s (default)
`);
expect(metadata).toEqual({
width: 1920,
height: 1080,
duration: 364.25,
mediaStartTime: 0,
streamStartTime: 0,
streamDuration: 364.25,
frameRate: 46.05,
codec: "h264 (High) (avc1 / 0x31637661)",
hasAudio: true,
audioCodec: "aac (LC) (mp4a / 0x6134706D)",
audioSampleRate: 48000,
});
});
it("parses video-only metadata and falls back to tbr when fps is absent", () => {
const metadata = parseNativeVideoMetadataProbeOutput(`
Input #0, matroska,webm, from 'recording.webm':
Duration: 00:00:10.50, start: 0.023000, bitrate: 1000 kb/s
Stream #0:0: Video: vp9, yuv420p, 1280x720, 30 tbr, 1k tbn
`);
expect(metadata).toEqual({
width: 1280,
height: 720,
duration: 10.5,
mediaStartTime: 0.023,
streamStartTime: 0.023,
streamDuration: 10.5,
frameRate: 30,
codec: "vp9",
hasAudio: false,
audioCodec: undefined,
audioSampleRate: undefined,
});
});
it("rejects output without usable video metadata", () => {
expect(parseNativeVideoMetadataProbeOutput("Duration: N/A")).toBeNull();
expect(parseNativeVideoMetadataProbeOutput("not a media file")).toBeNull();
});
});
describe("parseFfmpegDurationSeconds", () => {
it("parses HH:MM:SS timestamps", () => {
expect(parseFfmpegDurationSeconds("01:02:03.5")).toBe(3723.5);
expect(parseFfmpegDurationSeconds("bad")).toBeNull();
});
});
describe("parseFfmpegFrameRate", () => {
it("prefers fps and falls back to tbr", () => {
expect(parseFfmpegFrameRate("Video: h264, 1920x1080, 59.94 fps, 60 tbr")).toBe(59.94);
expect(parseFfmpegFrameRate("Video: h264, 1920x1080, 30 tbr")).toBe(30);
expect(parseFfmpegFrameRate("Video: h264")).toBeNull();
});
});
File diff suppressed because it is too large Load Diff
+28
View File
@@ -70,6 +70,34 @@ describe("getAudioSyncAdjustment", () => {
]);
});
it("pads the remaining tail after a measured late start delay", () => {
const filterParts: string[] = [];
appendSyncedAudioFilter(filterParts, "[1:a]", "aout", {
mode: "delay",
delayMs: 18051,
tempoRatio: 1,
durationDeltaMs: 1631070,
});
expect(filterParts).toEqual([
"[1:a]adelay=18051|18051,apad=pad_dur=1613.019,aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS[aout]",
]);
});
it("does not pad when a measured late start already explains the duration gap", () => {
const filterParts: string[] = [];
appendSyncedAudioFilter(filterParts, "[1:a]", "aout", {
mode: "delay",
delayMs: 10000,
tempoRatio: 1,
durationDeltaMs: 10000,
});
expect(filterParts).toEqual([
"[1:a]adelay=10000|10000,aresample=async=1:first_pts=0,asetpts=PTS-STARTPTS[aout]",
]);
});
it("can add a small gain boost before resampling", () => {
const filterParts: string[] = [];
appendSyncedAudioFilter(
+7
View File
@@ -114,6 +114,13 @@ export function appendSyncedAudioFilter(
filters.push(`adelay=${adjustment.delayMs}|${adjustment.delayMs}`);
}
if (
adjustment.mode === "delay" &&
adjustment.durationDeltaMs > adjustment.delayMs + 20
) {
filters.push(`apad=pad_dur=${formatFfmpegSeconds(adjustment.durationDeltaMs - adjustment.delayMs)}`);
}
if (adjustment.mode === "tempo") {
filters.push(...buildAtempoFilters(adjustment.tempoRatio));
}
+155
View File
@@ -2,7 +2,15 @@ import { describe, expect, it } from "vitest";
import { ATEMPO_FILTER_EPSILON } from "./ffmpeg/filters";
import {
buildEditedTrackSourceAudioFilter,
buildNativeConcatArgs,
buildNativeCudaOverlayStaticLayoutArgs,
buildNativeCudaScaleCpuPadStaticLayoutArgs,
buildNativePrecompositedStaticLayoutArgs,
buildNativeStaticBackgroundRenderArgs,
buildNativeStaticLayoutChunks,
buildTrimmedSourceAudioFilter,
createNativeSquircleMaskPgmBuffer,
isNativeCudaOutOfMemory,
} from "./nativeVideoExport";
describe("buildTrimmedSourceAudioFilter", () => {
@@ -118,3 +126,150 @@ describe("buildEditedTrackSourceAudioFilter", () => {
).toBeNull();
});
});
describe("native static layout command builders", () => {
const baseConfig = {
inputPath: "input.mp4",
outputPath: "chunk.mp4",
width: 1920,
height: 1080,
frameRate: 60,
bitrate: 8_000_000,
encodingMode: "fast" as const,
contentWidth: 1536,
contentHeight: 864,
offsetX: 192,
offsetY: 108,
backgroundColor: "#101010",
startSec: 120,
durationSec: 60,
};
it("builds the primary CUDA overlay layout command", () => {
const args = buildNativeCudaOverlayStaticLayoutArgs(baseConfig);
expect(args).toContain("-filter_complex");
expect(args).toContain(
"color=c=0x101010:s=1920x1080:r=60:d=60.000,format=nv12,hwupload_cuda[bg];" +
"[0:v]scale_cuda=w=1536:h=864:format=nv12,fps=60[fg];" +
"[bg][fg]overlay_cuda=192:108:shortest=0:repeatlast=1:eof_action=repeat,trim=duration=60.000,setpts=PTS-STARTPTS[out]",
);
expect(args).toContain("h264_nvenc");
expect(args).toContain("p1");
expect(args).not.toContain("yuv420p");
expect(args).toEqual(expect.arrayContaining(["-ss", "120.000", "-t", "60.000"]));
});
it("builds the stable CUDA scale plus CPU pad fallback command", () => {
const args = buildNativeCudaScaleCpuPadStaticLayoutArgs(baseConfig);
expect(args).toEqual(
expect.arrayContaining([
"-vf",
"scale_cuda=w=1536:h=864:format=nv12:passthrough=0,hwdownload,format=nv12,fps=60,pad=w=1920:h=1080:x=192:y=108:color=0x101010",
"-map",
"0:v:0",
"-an",
]),
);
});
it("sanitizes unsupported background colors to the safe dark fallback", () => {
const args = buildNativeCudaScaleCpuPadStaticLayoutArgs({
...baseConfig,
backgroundColor: "linear-gradient(red, blue)",
});
expect(args).toContain(
"scale_cuda=w=1536:h=864:format=nv12:passthrough=0,hwdownload,format=nv12,fps=60,pad=w=1920:h=1080:x=192:y=108:color=0x101010",
);
});
it("builds a precomposited background command for image wallpaper and shadow", () => {
const args = buildNativeStaticBackgroundRenderArgs({
...baseConfig,
outputPath: "background.png",
backgroundImagePath: "wallpaper.jpg",
maskPath: "mask.pgm",
shadowIntensity: 0.67,
});
const filterComplex = args[args.indexOf("-filter_complex") + 1];
expect(args).toEqual(expect.arrayContaining(["-i", "wallpaper.jpg", "-i", "mask.pgm"]));
expect(filterComplex).toContain(
"scale=w=1920:h=1080:force_original_aspect_ratio=increase,crop=w=1920:h=1080",
);
expect(filterComplex).toContain("split=3");
expect(filterComplex).toContain("gblur=sigma=32.16:steps=2");
expect(filterComplex).toContain("overlay=x=119:y=35:format=auto");
expect(args).toEqual(expect.arrayContaining(["-frames:v", "1", "background.png"]));
});
it("builds a precomposited static layout command with a squircle alpha mask", () => {
const args = buildNativePrecompositedStaticLayoutArgs({
...baseConfig,
staticBackgroundPath: "background.png",
maskPath: "mask.pgm",
borderRadius: 12.5,
});
const filterComplex = args[args.indexOf("-filter_complex") + 1];
expect(args).toEqual(expect.arrayContaining(["-i", "background.png", "-i", "mask.pgm"]));
expect(filterComplex).toContain(
"scale_cuda=w=1536:h=864:format=nv12:passthrough=0,hwdownload,format=nv12,fps=60,format=rgba",
);
expect(filterComplex).toContain("[fgbase][mask]alphamerge[fg]");
expect(filterComplex).toContain("overlay=x=192:y=108:format=auto");
expect(args).toContain("h264_nvenc");
expect(args).toEqual(expect.arrayContaining(["-pix_fmt", "yuv420p"]));
});
it("creates an opaque PGM mask for square video corners and a partial mask for radius", () => {
const squareMask = createNativeSquircleMaskPgmBuffer(4, 4, 0);
expect(squareMask.subarray(squareMask.length - 16)).toEqual(Buffer.alloc(16, 255));
const roundedMask = createNativeSquircleMaskPgmBuffer(8, 8, 4);
const header = Buffer.from("P5\n8 8\n255\n", "ascii");
const pixels = roundedMask.subarray(header.length);
expect(pixels[0]).toBeLessThan(255);
expect(pixels[4 * 8 + 4]).toBe(255);
});
it("splits long exports into bounded chunks", () => {
expect(buildNativeStaticLayoutChunks(367.5, 120)).toEqual([
{ index: 0, startSec: 0, durationSec: 120 },
{ index: 1, startSec: 120, durationSec: 120 },
{ index: 2, startSec: 240, durationSec: 120 },
{ index: 3, startSec: 360, durationSec: 7.5 },
]);
});
it("builds concat args for already encoded chunks", () => {
expect(buildNativeConcatArgs({ listPath: "chunks.txt", outputPath: "out.mp4" })).toEqual([
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"concat",
"-safe",
"0",
"-i",
"chunks.txt",
"-c",
"copy",
"-movflags",
"+faststart",
"out.mp4",
]);
});
it("detects CUDA OOM as a retryable fast-path failure", () => {
expect(
isNativeCudaOutOfMemory(
"cu->cuMemAlloc(&data, size) failed -> CUDA_ERROR_OUT_OF_MEMORY: out of memory",
),
).toBe(true);
expect(isNativeCudaOutOfMemory("FFmpeg exited with code 1")).toBe(false);
});
});
+424
View File
@@ -1,3 +1,8 @@
import {
getShadowFilterPadding,
VIDEO_SHADOW_LAYER_PROFILES,
} from "../../src/lib/exporter/shadowProfile";
import { getSquirclePathPoints } from "../../src/lib/geometry/squircle";
import { ATEMPO_FILTER_EPSILON, buildAtempoFilters } from "./ffmpeg/filters";
const NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL = 4;
@@ -33,6 +38,7 @@ export interface NativeVideoExportFinishOptions {
audioMode?: NativeVideoExportAudioMode;
audioSourcePath?: string | null;
audioSourceSampleRate?: number;
outputDurationSec?: number;
trimSegments?: NativeVideoExportAudioSegment[];
editedTrackStrategy?: NativeVideoExportEditedTrackStrategy;
editedTrackSegments?: NativeVideoExportEditedTrackSegment[];
@@ -50,6 +56,41 @@ export interface NativeVideoAudioMuxMetrics {
muxedVideoBytes?: number;
}
export type NativeStaticLayoutBackend =
| "cuda-overlay"
| "cuda-scale-cpu-pad"
| "cuda-static-composite"
| "nvidia-cuda-compositor"
| "windows-d3d11-compositor";
export interface NativeStaticLayoutExportArgsConfig {
inputPath: string;
outputPath: string;
width: number;
height: number;
frameRate: number;
bitrate: number;
encodingMode: NativeExportEncodingMode;
contentWidth: number;
contentHeight: number;
offsetX: number;
offsetY: number;
backgroundColor: string;
backgroundImagePath?: string | null;
staticBackgroundPath?: string | null;
maskPath?: string | null;
borderRadius?: number;
shadowIntensity?: number;
startSec?: number;
durationSec?: number;
}
export interface NativeStaticLayoutChunk {
index: number;
startSec: number;
durationSec: number;
}
export function getNativeVideoInputByteSize(width: number, height: number): number {
return width * height * NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL;
}
@@ -107,6 +148,121 @@ function getBitrateArgs(bitrate: number): string[] {
];
}
function getNvencStaticLayoutModeArgs(encodingMode: NativeExportEncodingMode): string[] {
const lowLatencyRateControlArgs = [
"-rc",
"vbr",
"-multipass",
"disabled",
"-rc-lookahead",
"0",
"-surfaces",
"32",
];
switch (encodingMode) {
case "quality":
return ["-preset", "p1", "-tune", "hq", ...lowLatencyRateControlArgs];
case "balanced":
return ["-preset", "p1", "-tune", "ll", ...lowLatencyRateControlArgs];
case "fast":
default:
return ["-preset", "p1", "-tune", "ull", ...lowLatencyRateControlArgs];
}
}
function formatFfmpegColor(value: string): string {
const trimmed = value.trim();
const hex = trimmed.match(/^#?([0-9a-f]{6})$/i)?.[1];
return hex ? `0x${hex.toLowerCase()}` : "0x101010";
}
function clampUnitInterval(value: number): number {
if (!Number.isFinite(value)) {
return 0;
}
return Math.min(1, Math.max(0, value));
}
function formatFfmpegNumber(value: number): string {
return Number.isInteger(value) ? String(value) : value.toFixed(6).replace(/0+$/, "").replace(/\.$/, "");
}
function isPointInsidePolygon(x: number, y: number, points: Array<{ x: number; y: number }>) {
let inside = false;
for (let index = 0, previousIndex = points.length - 1; index < points.length; previousIndex = index++) {
const current = points[index];
const previous = points[previousIndex];
const intersects =
(current.y > y) !== (previous.y > y) &&
x < ((previous.x - current.x) * (y - current.y)) / (previous.y - current.y) + current.x;
if (intersects) {
inside = !inside;
}
}
return inside;
}
export function createNativeSquircleMaskPgmBuffer(
width: number,
height: number,
radius: number,
): Buffer {
const safeWidth = Math.max(1, Math.round(width));
const safeHeight = Math.max(1, Math.round(height));
const clampedRadius = Math.min(Math.max(0, radius), Math.min(safeWidth, safeHeight) / 2);
const header = Buffer.from(`P5\n${safeWidth} ${safeHeight}\n255\n`, "ascii");
const pixels = Buffer.alloc(safeWidth * safeHeight, 255);
if (clampedRadius <= 0.5) {
return Buffer.concat([header, pixels]);
}
const points = getSquirclePathPoints({
x: 0,
y: 0,
width: safeWidth,
height: safeHeight,
radius: clampedRadius,
});
const samples = [
[0.25, 0.25],
[0.75, 0.25],
[0.25, 0.75],
[0.75, 0.75],
] as const;
for (let y = 0; y < safeHeight; y += 1) {
for (let x = 0; x < safeWidth; x += 1) {
let coveredSamples = 0;
for (const [sampleX, sampleY] of samples) {
if (isPointInsidePolygon(x + sampleX, y + sampleY, points)) {
coveredSamples += 1;
}
}
pixels[y * safeWidth + x] = Math.round((coveredSamples / samples.length) * 255);
}
}
return Buffer.concat([header, pixels]);
}
function pushFfmpegTimeSliceArgs(
args: string[],
startSec: number | undefined,
durationSec: number | undefined,
) {
if (Number.isFinite(startSec) && (startSec ?? 0) > 0) {
args.push("-ss", formatFfmpegSeconds((startSec ?? 0) * 1000));
}
if (Number.isFinite(durationSec) && (durationSec ?? 0) > 0) {
args.push("-t", formatFfmpegSeconds((durationSec ?? 0) * 1000));
}
}
export function buildNativeVideoExportArgs(
encoder: string,
options: NativeVideoExportStartOptions,
@@ -145,6 +301,274 @@ export function buildNativeVideoExportArgs(
return args;
}
export function buildNativeCudaOverlayStaticLayoutArgs(
config: NativeStaticLayoutExportArgsConfig,
): string[] {
const backgroundColor = formatFfmpegColor(config.backgroundColor);
const durationSec = formatFfmpegSeconds(Math.max(0.001, config.durationSec ?? 1) * 1000);
const args = ["-y", "-hide_banner", "-loglevel", "error"];
pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec);
args.push(
"-hwaccel",
"cuda",
"-hwaccel_output_format",
"cuda",
"-i",
config.inputPath,
"-filter_complex",
`color=c=${backgroundColor}:s=${config.width}x${config.height}:r=${config.frameRate}:d=${durationSec},format=nv12,hwupload_cuda[bg];[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12,fps=${config.frameRate}[fg];[bg][fg]overlay_cuda=${config.offsetX}:${config.offsetY}:shortest=0:repeatlast=1:eof_action=repeat,trim=duration=${durationSec},setpts=PTS-STARTPTS[out]`,
"-map",
"[out]",
"-an",
"-r",
String(config.frameRate),
"-c:v",
"h264_nvenc",
...getNvencStaticLayoutModeArgs(config.encodingMode),
...getBitrateArgs(config.bitrate),
"-movflags",
"+faststart",
config.outputPath,
);
return args;
}
export function buildNativeCudaScaleCpuPadStaticLayoutArgs(
config: NativeStaticLayoutExportArgsConfig,
): string[] {
const backgroundColor = formatFfmpegColor(config.backgroundColor);
const args = ["-y", "-hide_banner", "-loglevel", "error"];
pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec);
args.push(
"-hwaccel",
"cuda",
"-hwaccel_output_format",
"cuda",
"-i",
config.inputPath,
"-vf",
`scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12:passthrough=0,hwdownload,format=nv12,fps=${config.frameRate},pad=w=${config.width}:h=${config.height}:x=${config.offsetX}:y=${config.offsetY}:color=${backgroundColor}`,
"-map",
"0:v:0",
"-an",
"-r",
String(config.frameRate),
"-c:v",
"h264_nvenc",
...getNvencStaticLayoutModeArgs(config.encodingMode),
...getBitrateArgs(config.bitrate),
"-pix_fmt",
"yuv420p",
"-movflags",
"+faststart",
config.outputPath,
);
return args;
}
export function buildNativeStaticBackgroundRenderArgs(
config: NativeStaticLayoutExportArgsConfig,
): string[] {
const backgroundColor = formatFfmpegColor(config.backgroundColor);
const args = ["-y", "-hide_banner", "-loglevel", "error"];
if (config.backgroundImagePath) {
args.push("-i", config.backgroundImagePath);
} else {
args.push(
"-f",
"lavfi",
"-i",
`color=c=${backgroundColor}:s=${config.width}x${config.height}:d=1`,
);
}
const shadowStrength = clampUnitInterval(config.shadowIntensity ?? 0);
const shadowLayers =
shadowStrength > 0 && config.maskPath
? VIDEO_SHADOW_LAYER_PROFILES.map((layer) => ({
offsetY: layer.offsetScale * shadowStrength,
alpha: clampUnitInterval(layer.alphaScale * shadowStrength),
blur: Math.max(0, layer.blurScale * shadowStrength),
})).filter((layer) => layer.alpha > 0)
: [];
if (shadowLayers.length > 0 && config.maskPath) {
args.push("-i", config.maskPath);
}
const filterParts = [
config.backgroundImagePath
? `[0:v]scale=w=${config.width}:h=${config.height}:force_original_aspect_ratio=increase,crop=w=${config.width}:h=${config.height},setsar=1,format=rgba[bg0]`
: "[0:v]format=rgba[bg0]",
];
let currentBackgroundLabel = "bg0";
if (shadowLayers.length > 0) {
filterParts.push(
`[1:v]format=gray,split=${shadowLayers.length}${shadowLayers
.map((_, index) => `[shadow_mask_source_${index}]`)
.join("")}`,
);
}
shadowLayers.forEach((layer, index) => {
const padding = getShadowFilterPadding(layer.blur, layer.offsetY);
const paddedWidth = config.contentWidth + padding * 2;
const paddedHeight = config.contentHeight + padding * 2;
const positionedMaskLabel = `shadow_mask_positioned_${index}`;
const shadowMaskLabel = `shadow_mask_${index}`;
const shadowColorLabel = `shadow_color_${index}`;
const shadowLabel = `shadow_${index}`;
const nextBackgroundLabel = `bg${index + 1}`;
const blurFilter =
layer.blur > 0 ? `,gblur=sigma=${formatFfmpegNumber(layer.blur)}:steps=2` : "";
filterParts.push(
`[shadow_mask_source_${index}]lut=y=val*${formatFfmpegNumber(
layer.alpha,
)},pad=w=${paddedWidth}:h=${paddedHeight}:x=${padding}:y=${Math.round(
padding + layer.offsetY,
)}:color=black${blurFilter}[${positionedMaskLabel}]`,
`[${positionedMaskLabel}]format=gray[${shadowMaskLabel}]`,
`color=c=black:s=${paddedWidth}x${paddedHeight}:d=1,format=rgba[${shadowColorLabel}]`,
`[${shadowColorLabel}][${shadowMaskLabel}]alphamerge[${shadowLabel}]`,
`[${currentBackgroundLabel}][${shadowLabel}]overlay=x=${
config.offsetX - padding
}:y=${config.offsetY - padding}:format=auto[${nextBackgroundLabel}]`,
);
currentBackgroundLabel = nextBackgroundLabel;
});
filterParts.push(`[${currentBackgroundLabel}]format=rgba[out]`);
args.push(
"-filter_complex",
filterParts.join(";"),
"-map",
"[out]",
"-frames:v",
"1",
config.outputPath,
);
return args;
}
export function buildNativePrecompositedStaticLayoutArgs(
config: NativeStaticLayoutExportArgsConfig,
): string[] {
if (!config.staticBackgroundPath) {
throw new Error("Native precomposited static layout requires a static background path");
}
const durationSec = formatFfmpegSeconds(Math.max(0.001, config.durationSec ?? 1) * 1000);
const useMask = Boolean(config.maskPath && (config.borderRadius ?? 0) > 0.5);
const args = ["-y", "-hide_banner", "-loglevel", "error"];
pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec);
args.push(
"-hwaccel",
"cuda",
"-hwaccel_output_format",
"cuda",
"-i",
config.inputPath,
"-loop",
"1",
"-framerate",
String(config.frameRate),
"-t",
durationSec,
"-i",
config.staticBackgroundPath,
);
if (useMask && config.maskPath) {
args.push(
"-loop",
"1",
"-framerate",
String(config.frameRate),
"-t",
durationSec,
"-i",
config.maskPath,
);
}
const foregroundFilter = `[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12:passthrough=0,hwdownload,format=nv12,fps=${config.frameRate},format=rgba[fgbase]`;
const maskFilter = useMask
? ";[2:v]format=gray[mask];[fgbase][mask]alphamerge[fg]"
: "";
const foregroundLabel = useMask ? "fg" : "fgbase";
const filterComplex = `${foregroundFilter}${maskFilter};[1:v]format=rgba[bg];[bg][${foregroundLabel}]overlay=x=${config.offsetX}:y=${config.offsetY}:format=auto,trim=duration=${durationSec},setpts=PTS-STARTPTS,format=yuv420p[out]`;
args.push(
"-filter_complex",
filterComplex,
"-map",
"[out]",
"-an",
"-r",
String(config.frameRate),
"-c:v",
"h264_nvenc",
...getNvencStaticLayoutModeArgs(config.encodingMode),
...getBitrateArgs(config.bitrate),
"-pix_fmt",
"yuv420p",
"-movflags",
"+faststart",
config.outputPath,
);
return args;
}
export function buildNativeConcatArgs(config: {
listPath: string;
outputPath: string;
}): string[] {
return [
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"concat",
"-safe",
"0",
"-i",
config.listPath,
"-c",
"copy",
"-movflags",
"+faststart",
config.outputPath,
];
}
export function buildNativeStaticLayoutChunks(
durationSec: number,
chunkDurationSec: number,
): NativeStaticLayoutChunk[] {
if (!Number.isFinite(durationSec) || durationSec <= 0) {
return [];
}
const safeChunkDuration = Math.max(1, Math.min(300, Math.floor(chunkDurationSec)));
const chunks: NativeStaticLayoutChunk[] = [];
for (let startSec = 0, index = 0; startSec < durationSec; startSec += safeChunkDuration, index++) {
chunks.push({
index,
startSec,
durationSec: Math.min(safeChunkDuration, durationSec - startSec),
});
}
return chunks;
}
export function isNativeCudaOutOfMemory(stderr: string): boolean {
return /CUDA_ERROR_OUT_OF_MEMORY|cuMemAlloc.+out of memory/i.test(stderr);
}
function formatFfmpegSeconds(milliseconds: number): string {
return (milliseconds / 1000).toFixed(3);
}
+4
View File
@@ -63,6 +63,10 @@ export function resolvePreferredWindowsNativeHelperPath(
);
const prebundledPath = getPrebundledNativeHelperPath(binaryName);
if (app.isPackaged && existsSync(prebundledPath)) {
return prebundledPath;
}
if (existsSync(buildOutputPath)) {
return buildOutputPath;
}
+22
View File
@@ -115,4 +115,26 @@ describe("local media path policy", () => {
await expect(fs.readFile(thumbnailPath, "utf8")).resolves.toBe("png-thumbnail");
});
it("loads project files that start with a UTF-8 byte order mark", async () => {
const videoPath = path.join(tempPath, "recording.mp4");
const projectPath = path.join(tempPath, "recording.recordly");
await fs.writeFile(videoPath, "test-video");
await fs.writeFile(
projectPath,
`\uFEFF${JSON.stringify({
version: 1,
videoPath,
editor: {},
})}`,
"utf-8",
);
const { loadProjectFromPath } = await import("./manager");
const result = await loadProjectFromPath(projectPath);
expect(result.success).toBe(true);
expect(result.path).toBe(projectPath);
expect(result.project).toMatchObject({ videoPath });
});
});
+12 -12
View File
@@ -1,33 +1,33 @@
import { constants as fsConstants } from "node:fs";
import { existsSync } from "node:fs";
import { existsSync, constants as fsConstants } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { app } from "electron";
import { RECORDINGS_DIR, USER_DATA_PATH } from "../../appPaths";
import { isSupportedLocalMediaPath } from "../../mediaTypes";
import {
PROJECT_FILE_EXTENSION,
LEGACY_PROJECT_FILE_EXTENSIONS,
PROJECTS_DIRECTORY_NAME,
PROJECT_THUMBNAIL_SUFFIX,
RECENT_PROJECTS_FILE,
MAX_RECENT_PROJECTS,
PROJECT_FILE_EXTENSION,
PROJECT_THUMBNAIL_SUFFIX,
PROJECTS_DIRECTORY_NAME,
RECENT_PROJECTS_FILE,
RECORDINGS_SETTINGS_FILE,
} from "../constants";
import type { ProjectLibraryEntry, RecordingSessionData } from "../types";
import {
approvedLocalReadPaths,
currentProjectPath,
setCurrentProjectPath,
setCurrentVideoPath,
setCurrentRecordingSession,
approvedLocalReadPaths,
setCurrentVideoPath,
setCustomRecordingsDir,
setRecordingsDirLoaded,
} from "../state";
import type { ProjectLibraryEntry, RecordingSessionData } from "../types";
import {
getRecordingsDir,
normalizePath,
normalizeVideoSourcePath,
getRecordingsDir,
parseJsonWithByteOrderMark,
} from "../utils";
@@ -247,7 +247,7 @@ export async function saveProjectThumbnail(projectPath: string, thumbnailDataUrl
export async function loadRecentProjectPaths() {
try {
const content = await fs.readFile(RECENT_PROJECTS_FILE, "utf-8");
const parsed = JSON.parse(content) as { paths?: unknown };
const parsed = parseJsonWithByteOrderMark<{ paths?: unknown }>(content);
return Array.isArray(parsed.paths)
? parsed.paths.filter(
(value): value is string =>
@@ -364,7 +364,7 @@ export async function loadProjectFromPath(projectPath: string) {
let project: unknown;
try {
const content = await fs.readFile(normalizedPath, "utf-8");
project = JSON.parse(content);
project = parseJsonWithByteOrderMark(content);
} catch (error) {
return {
success: false,
+3 -2
View File
@@ -3,7 +3,7 @@ import fs from "node:fs/promises";
import path from "node:path";
import { RECORDING_SESSION_MANIFEST_SUFFIX } from "../constants";
import type { RecordingSessionData, RecordingSessionManifest } from "../types";
import { normalizeVideoSourcePath } from "../utils";
import { normalizeVideoSourcePath, parseJsonWithByteOrderMark } from "../utils";
function normalizeRecordingTimeOffsetMs(value: unknown): number {
return typeof value === "number" && Number.isFinite(value) ? Math.round(value) : 0;
@@ -51,7 +51,8 @@ export async function resolveRecordingSessionManifest(
try {
const content = await fs.readFile(manifestPath, "utf-8");
const parsed = JSON.parse(content) as Partial<RecordingSessionManifest>;
const parsed =
parseJsonWithByteOrderMark<Partial<RecordingSessionManifest>>(content);
if (parsed.version !== 1 && parsed.version !== 2) {
return null;
}
+12 -1
View File
@@ -140,7 +140,7 @@ describe("getCompanionAudioFallbackPaths", () => {
await Promise.all([
fs.writeFile(videoPath, "video"),
fs.writeFile(micPath, "mic"),
fs.writeFile(`${micPath}.json`, JSON.stringify({ startDelayMs: 2750 })),
fs.writeFile(`${micPath}.json`, `\ufeff${JSON.stringify({ startDelayMs: 2750 })}`),
]);
execFileMock.mockImplementation(
@@ -166,6 +166,17 @@ describe("getCompanionAudioFallbackPaths", () => {
});
});
it("scales audio mux timeout for long recordings", async () => {
const { getRecordingAudioMuxTimeoutMs } = await import("./diagnostics");
expect(getRecordingAudioMuxTimeoutMs(0)).toBe(5 * 60 * 1000);
expect(getRecordingAudioMuxTimeoutMs(29 * 60 + 29.41)).toBeGreaterThan(120000);
expect(getRecordingAudioMuxTimeoutMs(29 * 60 + 29.41)).toBeCloseTo(
(29 * 60 + 29.41) * 1000 + 60 * 1000,
0,
);
});
it("ignores invalid sidecar timing metadata values", async () => {
const micPath = path.join(tempRoot, "recording.mic.wav");
await Promise.all([
+17 -1
View File
@@ -5,9 +5,12 @@ import { COMPANION_AUDIO_LAYOUTS } from "../constants";
import { getFfmpegBinaryPath } from "../ffmpeg/binary";
import { lastNativeCaptureDiagnostics, setLastNativeCaptureDiagnostics } from "../state";
import type { CompanionAudioCandidate, NativeCaptureDiagnostics } from "../types";
import { parseJsonWithByteOrderMark } from "../utils";
const execFileAsync = promisify(execFile);
export const MIN_VALID_RECORDED_VIDEO_BYTES = 1024;
export const RECORDING_AUDIO_MUX_MIN_TIMEOUT_MS = 5 * 60 * 1000;
export const RECORDING_AUDIO_MUX_MAX_TIMEOUT_MS = 2 * 60 * 60 * 1000;
type CompanionAudioTimingMetadata = {
startDelayMs?: number;
@@ -53,6 +56,18 @@ export function parseFfmpegDurationSeconds(stderr: string) {
return hours * 3600 + minutes * 60 + seconds;
}
export function getRecordingAudioMuxTimeoutMs(durationSeconds: number) {
if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) {
return RECORDING_AUDIO_MUX_MIN_TIMEOUT_MS;
}
const realtimeMuxBudgetMs = Math.ceil(durationSeconds * 1000) + 60 * 1000;
return Math.min(
RECORDING_AUDIO_MUX_MAX_TIMEOUT_MS,
Math.max(RECORDING_AUDIO_MUX_MIN_TIMEOUT_MS, realtimeMuxBudgetMs),
);
}
/** Probe the duration of a media file (in seconds) using the container header. */
export async function probeMediaDurationSeconds(filePath: string): Promise<number> {
const ffmpegPath = getFfmpegBinaryPath();
@@ -108,7 +123,8 @@ async function readCompanionAudioTimingMetadata(
): Promise<CompanionAudioTimingMetadata | null> {
try {
const raw = await fs.readFile(`${companionPath}.json`, "utf8");
const parsed = JSON.parse(raw) as CompanionAudioTimingMetadata | null;
const parsed =
parseJsonWithByteOrderMark<CompanionAudioTimingMetadata | null>(raw);
if (!parsed || typeof parsed !== "object") {
return null;
}
+26 -15
View File
@@ -10,27 +10,28 @@ import {
import { getFfmpegBinaryPath } from "../ffmpeg/binary";
import { appendSyncedAudioFilter, getAudioSyncAdjustment } from "../ffmpeg/filters";
import {
nativeScreenRecordingActive,
setNativeScreenRecordingActive,
setNativeCaptureProcess,
nativeCaptureOutputBuffer,
nativeCaptureTargetPath,
setNativeCaptureTargetPath,
nativeCaptureStopRequested,
setNativeCaptureStopRequested,
nativeCaptureSystemAudioPath,
setNativeCaptureSystemAudioPath,
nativeCaptureMicrophonePath,
setNativeCaptureMicrophonePath,
lastNativeCaptureDiagnostics,
setCurrentVideoPath,
setCurrentProjectPath,
nativeCaptureMicrophonePath,
nativeCaptureOutputBuffer,
nativeCaptureStopRequested,
nativeCaptureSystemAudioPath,
nativeCaptureTargetPath,
nativeScreenRecordingActive,
selectedSource,
setCurrentProjectPath,
setCurrentVideoPath,
setNativeCaptureMicrophonePath,
setNativeCaptureProcess,
setNativeCaptureStopRequested,
setNativeCaptureSystemAudioPath,
setNativeCaptureTargetPath,
setNativeScreenRecordingActive,
} from "../state";
import type { AudioSyncAdjustment } from "../types";
import { isAutoRecordingPath, moveFileWithOverwrite } from "../utils";
import {
getFileSizeIfPresent,
getRecordingAudioMuxTimeoutMs,
getUsableCompanionAudioCandidates,
probeMediaDurationSeconds,
recordNativeCaptureDiagnostics,
@@ -163,6 +164,7 @@ export async function muxNativeMacRecordingWithAudio(
}
const videoDuration = await probeMediaDurationSeconds(videoPath);
const muxTimeoutMs = getRecordingAudioMuxTimeoutMs(videoDuration);
const audioAdjustments: Map<string, AudioSyncAdjustment> = new Map();
if (videoDuration > 0) {
@@ -207,6 +209,9 @@ export async function muxNativeMacRecordingWithAudio(
filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]");
args = [
"-y",
"-hide_banner",
"-nostdin",
"-nostats",
...inputs,
"-filter_complex",
filterParts.join(";"),
@@ -234,6 +239,9 @@ export async function muxNativeMacRecordingWithAudio(
appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment);
args = [
"-y",
"-hide_banner",
"-nostdin",
"-nostats",
...inputs,
"-filter_complex",
filterParts.join(";"),
@@ -255,7 +263,10 @@ export async function muxNativeMacRecordingWithAudio(
console.log("[mux] Running ffmpeg:", ffmpegPath, args.join(" "));
try {
await execFileAsync(ffmpegPath, args, { timeout: 120000, maxBuffer: 10 * 1024 * 1024 });
await execFileAsync(ffmpegPath, args, {
timeout: muxTimeoutMs,
maxBuffer: 20 * 1024 * 1024,
});
await validateRecordedVideo(mixedOutputPath);
} catch (error) {
const execError = error as NodeJS.ErrnoException & { stderr?: string };
+14 -1
View File
@@ -15,6 +15,7 @@ import {
isAutoRecordingPath,
normalizePath,
normalizeVideoSourcePath,
parseJsonWithByteOrderMark,
} from "../utils";
export async function hasSiblingProjectFile(videoPath: string) {
@@ -71,10 +72,22 @@ async function loadSavedProjectMediaPaths() {
})
.map(async (entry) => {
const projectPath = path.join(projectsDir, entry.name);
const rawProject = JSON.parse(await fs.readFile(projectPath, "utf-8")) as {
let rawProject: {
videoPath?: unknown;
editor?: { webcam?: { sourcePath?: unknown } };
};
try {
rawProject = parseJsonWithByteOrderMark<{
videoPath?: unknown;
editor?: { webcam?: { sourcePath?: unknown } };
}>(await fs.readFile(projectPath, "utf-8"));
} catch (error) {
console.warn("[prune] Skipping unreadable project while pruning recordings", {
projectPath,
error,
});
return;
}
const candidatePaths = [
rawProject.videoPath,
rawProject.editor?.webcam?.sourcePath,
+10 -2
View File
@@ -27,6 +27,7 @@ import type { AudioSyncAdjustment, PauseSegment } from "../types";
import { moveFileWithOverwrite } from "../utils";
import {
getCompanionAudioStartDelayMs,
getRecordingAudioMuxTimeoutMs,
probeMediaDurationSeconds,
validateRecordedVideo,
} from "./diagnostics";
@@ -194,6 +195,7 @@ export async function muxNativeWindowsVideoWithAudio(
if (audioInputs.length === 0) return;
const videoDuration = await probeMediaDurationSeconds(videoPath);
const muxTimeoutMs = getRecordingAudioMuxTimeoutMs(videoDuration);
const audioAdjustments: Map<string, AudioSyncAdjustment> = new Map();
if (videoDuration > 0) {
@@ -278,6 +280,9 @@ export async function muxNativeWindowsVideoWithAudio(
ffmpegPath,
[
"-y",
"-hide_banner",
"-nostdin",
"-nostats",
...inputs,
"-filter_complex",
filterParts.join(";"),
@@ -294,7 +299,7 @@ export async function muxNativeWindowsVideoWithAudio(
"-shortest",
mixedOutputPath,
],
{ timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
{ timeout: muxTimeoutMs, maxBuffer: 20 * 1024 * 1024 },
);
} else {
const pauseFilter = buildPausedAudioFilter(
@@ -329,6 +334,9 @@ export async function muxNativeWindowsVideoWithAudio(
ffmpegPath,
[
"-y",
"-hide_banner",
"-nostdin",
"-nostats",
...inputs,
"-filter_complex",
filterParts.join(";"),
@@ -345,7 +353,7 @@ export async function muxNativeWindowsVideoWithAudio(
"-shortest",
mixedOutputPath,
],
{ timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
{ timeout: muxTimeoutMs, maxBuffer: 20 * 1024 * 1024 },
);
}
+62
View File
@@ -0,0 +1,62 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
vi.mock("electron", () => ({
app: {
getAppPath: () => process.cwd(),
getPath: () => process.env.TEMP ?? process.cwd(),
isPackaged: false,
},
BrowserWindow: {
fromWebContents: () => null,
},
dialog: {
showSaveDialog: vi.fn(),
},
ipcMain: {
handle: vi.fn(),
},
powerSaveBlocker: {
isStarted: () => true,
start: () => 1,
stop: vi.fn(),
},
}));
vi.mock("../ffmpeg/binary", () => ({
getFfmpegBinaryPath: () => "ffmpeg",
}));
import { moveExportedTempFile } from "./export";
const tempDirs: string[] = [];
async function makeTempDir() {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "recordly-export-move-"));
tempDirs.push(dir);
return dir;
}
afterEach(async () => {
await Promise.allSettled(
tempDirs.splice(0).map((dir) => fs.rm(dir, { force: true, recursive: true })),
);
});
describe("moveExportedTempFile", () => {
it("moves an app-managed export temp file to the selected destination", async () => {
const dir = await makeTempDir();
const tempPath = path.join(dir, "export-temp.mp4");
const destinationPath = path.join(dir, "export-final.mp4");
await fs.writeFile(tempPath, "recordly-export");
await moveExportedTempFile(tempPath, destinationPath);
await expect(fs.readFile(destinationPath, "utf8")).resolves.toBe(
"recordly-export",
);
await expect(fs.access(tempPath)).rejects.toThrow();
});
});
+117 -11
View File
@@ -15,6 +15,7 @@ import {
} from "../export/exportStream";
import {
enqueueNativeVideoExportFrameWrite,
exportNativeStaticLayoutVideo,
flushNativeVideoExportPendingWriteRequests,
getNativeVideoExportMaxQueuedWriteBytes,
getNativeVideoExportSessionError,
@@ -22,8 +23,11 @@ import {
isIgnorableNativeVideoExportStreamError,
muxExportedVideoAudioBuffer,
muxNativeVideoExportAudio,
type NativeStaticLayoutExportOptions,
type NativeVideoExportSession,
nativeStaticLayoutExportSessions,
nativeVideoExportSessions,
probeNativeVideoMetadata,
removeTemporaryExportFile,
resolveNativeVideoEncoder,
sendNativeVideoExportWriteFrameResult,
@@ -39,7 +43,13 @@ import {
} from "../nativeVideoExport";
import { approveUserPath } from "../utils";
async function moveExportedTempFile(tempPath: string, destinationPath: string) {
function getPartialExportDestinationPath(destinationPath: string) {
const parsed = path.parse(destinationPath);
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
return path.join(parsed.dir, `.recordly-partial-${parsed.name}-${suffix}${parsed.ext}`);
}
export async function moveExportedTempFile(tempPath: string, destinationPath: string) {
await fs.mkdir(path.dirname(destinationPath), { recursive: true });
try {
await fs.rename(tempPath, destinationPath);
@@ -53,17 +63,33 @@ async function moveExportedTempFile(tempPath: string, destinationPath: string) {
// exporting to a different volume still works.
}
await fs.copyFile(tempPath, destinationPath);
const partialDestinationPath = getPartialExportDestinationPath(destinationPath);
try {
await fs.rm(tempPath, { force: true });
} catch (unlinkError) {
// Copy succeeded, so the export itself is safe; surface the leaked temp
// path instead of silently swallowing the failure so operators can
// reclaim disk space manually if the OS temp reaper misses it.
console.warn(
`[export] Failed to remove temp file after cross-volume copy (${tempPath}):`,
unlinkError,
);
await fs.copyFile(tempPath, partialDestinationPath);
try {
await fs.rename(partialDestinationPath, destinationPath);
} catch (renameError) {
const code = (renameError as NodeJS.ErrnoException).code;
if (code !== "EEXIST" && code !== "EPERM") {
throw renameError;
}
await fs.rm(destinationPath, { force: true });
await fs.rename(partialDestinationPath, destinationPath);
}
try {
await fs.rm(tempPath, { force: true });
} catch (unlinkError) {
// Copy succeeded, so the export itself is safe; surface the leaked temp
// path instead of silently swallowing the failure so operators can
// reclaim disk space manually if the OS temp reaper misses it.
console.warn(
`[export] Failed to remove temp file after cross-volume copy (${tempPath}):`,
unlinkError,
);
}
} catch (error) {
await fs.rm(partialDestinationPath, { force: true }).catch(() => undefined);
throw error;
}
}
@@ -218,6 +244,86 @@ export function registerExportHandlers() {
},
);
ipcMain.handle("probe-native-video-metadata", async (_, filePath: string) => {
try {
if (typeof filePath !== "string" || filePath.trim().length === 0) {
throw new Error("Native metadata probe requires a file path");
}
const metadata = await probeNativeVideoMetadata(getFfmpegBinaryPath(), filePath);
return {
success: true,
metadata,
};
} catch (error) {
console.warn("[probe-native-video-metadata] Failed:", error);
return {
success: false,
error: error instanceof Error ? error.message : String(error),
};
}
});
ipcMain.handle(
"native-static-layout-export",
async (event, options: NativeStaticLayoutExportOptions) => {
try {
if (!options || typeof options.inputPath !== "string") {
throw new Error("Native static layout export requires an input path");
}
const result = await exportNativeStaticLayoutVideo(
getFfmpegBinaryPath(),
options,
(progress) => {
if (event.sender.isDestroyed()) {
return;
}
event.sender.send("native-static-layout-export-progress", progress);
},
);
registerOwnedExportPath(result.outputPath);
const primaryBackend = result.metrics.chunks[0]?.backend;
return {
success: true,
tempPath: result.outputPath,
encoderName:
primaryBackend === "nvidia-cuda-compositor"
? "nvidia-cuda-compositor"
: primaryBackend === "windows-d3d11-compositor"
? "windows-d3d11-compositor"
: result.metrics.chunkCount > 1
? "chunked-h264-nvenc"
: "static-layout-h264-nvenc",
metrics: result.metrics,
};
} catch (error) {
console.warn("[native-static-layout-export] Failed:", error);
return {
success: false,
error: error instanceof Error ? error.message : String(error),
};
}
},
);
ipcMain.handle("native-static-layout-export-cancel", async (_, sessionId: string) => {
const session = nativeStaticLayoutExportSessions.get(sessionId);
if (!session) {
return { success: true };
}
session.terminating = true;
try {
session.currentProcess?.kill("SIGKILL");
} catch {
// Process may already be closed.
}
return { success: true };
});
ipcMain.on(
"native-video-export-write-frame-async",
(
+19 -18
View File
@@ -1,45 +1,46 @@
import { constants as fsConstants } from "node:fs";
import { randomUUID } from "node:crypto";
import { constants as fsConstants } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { dialog, ipcMain, shell } from "electron";
import { RECORDINGS_DIR } from "../../appPaths";
import { buildMediaUrl, getMediaServerBaseUrl } from "../../mediaServer";
import {
PROJECT_FILE_EXTENSION,
LEGACY_PROJECT_FILE_EXTENSIONS,
PROJECT_FILE_EXTENSION,
} from "../constants";
import {
currentProjectPath,
setCurrentProjectPath,
currentVideoPath,
setCurrentVideoPath,
currentRecordingSession,
setCurrentRecordingSession,
} from "../state";
import {
getProjectsDir,
getProjectThumbnailPath,
isPathInsideDirectory,
isTrustedProjectPath,
listProjectLibraryEntries,
loadRecentProjectPaths,
loadProjectFromPath,
loadRecentProjectPaths,
persistRecordingsDirectorySetting,
rememberRecentProject,
replaceApprovedSessionLocalReadPaths,
resolveApprovedLocalMediaPath,
rememberRecentProject,
saveRecentProjectPaths,
saveProjectThumbnail,
saveRecentProjectPaths,
} from "../project/manager";
import { persistRecordingSessionManifest, resolveRecordingSession } from "../project/session";
import {
currentProjectPath,
currentRecordingSession,
currentVideoPath,
setCurrentProjectPath,
setCurrentRecordingSession,
setCurrentVideoPath,
} from "../state";
import {
approveUserPath,
getRecordingsDir,
getTelemetryPathForVideo,
isAutoRecordingPath,
getRecordingsDir,
approveUserPath,
normalizeVideoSourcePath,
parseJsonWithByteOrderMark,
} from "../utils";
import { persistRecordingSessionManifest, resolveRecordingSession } from "../project/session";
function normalizeRecordingTimeOffsetMs(value: unknown): number {
return typeof value === "number" && Number.isFinite(value) ? Math.round(value) : 0;
@@ -158,7 +159,7 @@ async function ensureNamedProjectSaveDoesNotOverwriteDifferentProject(
try {
const existingProjectRaw = await fs.readFile(targetProjectPath, "utf-8");
const existingProjectData = JSON.parse(existingProjectRaw) as unknown;
const existingProjectData = parseJsonWithByteOrderMark(existingProjectRaw);
const existingProjectId = getProjectId(existingProjectData);
const existingVideoPath = getProjectVideoPath(existingProjectData);
@@ -607,7 +608,7 @@ export function registerProjectHandlers() {
await fs.unlink(resolvedPath);
// Also delete the cursor telemetry sidecar if it exists
const telemetryPath = getTelemetryPathForVideo(resolvedPath);
await fs.unlink(telemetryPath).catch(() => {});
await fs.unlink(telemetryPath).catch(() => undefined);
const currentResolved = currentVideoPath
? await fs.realpath(currentVideoPath).catch(() => currentVideoPath)
: null;
+8 -3
View File
@@ -134,6 +134,7 @@ import {
getTelemetryPathForVideo,
moveFileWithOverwrite,
normalizeVideoSourcePath,
parseJsonWithByteOrderMark,
parseWindowId,
} from "../utils";
import { resolveWindowsCaptureDisplay } from "../windowsCaptureSelection";
@@ -1341,11 +1342,15 @@ export function registerRecordingHandlers(
const telemetryPath = getTelemetryPathForVideo(targetVideoPath);
try {
const content = await fs.readFile(telemetryPath, "utf-8");
const parsed = JSON.parse(content);
const parsed = parseJsonWithByteOrderMark<unknown>(content);
const parsedObject =
parsed && typeof parsed === "object" && !Array.isArray(parsed)
? (parsed as { samples?: unknown })
: null;
const rawSamples = Array.isArray(parsed)
? parsed
: Array.isArray(parsed?.samples)
? parsed.samples
: Array.isArray(parsedObject?.samples)
? parsedObject.samples
: [];
const samples: CursorTelemetryPoint[] = rawSamples
+11 -10
View File
@@ -3,20 +3,21 @@ import { app, ipcMain } from "electron";
import { hideCursor } from "../../cursorHider";
import { closeCountdownWindow, createCountdownWindow, getCountdownWindow } from "../../windows";
import {
SHORTCUTS_FILE,
RECORDINGS_SETTINGS_FILE,
COUNTDOWN_SETTINGS_FILE,
RECORDINGS_SETTINGS_FILE,
SHORTCUTS_FILE,
} from "../constants";
import {
countdownTimer,
setCountdownTimer,
countdownCancelled,
setCountdownCancelled,
countdownInProgress,
setCountdownInProgress,
countdownRemaining,
countdownTimer,
setCountdownCancelled,
setCountdownInProgress,
setCountdownRemaining,
setCountdownTimer,
} from "../state";
import { parseJsonWithByteOrderMark } from "../utils";
export function registerSettingsHandlers() {
ipcMain.handle('app:getVersion', () => {
@@ -42,7 +43,7 @@ export function registerSettingsHandlers() {
ipcMain.handle('get-shortcuts', async () => {
try {
const data = await fs.readFile(SHORTCUTS_FILE, 'utf-8');
return JSON.parse(data);
return parseJsonWithByteOrderMark(data);
} catch {
return null;
}
@@ -64,7 +65,7 @@ export function registerSettingsHandlers() {
ipcMain.handle('get-recording-preferences', async () => {
try {
const content = await fs.readFile(RECORDINGS_SETTINGS_FILE, 'utf-8')
const parsed = JSON.parse(content) as Record<string, unknown>
const parsed = parseJsonWithByteOrderMark<Record<string, unknown>>(content)
return {
success: true,
microphoneEnabled: parsed.microphoneEnabled === true,
@@ -81,7 +82,7 @@ export function registerSettingsHandlers() {
let existing: Record<string, unknown> = {}
try {
const content = await fs.readFile(RECORDINGS_SETTINGS_FILE, 'utf-8')
existing = JSON.parse(content) as Record<string, unknown>
existing = parseJsonWithByteOrderMark<Record<string, unknown>>(content)
} catch {
// file doesn't exist yet
}
@@ -97,7 +98,7 @@ export function registerSettingsHandlers() {
ipcMain.handle('get-countdown-delay', async () => {
try {
const content = await fs.readFile(COUNTDOWN_SETTINGS_FILE, 'utf-8')
const parsed = JSON.parse(content) as { delay?: number }
const parsed = parseJsonWithByteOrderMark<{ delay?: number }>(content)
return { success: true, delay: parsed.delay ?? 3 }
} catch {
return { success: true, delay: 3 }
+12 -4
View File
@@ -1,15 +1,15 @@
import { createRequire } from "node:module";
import fs from "node:fs/promises";
import { createRequire } from "node:module";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { app } from "electron";
import { RECORDINGS_DIR } from "../appPaths";
import { RECORDINGS_SETTINGS_FILE, AUTO_RECORDING_PREFIX } from "./constants";
import { AUTO_RECORDING_PREFIX, RECORDINGS_SETTINGS_FILE } from "./constants";
import {
approvedLocalReadPaths,
customRecordingsDir,
setCustomRecordingsDir,
recordingsDirLoaded,
setCustomRecordingsDir,
setRecordingsDirLoaded,
} from "./state";
@@ -49,6 +49,14 @@ export function normalizeVideoSourcePath(videoPath?: string | null): string | nu
return trimmed;
}
export function stripJsonByteOrderMark(content: string) {
return content.charCodeAt(0) === 0xfeff ? content.slice(1) : content;
}
export function parseJsonWithByteOrderMark<T = unknown>(content: string): T {
return JSON.parse(stripJsonByteOrderMark(content)) as T;
}
export function parseWindowId(sourceId?: string) {
if (!sourceId) return null;
const match = sourceId.match(/^window:(\d+)/);
@@ -89,7 +97,7 @@ async function loadRecordingsDirectorySetting() {
try {
const content = await fs.readFile(RECORDINGS_SETTINGS_FILE, "utf-8");
const parsed = JSON.parse(content) as { recordingsDir?: unknown };
const parsed = parseJsonWithByteOrderMark<{ recordingsDir?: unknown }>(content);
if (typeof parsed.recordingsDir === "string" && parsed.recordingsDir.trim()) {
setCustomRecordingsDir(path.resolve(parsed.recordingsDir));
}
+12 -6
View File
@@ -892,13 +892,19 @@ app.whenReady().then(async () => {
registerExtensionIpcHandlers();
if (IS_SMOKE_EXPORT) {
if (IS_SMOKE_EXPORT || process.env.RECORDLY_DEV_OPEN_RECORDING_INPUT) {
await logSmokeExportGpuDiagnostics();
const smokeSource =
process.env.RECORDLY_SMOKE_EXPORT_PROJECT ??
process.env.RECORDLY_SMOKE_EXPORT_INPUT ??
"<missing input>";
console.log(`[smoke-export] Starting editor smoke export for ${smokeSource}`);
if (IS_SMOKE_EXPORT) {
const smokeSource =
process.env.RECORDLY_SMOKE_EXPORT_PROJECT ??
process.env.RECORDLY_SMOKE_EXPORT_INPUT ??
"<missing input>";
console.log(`[smoke-export] Starting editor smoke export for ${smokeSource}`);
} else {
console.log(
`[dev-open-recording] Starting editor for ${process.env.RECORDLY_DEV_OPEN_RECORDING_INPUT}`,
);
}
createEditorWindowWrapper();
return;
}
Binary file not shown.
@@ -5,17 +5,24 @@
"helpers": {
"wgc-capture": {
"binaryName": "wgc-capture.exe",
"binarySha256": "bb4c2aa4141e81e1a05b54bd49dbb114eb3a5ff4a666bf7d26525a9da585128b",
"binarySha256": "424c64fc9569e8f6add29130ee935dfcb4451c07c1036d3738c22d9b79ad39cd",
"sourceDir": "electron/native/wgc-capture",
"sourceFingerprint": "5bb02a69049a02909d38188cae1dfdfb94fd49465b51ed6ab78a98092b7520cc",
"updatedAt": "2026-04-25T09:17:16.237Z"
"sourceFingerprint": "4fd7e2f5d0e804a8aaaaddcba3a3e02b42739c0148a0d38be24325ddbb659ec1",
"updatedAt": "2026-05-03T15:39:44.054Z"
},
"cursor-monitor": {
"binaryName": "cursor-monitor.exe",
"binarySha256": "b0732abc06998a40c3e95078465ad750a6169901944571c39cfd7996effe39c0",
"binarySha256": "6ae6d91103b6e891a851e8ea5791e1c1f9aaab700134c18bc4c46cfffd7fdd12",
"sourceDir": "electron/native/cursor-monitor",
"sourceFingerprint": "6ad1b8b50bb336f2a48937b06f5ec56d90b6ab4a3e56a4bca278cf67a5d3e52e",
"updatedAt": "2026-03-29T02:15:38.286Z"
"updatedAt": "2026-05-03T15:39:52.446Z"
},
"recordly-gpu-export": {
"binaryName": "recordly-gpu-export.exe",
"binarySha256": "9b3d4dff520356e5db563cc3992d777f8bab8eaf6d0bb718f9c1d7d7da37fac5",
"sourceDir": "electron/native/gpu-export-probe",
"sourceFingerprint": "75bf080c4a5cbbcb1d42fb088a1545130c613e42d9e7681cc23f9151d1c8072b",
"updatedAt": "2026-05-03T15:39:47.938Z"
}
}
}
Binary file not shown.
@@ -0,0 +1,37 @@
cmake_minimum_required(VERSION 3.20)
project(gpu-export-probe LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
add_executable(gpu-export-probe
src/main.cpp
)
set(NVIDIA_VIDEO_SDK_SAMPLES_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../.tmp/video-sdk-samples/Samples")
set(NVIDIA_NVENC_SDK_DIR "${NVIDIA_VIDEO_SDK_SAMPLES_DIR}/NvCodec")
if(EXISTS "${NVIDIA_NVENC_SDK_DIR}/NvEncoder/NvEncoderD3D11.cpp")
target_sources(gpu-export-probe PRIVATE
"${NVIDIA_NVENC_SDK_DIR}/NvEncoder/NvEncoder.cpp"
"${NVIDIA_NVENC_SDK_DIR}/NvEncoder/NvEncoderD3D11.cpp"
)
target_include_directories(gpu-export-probe PRIVATE
"${NVIDIA_VIDEO_SDK_SAMPLES_DIR}"
"${NVIDIA_NVENC_SDK_DIR}"
)
target_compile_definitions(gpu-export-probe PRIVATE RECORDLY_GPU_EXPORT_ENABLE_NVENC_SDK=1)
endif()
target_compile_options(gpu-export-probe PRIVATE /EHsc /W4 /utf-8)
target_link_libraries(gpu-export-probe PRIVATE
d3d11
d3dcompiler
dxgi
mfplat
mfreadwrite
mf
mfuuid
ole32
windowscodecs
)
File diff suppressed because it is too large Load Diff
+2
View File
@@ -183,6 +183,8 @@ static void writeCompanionAudioTimingMetadata(
}
metadataFile << "{\"startDelayMs\":" << startDelayMs;
metadataFile << ",\"capturedDurationMs\":" << capture.capturedDurationMs();
metadataFile << ",\"dataBytes\":" << capture.totalDataBytes();
const uint32_t discontinuityCount = capture.dataDiscontinuityCount();
if (discontinuityCount > 0) {
metadataFile << ",\"dataDiscontinuityCount\":" << discontinuityCount;
@@ -2,9 +2,41 @@
#include <functiondiscoverykeys_devpkey.h>
#include <iostream>
#include <cstring>
#include <algorithm>
#pragma comment(lib, "ole32.lib")
namespace {
constexpr int64_t kHundredNanosecondsPerSecond = 10000000;
constexpr uint64_t kSilenceWriteChunkFrames = 4096;
bool isFloatFormat(const WAVEFORMATEX* format) {
if (!format) return false;
if (format->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) return true;
if (format->wFormatTag != WAVE_FORMAT_EXTENSIBLE) return false;
return reinterpret_cast<const WAVEFORMATEXTENSIBLE*>(format)->SubFormat ==
KSDATAFORMAT_SUBTYPE_IEEE_FLOAT;
}
bool isPcmFormat(const WAVEFORMATEX* format) {
if (!format) return false;
if (format->wFormatTag == WAVE_FORMAT_PCM) return true;
if (format->wFormatTag != WAVE_FORMAT_EXTENSIBLE) return false;
return reinterpret_cast<const WAVEFORMATEXTENSIBLE*>(format)->SubFormat ==
KSDATAFORMAT_SUBTYPE_PCM;
}
int16_t pcm24ToInt16(const BYTE* sample) {
int32_t value = static_cast<int32_t>(sample[0]) |
(static_cast<int32_t>(sample[1]) << 8) |
(static_cast<int32_t>(sample[2]) << 16);
if ((value & 0x800000) != 0) {
value |= ~0xFFFFFF;
}
return static_cast<int16_t>(value >> 8);
}
}
static const CLSID CLSID_MMDeviceEnumerator_ = __uuidof(MMDeviceEnumerator);
static const IID IID_IMMDeviceEnumerator_ = __uuidof(IMMDeviceEnumerator);
static const IID IID_IAudioClient_ = __uuidof(IAudioClient);
@@ -137,6 +169,7 @@ bool WasapiCapture::start() {
}
totalDataBytes_ = 0;
framesWritten_ = 0;
firstPacketQpcHns_ = -1;
dataDiscontinuityCount_ = 0;
timestampErrorCount_ = 0;
@@ -177,7 +210,10 @@ void WasapiCapture::stop() {
if (outputFile_ != INVALID_HANDLE_VALUE) {
SetFilePointer(outputFile_, 0, nullptr, FILE_BEGIN);
writeWavHeader(outputFile_, totalDataBytes_);
const uint64_t dataBytes = totalDataBytes_.load();
const DWORD wavDataBytes =
static_cast<DWORD>(std::min<uint64_t>(dataBytes, 0xFFFFFFFFu));
writeWavHeader(outputFile_, wavDataBytes);
CloseHandle(outputFile_);
outputFile_ = INVALID_HANDLE_VALUE;
}
@@ -217,6 +253,39 @@ bool WasapiCapture::writeWavHeader(HANDLE file, DWORD dataSize) {
return true;
}
void WasapiCapture::writePcmFrames(const int16_t* samples, UINT32 frameCount, WORD channels) {
if (!samples || frameCount == 0 || channels == 0 || outputFile_ == INVALID_HANDLE_VALUE) {
return;
}
const DWORD bytesToWrite = frameCount * channels * sizeof(int16_t);
DWORD written = 0;
WriteFile(outputFile_, samples, bytesToWrite, &written, nullptr);
totalDataBytes_.fetch_add(written);
framesWritten_.fetch_add(written / (channels * sizeof(int16_t)));
}
void WasapiCapture::writeSilenceFrames(uint64_t frameCount, WORD channels) {
if (frameCount == 0 || channels == 0 || outputFile_ == INVALID_HANDLE_VALUE) {
return;
}
std::vector<int16_t> silence(static_cast<size_t>(kSilenceWriteChunkFrames * channels), 0);
while (frameCount > 0) {
const uint64_t chunkFrames = std::min<uint64_t>(frameCount, kSilenceWriteChunkFrames);
writePcmFrames(silence.data(), static_cast<UINT32>(chunkFrames), channels);
frameCount -= chunkFrames;
}
}
uint64_t WasapiCapture::capturedDurationMs() const {
if (!mixFormat_ || mixFormat_->nSamplesPerSec == 0) {
return 0;
}
return (framesWritten_.load() * 1000) / mixFormat_->nSamplesPerSec;
}
void WasapiCapture::captureThread() {
// COM must be initialized on every thread that uses COM objects.
// The main thread calls winrt::init_apartment(MTA) but that only covers
@@ -225,9 +294,12 @@ void WasapiCapture::captureThread() {
CoInitializeEx(nullptr, COINIT_MULTITHREADED);
WORD channels = static_cast<WORD>(mixFormat_->nChannels);
bool isFloat = (mixFormat_->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) ||
(mixFormat_->wFormatTag == WAVE_FORMAT_EXTENSIBLE &&
reinterpret_cast<WAVEFORMATEXTENSIBLE*>(mixFormat_)->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT);
const bool isFloat = isFloatFormat(mixFormat_);
const bool isPcm = isPcmFormat(mixFormat_);
const WORD bitsPerSample = mixFormat_->wBitsPerSample;
const WORD sourceBlockAlign = mixFormat_->nBlockAlign;
const WORD sourceBytesPerSample =
channels > 0 ? static_cast<WORD>(sourceBlockAlign / channels) : 0;
std::vector<int16_t> pcmBuffer;
@@ -273,38 +345,96 @@ void WasapiCapture::captureThread() {
if ((flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) != 0) {
timestampErrorCount_.fetch_add(1);
}
if (
const bool hasReliableTimestamp =
numFrames > 0 &&
qpcPosition > 0 &&
(flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) == 0
) {
(flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) == 0;
if (hasReliableTimestamp) {
int64_t expected = -1;
firstPacketQpcHns_.compare_exchange_strong(
expected,
static_cast<int64_t>(qpcPosition));
const int64_t firstPacketQpcHns = firstPacketQpcHns_.load();
if (
firstPacketQpcHns >= 0 &&
static_cast<int64_t>(qpcPosition) > firstPacketQpcHns
) {
const int64_t elapsedHns =
static_cast<int64_t>(qpcPosition) - firstPacketQpcHns;
const uint64_t expectedStartFrame =
(static_cast<uint64_t>(elapsedHns) * mixFormat_->nSamplesPerSec +
kHundredNanosecondsPerSecond / 2) /
kHundredNanosecondsPerSecond;
const uint64_t writtenFrames = framesWritten_.load();
const uint64_t gapThresholdFrames = mixFormat_->nSamplesPerSec / 100;
if (expectedStartFrame > writtenFrames + gapThresholdFrames) {
writeSilenceFrames(expectedStartFrame - writtenFrames, channels);
}
}
}
UINT32 totalSamples = numFrames * channels;
if (flags & AUDCLNT_BUFFERFLAGS_SILENT) {
pcmBuffer.assign(totalSamples, 0);
} else if (isFloat) {
} else if (isFloat && bitsPerSample == 32 && sourceBytesPerSample >= 4) {
pcmBuffer.resize(totalSamples);
const float* src = reinterpret_cast<const float*>(data);
for (UINT32 i = 0; i < totalSamples; i++) {
pcmBuffer[i] = floatToInt16(src[i]);
}
} else {
} else if (isPcm && bitsPerSample == 16 && sourceBytesPerSample >= 2) {
pcmBuffer.resize(totalSamples);
std::memcpy(pcmBuffer.data(), data, totalSamples * sizeof(int16_t));
if (sourceBytesPerSample == sizeof(int16_t)) {
std::memcpy(pcmBuffer.data(), data, totalSamples * sizeof(int16_t));
} else {
for (UINT32 frame = 0; frame < numFrames; frame++) {
const BYTE* frameData = data + frame * sourceBlockAlign;
for (WORD channel = 0; channel < channels; channel++) {
const BYTE* sample = frameData + channel * sourceBytesPerSample;
pcmBuffer[frame * channels + channel] =
*reinterpret_cast<const int16_t*>(sample);
}
}
}
} else if (isPcm && bitsPerSample == 24 && sourceBytesPerSample >= 3) {
pcmBuffer.resize(totalSamples);
for (UINT32 frame = 0; frame < numFrames; frame++) {
const BYTE* frameData = data + frame * sourceBlockAlign;
for (WORD channel = 0; channel < channels; channel++) {
const BYTE* sample = frameData + channel * sourceBytesPerSample;
pcmBuffer[frame * channels + channel] = pcm24ToInt16(sample);
}
}
} else if (isPcm && bitsPerSample == 32 && sourceBytesPerSample >= 4) {
pcmBuffer.resize(totalSamples);
for (UINT32 frame = 0; frame < numFrames; frame++) {
const BYTE* frameData = data + frame * sourceBlockAlign;
for (WORD channel = 0; channel < channels; channel++) {
const BYTE* sample = frameData + channel * sourceBytesPerSample;
const int32_t value = *reinterpret_cast<const int32_t*>(sample);
pcmBuffer[frame * channels + channel] = static_cast<int16_t>(value >> 16);
}
}
} else if (isPcm && bitsPerSample == 8 && sourceBytesPerSample >= 1) {
pcmBuffer.resize(totalSamples);
for (UINT32 frame = 0; frame < numFrames; frame++) {
const BYTE* frameData = data + frame * sourceBlockAlign;
for (WORD channel = 0; channel < channels; channel++) {
const BYTE value = *(frameData + channel * sourceBytesPerSample);
pcmBuffer[frame * channels + channel] =
static_cast<int16_t>((static_cast<int>(value) - 128) << 8);
}
}
} else {
pcmBuffer.assign(totalSamples, 0);
}
captureClient_->ReleaseBuffer(numFrames);
DWORD bytesToWrite = totalSamples * sizeof(int16_t);
DWORD written;
WriteFile(outputFile_, pcmBuffer.data(), bytesToWrite, &written, nullptr);
totalDataBytes_ += written;
writePcmFrames(pcmBuffer.data(), numFrames, channels);
hr = captureClient_->GetNextPacketSize(&packetLength);
if (FAILED(hr)) {
@@ -20,6 +20,8 @@ public:
bool resume();
void stop();
int64_t firstPacketQpcHns() const { return firstPacketQpcHns_.load(); }
uint64_t capturedDurationMs() const;
uint64_t totalDataBytes() const { return totalDataBytes_.load(); }
uint32_t dataDiscontinuityCount() const { return dataDiscontinuityCount_.load(); }
uint32_t timestampErrorCount() const { return timestampErrorCount_.load(); }
@@ -27,6 +29,8 @@ private:
bool initializeCommon();
void captureThread();
bool writeWavHeader(HANDLE file, DWORD dataSize);
void writePcmFrames(const int16_t* samples, UINT32 frameCount, WORD channels);
void writeSilenceFrames(uint64_t frameCount, WORD channels);
IMMDevice* findCaptureDeviceByName(const std::wstring& name);
std::string outputPath_;
@@ -34,7 +38,8 @@ private:
std::atomic<bool> capturing_{false};
std::atomic<bool> paused_{false};
HANDLE outputFile_ = INVALID_HANDLE_VALUE;
DWORD totalDataBytes_ = 0;
std::atomic<uint64_t> totalDataBytes_{0};
std::atomic<uint64_t> framesWritten_{0};
IMMDeviceEnumerator* enumerator_ = nullptr;
IMMDevice* device_ = nullptr;
+166
View File
@@ -10,6 +10,84 @@ type NativeVideoAudioMuxMetrics = {
tempEditedAudioBytes?: number;
muxedVideoBytes?: number;
};
type WindowsGpuExportSummary = {
success?: boolean;
width?: number;
height?: number;
fps?: number;
seconds?: number;
mediaMs?: number;
frames?: number;
cursorOverlay?: boolean;
zoomOverlay?: boolean;
adapterVendorId?: number;
adapterDeviceId?: number;
adapterDedicatedVideoMemoryMB?: number;
initializeMs?: number;
initCoInitializeMs?: number;
initMfStartupMs?: number;
initD3DDeviceMs?: number;
initSourceReaderMs?: number;
initWebcamReaderMs?: number;
initVideoProcessorMs?: number;
initTexturesMs?: number;
initShaderPipelineMs?: number;
initSinkWriterMs?: number;
totalMs?: number;
readMs?: number;
clearMs?: number;
videoProcessMs?: number;
writeSampleMs?: number;
finalizeMs?: number;
realtimeMultiplier?: number;
};
type NativeStaticLayoutChunkMetric = {
index: number;
startSec: number;
durationSec: number;
backend:
| "cuda-overlay"
| "cuda-scale-cpu-pad"
| "cuda-static-composite"
| "nvidia-cuda-compositor"
| "windows-d3d11-compositor";
elapsedMs: number;
outputBytes: number;
fallbackReason?: string;
windowsGpuSummary?: WindowsGpuExportSummary;
};
type NativeStaticLayoutMetrics = NativeVideoAudioMuxMetrics & {
chunkCount: number;
chunkDurationSec: number;
chunkExecMs: number;
concatExecMs?: number;
staticAssetExecMs?: number;
fallbackChunkCount: number;
videoOnlyBytes?: number;
chunks: NativeStaticLayoutChunkMetric[];
};
type NativeStaticLayoutProgress = {
sessionId?: string;
backend?: NativeStaticLayoutChunkMetric["backend"];
elapsedMs?: number;
averageFps?: number;
currentFrame: number;
totalFrames: number;
percentage: number;
};
type NativeVideoMetadataProbe = {
width: number;
height: number;
duration: number;
mediaStartTime?: number;
streamStartTime?: number;
streamDuration?: number;
frameRate: number;
codec: string;
hasAudio: boolean;
audioCodec?: string;
audioSampleRate?: number;
};
const nativeVideoExportWriteRequests = new Map<
number,
@@ -114,6 +192,93 @@ contextBridge.exposeInMainWorld("electronAPI", {
generateWallpaperThumbnail: (filePath: string) => {
return ipcRenderer.invoke("generate-wallpaper-thumbnail", filePath);
},
probeNativeVideoMetadata: (filePath: string) => {
return ipcRenderer.invoke("probe-native-video-metadata", filePath) as Promise<{
success: boolean;
metadata?: NativeVideoMetadataProbe;
error?: string;
}>;
},
nativeStaticLayoutExport: (options: {
sessionId?: string;
inputPath: string;
width: number;
height: number;
frameRate: number;
bitrate: number;
encodingMode: "fast" | "balanced" | "quality";
durationSec: number;
contentWidth: number;
contentHeight: number;
offsetX: number;
offsetY: number;
backgroundColor: string;
backgroundImagePath?: string | null;
borderRadius?: number;
shadowIntensity?: number;
webcamInputPath?: string | null;
webcamLeft?: number;
webcamTop?: number;
webcamSize?: number;
webcamRadius?: number;
webcamShadowIntensity?: number;
webcamMirror?: boolean;
webcamTimeOffsetMs?: number;
cursorTelemetry?: Array<{
timeMs: number;
cx: number;
cy: number;
cursorTypeIndex?: number;
bounceScale?: number;
}>;
cursorSize?: number;
cursorAtlasPngDataUrl?: string | null;
cursorAtlasEntries?: Array<{
index: number;
x: number;
y: number;
width: number;
height: number;
anchorX: number;
anchorY: number;
aspectRatio: number;
}>;
zoomTelemetry?: Array<{ timeMs: number; scale: number; x: number; y: number }>;
chunkDurationSec?: number;
experimentalWindowsGpuCompositor?: boolean;
audioOptions?: {
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
audioSourcePath?: string | null;
audioSourceSampleRate?: number;
outputDurationSec?: number;
trimSegments?: Array<{ startMs: number; endMs: number }>;
editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
editedAudioData?: ArrayBuffer;
editedAudioMimeType?: string | null;
};
}) => {
return ipcRenderer.invoke("native-static-layout-export", options) as Promise<{
success: boolean;
tempPath?: string;
encoderName?: string;
error?: string;
metrics?: NativeStaticLayoutMetrics;
}>;
},
nativeStaticLayoutExportCancel: (sessionId: string) => {
return ipcRenderer.invoke("native-static-layout-export-cancel", sessionId) as Promise<{
success: boolean;
}>;
},
onNativeStaticLayoutExportProgress: (
callback: (progress: NativeStaticLayoutProgress) => void,
) => {
const listener = (_event: Electron.IpcRendererEvent, payload: NativeStaticLayoutProgress) =>
callback(payload);
ipcRenderer.on("native-static-layout-export-progress", listener);
return () => ipcRenderer.removeListener("native-static-layout-export-progress", listener);
},
nativeVideoExportStart: (options: {
width: number;
height: number;
@@ -147,6 +312,7 @@ contextBridge.exposeInMainWorld("electronAPI", {
audioMode?: "none" | "copy-source" | "trim-source" | "edited-track";
audioSourcePath?: string | null;
audioSourceSampleRate?: number;
outputDurationSec?: number;
trimSegments?: Array<{ startMs: number; endMs: number }>;
editedTrackStrategy?: "filtergraph-fast-path" | "offline-render-fallback";
editedTrackSegments?: Array<{ startMs: number; endMs: number; speed: number }>;
+10
View File
@@ -48,6 +48,13 @@ function getEditorWindowQuery(): Record<string, string> {
windowType: "editor",
};
if (process.env.RECORDLY_DEV_OPEN_RECORDING_INPUT) {
query.devOpenInput = process.env.RECORDLY_DEV_OPEN_RECORDING_INPUT;
}
if (process.env.RECORDLY_DEV_OPEN_RECORDING_WEBCAM) {
query.devOpenWebcam = process.env.RECORDLY_DEV_OPEN_RECORDING_WEBCAM;
}
if (process.env.RECORDLY_SMOKE_EXPORT === "1") {
query.smokeExport = "1";
if (process.env.RECORDLY_SMOKE_EXPORT_INPUT) {
@@ -80,6 +87,9 @@ function getEditorWindowQuery(): Record<string, string> {
if (process.env.RECORDLY_SMOKE_EXPORT_BACKEND) {
query.smokeBackendPreference = process.env.RECORDLY_SMOKE_EXPORT_BACKEND;
}
if (process.env.RECORDLY_SMOKE_EXPORT_RENDER_BACKEND) {
query.smokeRenderBackend = process.env.RECORDLY_SMOKE_EXPORT_RENDER_BACKEND;
}
if (process.env.RECORDLY_SMOKE_EXPORT_MAX_ENCODE_QUEUE) {
query.smokeMaxEncodeQueue = process.env.RECORDLY_SMOKE_EXPORT_MAX_ENCODE_QUEUE;
}
+2 -1
View File
@@ -25,7 +25,8 @@
"rebuild:native": "node ./node_modules/@electron/rebuild/lib/cli.js --force --only uiohook-napi",
"build:native-helpers": "node scripts/build-native-helpers.mjs",
"build:whisper-runtime": "node scripts/build-whisper-runtime.mjs",
"build:platform-native-helpers": "npm run build:native-helpers && npm run build:windows-capture && npm run build:cursor-monitor && npm run build:whisper-runtime",
"build:platform-native-helpers": "npm run build:native-helpers && npm run build:windows-capture && npm run build:windows-gpu-export && npm run build:cursor-monitor && npm run build:whisper-runtime",
"build:windows-gpu-export": "node scripts/build-windows-gpu-export.mjs",
"build:windows-capture": "node scripts/build-windows-capture.mjs",
"build:cursor-monitor": "node scripts/build-cursor-monitor.mjs",
"build:mac": "npm run build:platform-native-helpers && tsc && vite build --config vite.config.ts && electron-builder --mac",
+71 -13
View File
@@ -11,7 +11,7 @@ import ffmpegStatic from "ffmpeg-static";
const execFileAsync = promisify(execFile);
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const repoRoot = path.resolve(__dirname, "..");
const mainEntry = path.join(repoRoot, "dist-electron", "main.js");
const mainEntry = path.join(repoRoot, "dist-electron", "main.cjs");
const rendererEntry = path.join(repoRoot, "dist", "index.html");
const width = parseEvenInteger(process.env.RECORDLY_BENCH_EXPORT_WIDTH ?? "1280", "Width");
@@ -28,9 +28,13 @@ const timeoutMs = parsePositiveInteger(
const runsPerVariant = parsePositiveInteger(process.env.RECORDLY_BENCH_EXPORT_RUNS ?? "2", "Runs");
const useNativeExport = process.env.RECORDLY_BENCH_EXPORT_USE_NATIVE === "1";
const useWebcamOverlay = process.env.RECORDLY_BENCH_EXPORT_ENABLE_WEBCAM === "1";
const providedInputPath = process.env.RECORDLY_BENCH_EXPORT_INPUT ?? null;
const providedWebcamInputPath = process.env.RECORDLY_BENCH_EXPORT_WEBCAM_INPUT ?? null;
const keepTempArtifacts = process.env.RECORDLY_BENCH_EXPORT_KEEP_TEMP === "1";
const exportEncodingMode = parseExportEncodingMode(
process.env.RECORDLY_BENCH_EXPORT_ENCODING_MODE ?? null,
);
const exportQuality = parseExportQuality(process.env.RECORDLY_BENCH_EXPORT_QUALITY ?? null);
const exportShadowIntensity = parseExportShadowIntensity(
process.env.RECORDLY_BENCH_EXPORT_SHADOW_INTENSITY ?? null,
);
@@ -49,6 +53,9 @@ const webcamSize = parseExportWebcamSize(process.env.RECORDLY_BENCH_EXPORT_WEBCA
const MODERN_BACKEND_SWEEP = ["auto", "webcodecs", "breeze"];
const exportPipeline = parseExportPipeline(process.env.RECORDLY_BENCH_EXPORT_PIPELINE ?? null);
const exportBackend = parseExportBackend(process.env.RECORDLY_BENCH_EXPORT_BACKEND ?? null);
const exportRenderBackend = parseRenderBackend(
process.env.RECORDLY_BENCH_EXPORT_RENDER_BACKEND ?? null,
);
const exportBackendList = parseExportBackendList(
process.env.RECORDLY_BENCH_EXPORT_BACKENDS ?? null,
);
@@ -113,6 +120,18 @@ function parseExportBackend(rawValue) {
throw new Error("RECORDLY_BENCH_EXPORT_BACKEND must be 'auto', 'webcodecs', or 'breeze'");
}
function parseRenderBackend(rawValue) {
if (rawValue === null || rawValue === "") {
return null;
}
if (rawValue === "webgl" || rawValue === "webgpu") {
return rawValue;
}
throw new Error("RECORDLY_BENCH_EXPORT_RENDER_BACKEND must be 'webgl' or 'webgpu'");
}
function parseExportBackendList(rawValue) {
if (rawValue === null || rawValue === "") {
return null;
@@ -177,6 +196,18 @@ function parseExportEncodingMode(rawValue) {
throw new Error("RECORDLY_BENCH_EXPORT_ENCODING_MODE must be 'fast', 'balanced', or 'quality'");
}
function parseExportQuality(rawValue) {
if (rawValue === null || rawValue === "") {
return null;
}
if (rawValue === "medium" || rawValue === "good" || rawValue === "high" || rawValue === "source") {
return rawValue;
}
throw new Error("RECORDLY_BENCH_EXPORT_QUALITY must be 'medium', 'good', 'high', or 'source'");
}
function parseExportShadowIntensity(rawValue) {
if (rawValue === null || rawValue === "") {
return null;
@@ -477,8 +508,10 @@ function buildRequestedConfigRows(benchmarkRequests) {
{ key: "Runs per variant", value: runsPerVariant },
{ key: "Pipeline", value: exportPipeline ?? "default" },
{ key: "Requested backends", value: benchmarkRequests.map((request) => request.label) },
{ key: "Render backend", value: exportRenderBackend ?? "default" },
{ key: "Backend sweep", value: formatBoolean(benchmarkRequests.length > 1) },
{ key: "Encoding mode", value: exportEncodingMode ?? "default" },
{ key: "Quality", value: exportQuality ?? "default" },
{ key: "Shadow intensity", value: exportShadowIntensity ?? "default" },
{ key: "Webcam enabled", value: formatBoolean(useWebcamOverlay) },
{ key: "Experimental native override", value: formatBoolean(useNativeExport) },
@@ -665,6 +698,7 @@ async function runVariant(
...(exportEncodingMode
? { RECORDLY_SMOKE_EXPORT_ENCODING_MODE: exportEncodingMode }
: {}),
...(exportQuality ? { RECORDLY_SMOKE_EXPORT_QUALITY: exportQuality } : {}),
...(exportShadowIntensity !== null
? { RECORDLY_SMOKE_EXPORT_SHADOW_INTENSITY: String(exportShadowIntensity) }
: {}),
@@ -681,6 +715,9 @@ async function runVariant(
...(benchmarkRequest.backend
? { RECORDLY_SMOKE_EXPORT_BACKEND: benchmarkRequest.backend }
: {}),
...(exportRenderBackend
? { RECORDLY_SMOKE_EXPORT_RENDER_BACKEND: exportRenderBackend }
: {}),
...(typeof variant.maxEncodeQueue === "number"
? { RECORDLY_SMOKE_EXPORT_MAX_ENCODE_QUEUE: String(variant.maxEncodeQueue) }
: {}),
@@ -869,28 +906,45 @@ async function main() {
requestedPipeline: exportPipeline,
requestedBackend: exportBackend,
requestedBackends: benchmarkRequests.map((request) => request.label),
requestedRenderBackend: exportRenderBackend,
backendSweepEnabled: benchmarkRequests.length > 1,
requestedEncodingMode: exportEncodingMode,
requestedQuality: exportQuality,
requestedShadowIntensity: exportShadowIntensity,
webcamEnabled: useWebcamOverlay,
providedInput: providedInputPath,
providedWebcamInput: providedWebcamInputPath,
keepTempArtifacts,
requestedWebcamShadowIntensity: webcamShadowIntensity,
requestedWebcamSize: webcamSize,
}),
);
printRequestedConfigTable(benchmarkRequests);
console.log(`[benchmark-export-queues] Generating fixture video: ${inputPath}`);
await createFixtureVideo(ffmpegStatic, inputPath);
if (providedInputPath) {
console.log(`[benchmark-export-queues] Using provided input video: ${providedInputPath}`);
await fs.copyFile(providedInputPath, inputPath);
} else {
console.log(`[benchmark-export-queues] Generating fixture video: ${inputPath}`);
await createFixtureVideo(ffmpegStatic, inputPath);
}
if (webcamInputPath) {
console.log(
`[benchmark-export-queues] Generating webcam fixture video: ${webcamInputPath}`,
);
await createFixtureVideo(ffmpegStatic, webcamInputPath, {
fixtureWidth: webcamWidth,
fixtureHeight: webcamHeight,
includeAudio: false,
videoFilter: `testsrc=size=${webcamWidth}x${webcamHeight}:rate=${frameRate}`,
});
if (providedWebcamInputPath) {
console.log(
`[benchmark-export-queues] Using provided webcam video: ${providedWebcamInputPath}`,
);
await fs.copyFile(providedWebcamInputPath, webcamInputPath);
} else {
console.log(
`[benchmark-export-queues] Generating webcam fixture video: ${webcamInputPath}`,
);
await createFixtureVideo(ffmpegStatic, webcamInputPath, {
fixtureWidth: webcamWidth,
fixtureHeight: webcamHeight,
includeAudio: false,
videoFilter: `testsrc=size=${webcamWidth}x${webcamHeight}:rate=${frameRate}`,
});
}
}
const benchmarkResults = [];
@@ -968,7 +1022,11 @@ async function main() {
);
}
} finally {
await fs.rm(tempDir, { recursive: true, force: true });
if (keepTempArtifacts) {
console.log(`[benchmark-export-queues] Preserved temp artifacts: ${tempDir}`);
} else {
await fs.rm(tempDir, { recursive: true, force: true });
}
}
}
+166
View File
@@ -0,0 +1,166 @@
import { execSync } from "node:child_process";
import { copyFileSync, existsSync, mkdirSync, rmSync } from "node:fs";
import path from "node:path";
import {
formatNativeHelperManifestWarning,
updateNativeHelperManifest,
verifyNativeHelperManifest,
} from "./native-helper-manifest.mjs";
const projectRoot = process.cwd();
const sourceDir = path.join(projectRoot, "electron", "native", "gpu-export-probe");
const buildDir = path.join(sourceDir, "build");
const bundledDir = path.join(
projectRoot,
"electron",
"native",
"bin",
process.arch === "arm64" ? "win32-arm64" : "win32-x64",
);
const bundledExePath = path.join(bundledDir, "recordly-gpu-export.exe");
const helperId = "recordly-gpu-export";
const generatorArch = process.arch === "arm64" ? "ARM64" : "x64";
if (process.platform !== "win32") {
console.log("[build-windows-gpu-export] Skipping Windows GPU export helper build.");
process.exit(0);
}
if (!existsSync(path.join(sourceDir, "CMakeLists.txt"))) {
console.error("[build-windows-gpu-export] CMakeLists.txt not found at", sourceDir);
process.exit(1);
}
function findCmake() {
try {
execSync("cmake --version", { stdio: "pipe" });
return "cmake";
} catch {
// Continue probing common Windows install locations.
}
const standaloneCmakePaths = [
path.join("C:", "Program Files", "CMake", "bin", "cmake.exe"),
path.join("C:", "Program Files (x86)", "CMake", "bin", "cmake.exe"),
];
for (const cmakePath of standaloneCmakePaths) {
if (existsSync(cmakePath)) {
return `"${cmakePath}"`;
}
}
const vsRoots = [
path.join("C:", "Program Files", "Microsoft Visual Studio"),
path.join("C:", "Program Files (x86)", "Microsoft Visual Studio"),
];
const vsEditions = ["Community", "Professional", "Enterprise", "BuildTools"];
const vsVersions = ["2022", "2019"];
for (const root of vsRoots) {
for (const version of vsVersions) {
for (const edition of vsEditions) {
const cmakePath = path.join(
root,
version,
edition,
"Common7",
"IDE",
"CommonExtensions",
"Microsoft",
"CMake",
"CMake",
"bin",
"cmake.exe",
);
if (existsSync(cmakePath)) {
return `"${cmakePath}"`;
}
}
}
}
return null;
}
const cmake = findCmake();
if (!cmake) {
if (existsSync(bundledExePath)) {
const verification = verifyNativeHelperManifest({
projectRoot,
helperId,
sourceDir,
binaryPath: bundledExePath,
binaryName: "recordly-gpu-export.exe",
});
if (!verification.ok) {
console.warn(formatNativeHelperManifestWarning("build-windows-gpu-export", verification));
}
console.log(`[build-windows-gpu-export] Using bundled helper: ${bundledExePath}`);
process.exit(0);
}
console.error(
"[build-windows-gpu-export] CMake not found. Install Visual Studio with C++ CMake tools or standalone CMake.",
);
process.exit(1);
}
mkdirSync(buildDir, { recursive: true });
function clearCmakeCache() {
rmSync(path.join(buildDir, "CMakeCache.txt"), { force: true });
rmSync(path.join(buildDir, "CMakeFiles"), { recursive: true, force: true });
}
console.log("[build-windows-gpu-export] Configuring CMake...");
try {
clearCmakeCache();
execSync(`${cmake} .. -G "Visual Studio 17 2022" -A ${generatorArch}`, {
cwd: buildDir,
stdio: "inherit",
timeout: 120000,
});
} catch {
console.log("[build-windows-gpu-export] VS 2022 generator not found, trying VS 2019...");
try {
clearCmakeCache();
execSync(`${cmake} .. -G "Visual Studio 16 2019" -A ${generatorArch}`, {
cwd: buildDir,
stdio: "inherit",
timeout: 120000,
});
} catch (error) {
console.error("[build-windows-gpu-export] CMake configure failed:", error.message);
process.exit(1);
}
}
console.log("[build-windows-gpu-export] Building Windows GPU export helper...");
try {
execSync(`${cmake} --build . --config Release`, {
cwd: buildDir,
stdio: "inherit",
timeout: 300000,
});
} catch (error) {
console.error("[build-windows-gpu-export] Build failed:", error.message);
process.exit(1);
}
const exePath = path.join(buildDir, "Release", "gpu-export-probe.exe");
if (!existsSync(exePath)) {
console.error("[build-windows-gpu-export] Expected exe not found at", exePath);
process.exit(1);
}
mkdirSync(bundledDir, { recursive: true });
copyFileSync(exePath, bundledExePath);
console.log(`[build-windows-gpu-export] Staged bundled helper: ${bundledExePath}`);
const manifestPath = updateNativeHelperManifest({
projectRoot,
helperId,
sourceDir,
binaryPath: bundledExePath,
binaryName: "recordly-gpu-export.exe",
});
console.log(`[build-windows-gpu-export] Updated helper manifest: ${manifestPath}`);
+5
View File
@@ -166,6 +166,11 @@ function getExpectedNativeHelperFiles(archTag) {
label: "Windows cursor monitor helper",
executable: true,
},
{
name: "recordly-gpu-export.exe",
label: "Windows GPU export helper",
executable: true,
},
{ name: "helpers-manifest.json", label: "Windows helper manifest" },
{ name: "whisper-cli.exe", label: "Whisper CLI runtime", executable: true },
{ name: "whisper-runtime.json", label: "Whisper runtime manifest" },
+134 -53
View File
@@ -54,6 +54,7 @@ import {
type ExportPipelineModel,
type ExportProgress,
type ExportQuality,
type ExportRenderBackend,
type ExportSettings,
FrameRenderer,
GIF_SIZE_PRESETS,
@@ -66,6 +67,7 @@ import {
type SupportedMp4Dimensions,
VideoExporter,
} from "@/lib/exporter";
import { getMp4ExportBitrate, getSourceQualityBitrate } from "@/lib/exporter/exportBitrate";
import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource";
import { resolveSourceAudioFallbackPaths } from "@/lib/exporter/sourceAudioFallback";
import {
@@ -230,6 +232,7 @@ type SmokeExportConfig = {
webcamSize?: number;
pipelineModel?: ExportPipelineModel;
backendPreference?: ExportBackendPreference;
renderBackend?: ExportRenderBackend;
maxEncodeQueue?: number;
maxDecodeQueue?: number;
maxPendingFrames?: number;
@@ -245,6 +248,11 @@ type SaveProjectOptions = {
captureThumbnail?: boolean;
};
type DevOpenRecordingConfig = {
inputPath: string | null;
webcamInputPath: string | null;
};
async function writeSmokeExportReport(
outputPath: string | null,
report: Record<string, unknown>,
@@ -268,22 +276,11 @@ async function writeSmokeExportReport(
}
}
const SMOKE_EXPORT_READY_TIMEOUT_MS = 30_000;
const DEFAULT_MP4_EXPORT_FRAME_RATE: ExportMp4FrameRate = 30;
const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error";
const PROJECT_AUTOSAVE_DELAY_MS = 1000;
function getEncodingModeBitrateMultiplier(encodingMode: ExportEncodingMode): number {
switch (encodingMode) {
case "fast":
return 0.1;
case "quality":
return 0.9;
case "balanced":
default:
return 0.5;
}
}
function summarizeErrorMessage(message: string): string {
const firstLine = message
.split(/\r?\n/)
@@ -328,6 +325,10 @@ function parseSmokeExportFps(value: string | null): ExportMp4FrameRate | undefin
return isValidMp4FrameRate(parsed) ? parsed : undefined;
}
function parseSmokeRenderBackend(value: string | null): ExportRenderBackend | undefined {
return value === "webgl" || value === "webgpu" ? value : undefined;
}
function getSmokeExportConfig(search: string): SmokeExportConfig {
const params = new URLSearchParams(search);
const enabled = params.get("smokeExport") === "1";
@@ -369,6 +370,7 @@ function getSmokeExportConfig(search: string): SmokeExportConfig {
: enabled && params.get("smokeBackendPreference") === "breeze"
? "breeze"
: undefined,
renderBackend: enabled ? parseSmokeRenderBackend(params.get("smokeRenderBackend")) : undefined,
maxEncodeQueue: enabled
? parseSmokeExportNumber(params.get("smokeMaxEncodeQueue"))
: undefined,
@@ -384,6 +386,14 @@ function getSmokeExportConfig(search: string): SmokeExportConfig {
};
}
function getDevOpenRecordingConfig(search: string): DevOpenRecordingConfig {
const params = new URLSearchParams(search);
return {
inputPath: params.get("devOpenInput"),
webcamInputPath: params.get("devOpenWebcam"),
};
}
function isComparableObject(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
@@ -496,17 +506,6 @@ function calculateMp4ExportDimensions(
};
}
function getSourceQualityBitrate(width: number, height: number): number {
const totalPixels = width * height;
if (totalPixels > 2560 * 1440) {
return 80_000_000;
}
if (totalPixels > 1920 * 1080) {
return 50_000_000;
}
return 30_000_000;
}
function getErrorMessage(error: unknown): string {
if (error instanceof Error) {
return error.message;
@@ -525,6 +524,13 @@ export default function VideoEditor() {
() => getSmokeExportConfig(typeof window === "undefined" ? "" : window.location.search),
[],
);
const devOpenRecordingConfig = useMemo(
() =>
getDevOpenRecordingConfig(
typeof window === "undefined" ? "" : window.location.search,
),
[],
);
const [appPlatform, setAppPlatform] = useState<string>(
typeof navigator !== "undefined" && /Mac/i.test(navigator.platform) ? "darwin" : "",
);
@@ -721,6 +727,7 @@ export default function VideoEditor() {
const smokeExportStartedRef = useRef(false);
const projectAutosaveTimeoutRef = useRef<number | null>(null);
const projectSaveQueueRef = useRef<Promise<unknown>>(Promise.resolve());
const smokeExportReadyStateRef = useRef<Record<string, unknown>>({});
const [historyVersion, setHistoryVersion] = useState(0);
const timelineRef = useRef<TimelineEditorHandle>(null);
@@ -2161,6 +2168,29 @@ export default function VideoEditor() {
return;
}
if (!smokeExportConfig.enabled && devOpenRecordingConfig.inputPath) {
const sourcePath = fromFileUrl(devOpenRecordingConfig.inputPath);
const sourceVideoUrl = await resolveVideoUrl(sourcePath);
const webcamSourcePath = devOpenRecordingConfig.webcamInputPath
? fromFileUrl(devOpenRecordingConfig.webcamInputPath)
: null;
setVideoSourcePath(sourcePath);
setVideoPath(sourceVideoUrl);
setCurrentProjectPath(null);
setLastSavedSnapshot(null);
pendingFreshRecordingAutoZoomPathRef.current = autoApplyFreshRecordingAutoZooms
? sourceVideoUrl
: null;
setWebcam((prev) => ({
...prev,
enabled: Boolean(webcamSourcePath),
sourcePath: webcamSourcePath,
timeOffsetMs: DEFAULT_WEBCAM_TIME_OFFSET_MS,
}));
setError(null);
return;
}
if (smokeExportConfig.enabled) {
if (!smokeExportConfig.inputPath) {
setError("Smoke export input path is missing.");
@@ -2275,6 +2305,8 @@ export default function VideoEditor() {
}, [
applyLoadedProject,
autoApplyFreshRecordingAutoZooms,
devOpenRecordingConfig.inputPath,
devOpenRecordingConfig.webcamInputPath,
initialEditorPreferences,
smokeExportConfig.enabled,
smokeExportConfig.inputPath,
@@ -4429,9 +4461,14 @@ export default function VideoEditor() {
? (smokeExportConfig.pipelineModel ??
(smokeExportConfig.useNativeExport ? "modern" : "legacy"))
: (settings.pipelineModel ?? exportPipelineModel);
const useExperimentalNativeExport =
pipelineModel === "modern" &&
(smokeExportConfig.enabled ? smokeExportConfig.useNativeExport : true);
const backendPreference =
pipelineModel === "legacy"
? "webcodecs"
: useExperimentalNativeExport
? "auto"
: smokeExportConfig.enabled
? (smokeExportConfig.backendPreference ??
(smokeExportConfig.useNativeExport ? "breeze" : "webcodecs"))
@@ -4444,33 +4481,14 @@ export default function VideoEditor() {
supportedSourceDimensions.height,
quality,
);
let bitrate: number;
if (quality === "source") {
// Calculate visually lossless bitrate matching screen recording optimization
const totalPixels = exportWidth * exportHeight;
bitrate = 30_000_000;
if (totalPixels > 1920 * 1080 && totalPixels <= 2560 * 1440) {
bitrate = 50_000_000;
} else if (totalPixels > 2560 * 1440) {
bitrate = 80_000_000;
}
} else {
// Adjust bitrate for lower resolutions
const totalPixels = exportWidth * exportHeight;
if (totalPixels <= 1280 * 720) {
bitrate = 10_000_000;
} else if (totalPixels <= 1920 * 1080) {
bitrate = 20_000_000;
} else {
bitrate = 30_000_000;
}
}
bitrate = Math.max(
2_000_000,
Math.round(bitrate * getEncodingModeBitrateMultiplier(encodingMode)),
);
const bitrate = getMp4ExportBitrate({
width: exportWidth,
height: exportHeight,
frameRate: selectedMp4FrameRate,
quality,
encodingMode,
useModernNativeStaticLayout: useExperimentalNativeExport,
});
const exporterConfig = {
videoUrl: videoPath,
@@ -4481,7 +4499,8 @@ export default function VideoEditor() {
codec: DEFAULT_MP4_CODEC,
encodingMode,
preferredEncoderPath: supportedSourceDimensions.encoderPath,
experimentalNativeExport: smokeExportConfig.useNativeExport,
preferredRenderBackend: smokeExportConfig.renderBackend,
experimentalNativeExport: useExperimentalNativeExport,
maxEncodeQueue: smokeExportConfig.maxEncodeQueue,
maxDecodeQueue: smokeExportConfig.maxDecodeQueue,
maxPendingFrames: smokeExportConfig.maxPendingFrames,
@@ -4787,6 +4806,7 @@ export default function VideoEditor() {
remountPreview,
showExportSuccessToast,
smokeExportConfig.backendPreference,
smokeExportConfig.renderBackend,
smokeExportConfig.enabled,
smokeExportConfig.useNativeExport,
smokeExportConfig.maxDecodeQueue,
@@ -4803,6 +4823,48 @@ export default function VideoEditor() {
],
);
useEffect(() => {
smokeExportReadyStateRef.current = {
cursorTelemetrySourcePath,
duration,
hasVideoPath: Boolean(videoPath),
isPreviewReady,
loading,
projectPath: smokeExportConfig.projectPath ?? null,
videoSourcePath,
};
}, [
cursorTelemetrySourcePath,
duration,
isPreviewReady,
loading,
smokeExportConfig.projectPath,
videoPath,
videoSourcePath,
]);
useEffect(() => {
if (!smokeExportConfig.enabled) {
return;
}
const timeoutId = window.setTimeout(() => {
if (smokeExportStartedRef.current) {
return;
}
smokeExportStartedRef.current = true;
void writeSmokeExportReport(smokeExportConfig.outputPath, {
success: false,
phase: "ready",
error: `Smoke export did not become ready within ${SMOKE_EXPORT_READY_TIMEOUT_MS}ms.`,
readyState: smokeExportReadyStateRef.current,
}).finally(() => window.close());
}, SMOKE_EXPORT_READY_TIMEOUT_MS);
return () => window.clearTimeout(timeoutId);
}, [smokeExportConfig.enabled, smokeExportConfig.outputPath]);
useEffect(() => {
if (!smokeExportConfig.enabled || smokeExportStartedRef.current) {
return;
@@ -4811,11 +4873,16 @@ export default function VideoEditor() {
if (error) {
smokeExportStartedRef.current = true;
console.error(`[smoke-export] ${error}`);
window.close();
void writeSmokeExportReport(smokeExportConfig.outputPath, {
success: false,
phase: "load",
error,
readyState: smokeExportReadyStateRef.current,
}).finally(() => window.close());
return;
}
if (!videoPath || loading) {
if (!videoPath || loading || !isPreviewReady || duration <= 0) {
return;
}
@@ -4841,9 +4908,12 @@ export default function VideoEditor() {
cursorTelemetrySourcePath,
error,
handleExport,
isPreviewReady,
loading,
duration,
smokeExportConfig.enabled,
smokeExportConfig.encodingMode,
smokeExportConfig.outputPath,
smokeExportConfig.projectPath,
videoPath,
videoSourcePath,
@@ -5055,6 +5125,8 @@ export default function VideoEditor() {
exportFormat === "mp4" &&
exportPipelineModel === "modern" &&
(isExporting || exportProgress !== null);
const shouldSuspendPreviewRendering =
isExporting && exportFormat === "mp4" && exportPipelineModel === "modern";
const isLegacyExportInProgress =
exportFormat === "mp4" &&
exportPipelineModel === "legacy" &&
@@ -5095,6 +5167,9 @@ export default function VideoEditor() {
return encoderName ? `${pathLabel} (${encoderName})` : pathLabel;
}, [exportProgress]);
const exportNativeSkipLabel = exportProgress?.nativeStaticLayoutSkipReason
? `Native skipped: ${exportProgress.nativeStaticLayoutSkipReason}`
: null;
const exportPercentLabel = exportProgress
? isExportSaving
? t("editor.exportStatus.saving", "Opening save dialog...")
@@ -5463,6 +5538,11 @@ export default function VideoEditor() {
Path: {exportRuntimeLabel}
</p>
) : null}
{exportNativeSkipLabel ? (
<p className="mt-1 text-[11px] text-amber-500/80">
{exportNativeSkipLabel}
</p>
) : null}
</div>
) : exportError ? (
<div className="rounded-2xl border border-foreground/10 bg-editor-surface p-4 text-foreground shadow-2xl">
@@ -5960,6 +6040,7 @@ export default function VideoEditor() {
}
cursorSway={cursorSway}
volume={shouldMutePreviewVideo ? 0 : previewVolume}
suspendRendering={shouldSuspendPreviewRendering}
/>
</div>
</div>
+209 -18
View File
@@ -159,6 +159,56 @@ function createPlaybackAnimationState(): PlaybackAnimationState {
};
}
type PixiPreviewBackend = "webgpu" | "webgl";
type PixiRendererAttempt = {
backend: PixiPreviewBackend;
message: string;
};
const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000;
function isCanvasRenderer(application: Application): boolean {
const rendererName = application?.renderer?.constructor?.name?.toLowerCase();
return Boolean(rendererName && (rendererName.includes("canvasrenderer") || rendererName.includes("canvas")));
}
function toRendererErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error ?? "Unknown renderer init error");
}
function isRendererUnavailableError(error: unknown): boolean {
const message = toRendererErrorMessage(error).toLowerCase();
return message.includes("canvasrenderer is not yet implemented") || message.includes("no available renderer");
}
function summarizeRendererAttempts(attempts: readonly PixiRendererAttempt[]): string {
const details = attempts.map((attempt) => `${attempt.backend}: ${attempt.message}`).join(" | ");
return `No supported Pixi preview renderer was available. Attempted: ${details}`;
}
type PixiInitOptions = Parameters<Application["init"]>[0];
async function initApplicationWithTimeout(
app: Application,
options: PixiInitOptions,
backend: PixiPreviewBackend,
): Promise<void> {
const timeoutErrorMessage = `Initialization timed out after ${PIXI_RENDERER_INIT_TIMEOUT_MS}ms for ${backend} renderer`;
let timeoutId: ReturnType<typeof setTimeout> | undefined;
const timeoutPromise = new Promise<never>((_, reject) => {
timeoutId = setTimeout(() => {
reject(new Error(timeoutErrorMessage));
}, PIXI_RENDERER_INIT_TIMEOUT_MS);
});
try {
await Promise.race([app.init(options), timeoutPromise]);
} finally {
if (timeoutId !== undefined) {
clearTimeout(timeoutId);
}
}
}
function getCursorPositionAtTime(
telemetry: CursorTelemetryPoint[],
timeMs: number,
@@ -273,6 +323,7 @@ interface VideoPlaybackProps {
cursorClickBounceDuration?: number;
cursorSway?: number;
volume?: number;
suspendRendering?: boolean;
}
type CaptionEditSession = {
@@ -349,6 +400,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
cursorClickBounceDuration = DEFAULT_CURSOR_CLICK_BOUNCE_DURATION,
cursorSway = DEFAULT_CURSOR_SWAY,
volume = 1,
suspendRendering = false,
},
ref,
) => {
@@ -364,6 +416,10 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const timeUpdateAnimationRef = useRef<number | null>(null);
const [pixiReady, setPixiReady] = useState(false);
const [videoReady, setVideoReady] = useState(false);
const [pixiRendererError, setPixiRendererError] = useState<string | null>(null);
const [pixiRendererBackend, setPixiRendererBackend] = useState<PixiPreviewBackend | null>(
null,
);
const overlayRef = useRef<HTMLDivElement | null>(null);
const focusIndicatorRef = useRef<HTMLDivElement | null>(null);
const webcamVideoRef = useRef<HTMLVideoElement | null>(null);
@@ -404,6 +460,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const frameContainerRef = useRef<Container | null>(null);
const frameIdRef = useRef<string | null>(frame);
const isPlayingRef = useRef(isPlaying);
const suspendRenderingRef = useRef(suspendRendering);
const isSeekingRef = useRef(false);
const allowPlaybackRef = useRef(false);
const lockedVideoDimensionsRef = useRef<{
@@ -451,6 +508,75 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
createCursorFollowCameraState(),
);
const initializePixiRenderer = useCallback(
async (container: HTMLDivElement): Promise<{
app: Application;
backend: PixiPreviewBackend;
}> => {
const backendOrder: PixiPreviewBackend[] = ["webgl", "webgpu"];
const attempts: PixiRendererAttempt[] = [];
for (const backend of backendOrder) {
if (
backend === "webgpu" &&
!(typeof navigator !== "undefined" && "gpu" in navigator)
) {
attempts.push({
backend,
message: "WebGPU runtime is unavailable in this browser.",
});
continue;
}
const rendererApp = new Application();
const initStarted = typeof performance === "undefined" ? Date.now() : performance.now();
try {
await initApplicationWithTimeout(
rendererApp,
{
width: container.clientWidth,
height: container.clientHeight,
backgroundAlpha: 0,
antialias: true,
failIfMajorPerformanceCaveat: false,
resolution: window.devicePixelRatio || 1,
autoDensity: true,
preference: backend,
autoStart: true,
sharedTicker: false,
},
backend,
);
const elapsed = Math.round(
(typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
);
if (isCanvasRenderer(rendererApp)) {
throw new Error(
`Renderer initialized with unsupported fallback backend after ${elapsed}ms: ${rendererApp.renderer.constructor?.name ?? "unknown"}`,
);
}
return { app: rendererApp, backend };
} catch (error) {
const elapsed = Math.round(
(typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
);
attempts.push({ backend, message: `${toRendererErrorMessage(error)} (after ${elapsed}ms)` });
const statusMessage = isRendererUnavailableError(error)
? "renderer backend unavailable in this runtime"
: "renderer init failed";
console.warn(
`[VideoPlayback] Failed to init ${backend} renderer (${statusMessage}) after ${elapsed}ms; trying fallback.`,
error,
);
rendererApp.destroy(true);
}
}
throw new Error(summarizeRendererAttempts(attempts));
},
[],
);
const activeCaptionLayout = useMemo(() => {
if (
!autoCaptionSettings?.enabled ||
@@ -864,7 +990,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const maskRect = result.maskRect;
const insets = frameData.screenInsets;
if (insets) {
// Frame is larger than screen area — compute full frame size from insets
// Frame is larger than screen area - compute full frame size from insets
const screenW = maskRect.width;
const screenH = maskRect.height;
const frameW = screenW / (1 - insets.left - insets.right);
@@ -1171,6 +1297,54 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
}
}, [isPlaying]);
useEffect(() => {
suspendRenderingRef.current = suspendRendering;
const app = appRef.current;
if (!app?.ticker) {
return;
}
if (suspendRendering) {
bgVideoRef.current?.pause();
webcamVideoRef.current?.pause();
layoutVideoContentRef.current?.();
const videoTextureSource = videoSpriteRef.current?.texture?.source as
| { update?: () => void }
| undefined;
videoTextureSource?.update?.();
app.render();
return;
}
app.ticker.start();
const video = videoRef.current;
if (video) {
const targetTime = clampMediaTimeToDuration(
currentTimeRef.current / 1000,
Number.isFinite(video.duration) ? video.duration : null,
);
if (Math.abs(video.currentTime - targetTime) > 0.001) {
try {
video.currentTime = targetTime;
} catch {
// no-op
}
}
}
layoutVideoContentRef.current?.();
const videoTextureSource = videoSpriteRef.current?.texture?.source as
| { update?: () => void }
| undefined;
videoTextureSource?.update?.();
requestAnimationFrame(() => {
appRef.current?.render();
});
if (isPlayingRef.current) {
bgVideoRef.current?.play().catch(() => undefined);
webcamVideoRef.current?.play().catch(() => undefined);
}
}, [pixiReady, suspendRendering]);
// Keep video wallpapers locked to the same source timestamp as the main clip.
useEffect(() => {
const bgVideo = bgVideoRef.current;
@@ -1519,19 +1693,12 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
error,
);
}
setPixiRendererError(null);
setPixiRendererBackend(null);
app = new Application();
await app.init({
width: container.clientWidth,
height: container.clientHeight,
backgroundAlpha: 0,
antialias: true,
failIfMajorPerformanceCaveat: false,
resolution: window.devicePixelRatio || 1,
autoDensity: true,
preference: "webgl",
});
const result = await initializePixiRenderer(container);
app = result.app;
setPixiRendererBackend(result.backend);
app.ticker.maxFPS = 60;
@@ -1557,7 +1724,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
videoContainerRef.current = videoContainer;
cameraContainer.addChild(videoContainer);
// Device frame overlay container — sits above video but below cursor
// Device frame overlay container - sits above video but below cursor
const frameContainer = new Container();
frameContainerRef.current = frameContainer;
cameraContainer.addChild(frameContainer);
@@ -1586,7 +1753,12 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
setPixiReady(true);
})().catch((error) => {
const errorMessage =
error instanceof Error
? error.message
: "Failed to initialize preview renderer";
console.error("Failed to initialize preview renderer:", error);
setPixiRendererError(errorMessage);
onError(
error instanceof Error
? error.message
@@ -1597,6 +1769,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
return () => {
mounted = false;
setPixiReady(false);
setPixiRendererError(null);
setPixiRendererBackend(null);
if (cursorOverlayRef.current) {
cursorOverlayRef.current.destroy();
cursorOverlayRef.current = null;
@@ -1616,7 +1790,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
cursorContainerRef.current = null;
videoSpriteRef.current = null;
};
}, [onError]);
}, [initializePixiRenderer, onError]);
useEffect(() => {
const video = videoRef.current;
@@ -1675,7 +1849,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
blurFilter.resolution = app.renderer.resolution;
blurFilter.blur = 0;
const motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0);
// Don't attach filters by default — the filter pipeline forces the video
// Don't attach filters by default - the filter pipeline forces the video
// through an intermediate RenderTexture at renderer resolution, downsampling
// the native video and destroying detail. Filters are attached conditionally
// in the ticker only when zoom motion blur is actually active.
@@ -1783,6 +1957,10 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
};
const ticker = () => {
if (suspendRenderingRef.current) {
return;
}
const { region, strength, blendedScale, transition } = findDominantRegion(
zoomRegionsRef.current,
currentTimeRef.current,
@@ -2407,6 +2585,10 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
: resolvedWallpaperKind === "video"
? {}
: { background: resolvedWallpaper || "" };
const fallbackVideoClassName = pixiRendererError
? "absolute inset-0 h-full w-full object-cover"
: "pointer-events-none absolute left-0 top-0 h-px w-px opacity-0";
const hasRendererFallback = Boolean(pixiRendererError);
const nativeAspectRatio = (() => {
const locked = lockedVideoDimensionsRef.current;
@@ -2464,9 +2646,18 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
filter:
showShadow && shadowIntensity > 0
? `drop-shadow(0 ${shadowIntensity * 12}px ${shadowIntensity * 48}px rgba(0,0,0,${shadowIntensity * 0.7})) drop-shadow(0 ${shadowIntensity * 4}px ${shadowIntensity * 16}px rgba(0,0,0,${shadowIntensity * 0.5})) drop-shadow(0 ${shadowIntensity * 2}px ${shadowIntensity * 8}px rgba(0,0,0,${shadowIntensity * 0.3}))`
: "none",
: "none",
}}
/>
{hasRendererFallback && (
<div className="absolute inset-0 z-10 flex items-center justify-center bg-black/60 p-2 text-center">
<div className="rounded-md bg-black/70 px-3 py-1.5 text-xs text-white">
{`Pixi renderer unavailable on this environment (${pixiRendererBackend ?? "unknown"}).`}
<br />
Fallback to 2D native preview so you can continue working while the GPU path is unavailable.
</div>
</div>
)}
{/* Canvas overlay for extension cursor effects (drawn via Canvas 2D API) */}
<canvas
ref={cursorEffectsCanvasRef}
@@ -2792,7 +2983,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
<video
ref={videoRef}
src={videoPath}
className="pointer-events-none absolute left-0 top-0 h-px w-px opacity-0"
className={fallbackVideoClassName}
preload="metadata"
playsInline
aria-hidden="true"
@@ -42,6 +42,26 @@ type CursorPackSource = {
pointerAnchor: { x: number; y: number };
};
export type NativeCursorAtlasEntry = {
cursorType: CursorAssetKey;
index: number;
x: number;
y: number;
width: number;
height: number;
anchorX: number;
anchorY: number;
aspectRatio: number;
};
export type NativeCursorAtlas = {
style: CursorStyle;
width: number;
height: number;
dataUrl: string;
entries: NativeCursorAtlasEntry[];
};
/**
* Configuration for cursor rendering.
*/
@@ -95,6 +115,8 @@ const CURSOR_SHADOW_OFFSET_X = 0;
const CURSOR_SHADOW_OFFSET_Y = 2;
const CURSOR_SHADOW_BLUR = 3;
const CURSOR_SHADOW_PADDING = 12;
const NATIVE_CURSOR_ATLAS_DRAW_HEIGHT = 256;
const NATIVE_CURSOR_ATLAS_PADDING = 2;
let cursorAssetsPromise: Promise<void> | null = null;
let cursorPackAssetsPromise: Promise<void> | null = null;
let loadedCursorPackSourcesSignature = "";
@@ -519,6 +541,77 @@ export async function preloadCursorAssets() {
await ensureCursorPackAssetsLoaded();
}
function getNativeCursorAtlasAsset(style: CursorStyle, key: CursorAssetKey) {
if (isStatefulCursorStyle(style)) {
return getStatefulCursorAsset(style, key);
}
if (isSingleCursorStyle(style)) {
return getCursorStyleAsset(style);
}
return getCursorPackStyleAsset(style, key);
}
export async function buildNativeCursorAtlas(
style: CursorStyle = DEFAULT_CURSOR_STYLE,
): Promise<NativeCursorAtlas | null> {
if (typeof document === "undefined") {
return null;
}
await preloadCursorAssets();
const entries: NativeCursorAtlasEntry[] = [];
const packedAssets = SUPPORTED_CURSOR_KEYS.map((key, index) => {
const asset = getNativeCursorAtlasAsset(style, key);
const height = NATIVE_CURSOR_ATLAS_DRAW_HEIGHT;
const width = Math.max(1, Math.round(height * asset.aspectRatio));
return { key, index, asset, width, height };
});
const atlasWidth = packedAssets.reduce(
(total, item) => total + item.width + NATIVE_CURSOR_ATLAS_PADDING,
NATIVE_CURSOR_ATLAS_PADDING,
);
const atlasHeight = NATIVE_CURSOR_ATLAS_DRAW_HEIGHT + NATIVE_CURSOR_ATLAS_PADDING * 2;
const canvas = document.createElement("canvas");
canvas.width = atlasWidth;
canvas.height = atlasHeight;
const ctx = canvas.getContext("2d");
if (!ctx) {
return null;
}
ctx.clearRect(0, 0, atlasWidth, atlasHeight);
let x = NATIVE_CURSOR_ATLAS_PADDING;
for (const { key, index, asset, width, height } of packedAssets) {
const y = NATIVE_CURSOR_ATLAS_PADDING;
ctx.drawImage(asset.image, x, y, width, height);
entries.push({
cursorType: key,
index,
x,
y,
width,
height,
anchorX: asset.anchorX,
anchorY: asset.anchorY,
aspectRatio: asset.aspectRatio,
});
x += width + NATIVE_CURSOR_ATLAS_PADDING;
}
return {
style,
width: atlasWidth,
height: atlasHeight,
dataUrl: canvas.toDataURL("image/png"),
entries,
};
}
/**
* Interpolates cursor position from telemetry samples at a given time.
* Uses linear interpolation between the two nearest samples.
+40 -1
View File
@@ -1,5 +1,10 @@
import { describe, expect, it } from "vitest";
import { selectRecordingMimeType } from "./recordingMimeType";
import {
getVideoExtensionForMimeType,
isWebmMimeType,
selectRecordingMimeType,
selectWebcamRecordingMimeType,
} from "./recordingMimeType";
describe("selectRecordingMimeType", () => {
it("prefers codecs the editor can play back", () => {
@@ -55,4 +60,38 @@ describe("selectRecordingMimeType", () => {
expect(mimeType).toBeUndefined();
});
it("prefers MP4/H.264 for webcam captures when supported", () => {
const mimeType = selectWebcamRecordingMimeType({
isTypeSupported: (type) =>
["video/mp4;codecs=avc1.42E01E", "video/webm;codecs=vp9"].includes(
type,
),
canPlayType: () => "probably",
});
expect(mimeType).toBe("video/mp4;codecs=avc1.42E01E");
});
it("falls back to WebM webcam capture when MP4 is unavailable", () => {
const mimeType = selectWebcamRecordingMimeType({
isTypeSupported: (type) =>
["video/webm;codecs=vp9", "video/webm"].includes(type),
canPlayType: () => "probably",
});
expect(mimeType).toBe("video/webm;codecs=vp9");
});
it("maps recording MIME types to the saved file extension", () => {
expect(getVideoExtensionForMimeType("video/mp4;codecs=avc1")).toBe(".mp4");
expect(getVideoExtensionForMimeType("video/webm;codecs=vp9")).toBe(".webm");
expect(getVideoExtensionForMimeType(undefined)).toBe(".webm");
});
it("detects WebM MIME types for duration repair", () => {
expect(isWebmMimeType("video/webm;codecs=vp9")).toBe(true);
expect(isWebmMimeType("video/mp4;codecs=avc1")).toBe(false);
expect(isWebmMimeType(undefined)).toBe(false);
});
});
+34 -4
View File
@@ -6,12 +6,24 @@ const RECORDING_MIME_TYPE_PREFERENCES = [
"video/webm;codecs=h264",
] as const;
const WEBCAM_RECORDING_MIME_TYPE_PREFERENCES = [
"video/mp4;codecs=avc1.42E01E",
"video/mp4;codecs=avc1",
"video/mp4;codecs=h264",
"video/mp4",
"video/webm;codecs=h264",
"video/webm;codecs=vp9",
"video/webm",
"video/webm;codecs=vp8",
] as const;
type MimeTypeSelectorOptions = {
isTypeSupported?: (type: string) => boolean;
canPlayType?: (type: string) => string;
};
export function selectRecordingMimeType(
function selectMimeTypeFromPreferences(
preferences: readonly string[],
options: MimeTypeSelectorOptions = {},
): string | undefined {
const isTypeSupported =
@@ -20,10 +32,28 @@ export function selectRecordingMimeType(
options.canPlayType ??
((type: string) => document.createElement("video").canPlayType(type));
const supportedTypes = RECORDING_MIME_TYPE_PREFERENCES.filter((type) =>
isTypeSupported(type),
);
const supportedTypes = preferences.filter((type) => isTypeSupported(type));
const playableType = supportedTypes.find((type) => canPlayType(type) !== "");
return playableType ?? supportedTypes[0];
}
export function selectRecordingMimeType(
options: MimeTypeSelectorOptions = {},
): string | undefined {
return selectMimeTypeFromPreferences(RECORDING_MIME_TYPE_PREFERENCES, options);
}
export function selectWebcamRecordingMimeType(
options: MimeTypeSelectorOptions = {},
): string | undefined {
return selectMimeTypeFromPreferences(WEBCAM_RECORDING_MIME_TYPE_PREFERENCES, options);
}
export function isWebmMimeType(mimeType: string | undefined | null): boolean {
return /^video\/webm(?:[;\s]|$)/i.test(mimeType ?? "");
}
export function getVideoExtensionForMimeType(mimeType: string | undefined | null): ".mp4" | ".webm" {
return /^video\/mp4(?:[;\s]|$)/i.test(mimeType ?? "") ? ".mp4" : ".webm";
}
+19 -8
View File
@@ -2,7 +2,12 @@ import { fixWebmDuration } from "@fix-webm-duration/fix";
import { useCallback, useEffect, useRef, useState } from "react";
import { toast } from "sonner";
import { getEffectiveRecordingDurationMs } from "@/lib/mediaTiming";
import { selectRecordingMimeType } from "./recordingMimeType";
import {
getVideoExtensionForMimeType,
isWebmMimeType,
selectRecordingMimeType,
selectWebcamRecordingMimeType,
} from "./recordingMimeType";
const TARGET_FRAME_RATE = 60;
const TARGET_WIDTH = 3840;
@@ -290,6 +295,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
return selectRecordingMimeType();
}, []);
const selectWebcamMimeType = useCallback(() => {
return selectWebcamRecordingMimeType();
}, []);
const computeBitrate = (width: number, height: number) => {
const pixels = width * height;
const highFrameRateBoost =
@@ -576,7 +585,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
audio: false,
});
const mimeType = selectMimeType();
const mimeType = selectWebcamMimeType();
webcamChunks.current = [];
resolvedWebcamPath.current = null;
webcamStopPromise.current = new Promise((resolve) => {
@@ -601,7 +610,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
};
recorder.onstop = async () => {
const sessionTimestamp = recordingSessionTimestamp.current ?? Date.now();
const webcamFileName = `${RECORDING_FILE_PREFIX}${sessionTimestamp}${WEBCAM_SUFFIX}${VIDEO_FILE_EXTENSION}`;
const webcamMimeType = recorder.mimeType || mimeType;
const webcamFileName = `${RECORDING_FILE_PREFIX}${sessionTimestamp}${WEBCAM_SUFFIX}${getVideoExtensionForMimeType(webcamMimeType)}`;
try {
if (webcamChunks.current.length === 0) {
@@ -613,14 +623,15 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
0,
getRecordingDurationMs(Date.now()) - webcamTimeOffsetMs.current,
);
const webcamBlobType = recorder.mimeType || mimeType;
const webcamBlob = new Blob(
webcamChunks.current,
webcamBlobType ? { type: webcamBlobType } : undefined,
webcamMimeType ? { type: webcamMimeType } : undefined,
);
webcamChunks.current = [];
const fixedBlob = await fixWebmDuration(webcamBlob, duration);
const arrayBuffer = await fixedBlob.arrayBuffer();
const finalBlob = isWebmMimeType(webcamMimeType)
? await fixWebmDuration(webcamBlob, duration)
: webcamBlob;
const arrayBuffer = await finalBlob.arrayBuffer();
const result = await window.electronAPI.storeRecordedVideo(
arrayBuffer,
webcamFileName,
@@ -655,7 +666,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
webcamStream.current = null;
}
}
}, [getRecordingDurationMs, selectMimeType, webcamDeviceId, webcamEnabled]);
}, [getRecordingDurationMs, selectWebcamMimeType, webcamDeviceId, webcamEnabled]);
/** Start the prepared webcam MediaRecorder. Call after main recording begins. */
const beginWebcamCapture = useCallback(() => {
+6 -3
View File
@@ -1,3 +1,5 @@
import { resolveAvailableWallpaperPath } from "./wallpapers";
function encodeRelativeAssetPath(relativePath: string): string {
return relativePath
.replace(/^\/+/, "")
@@ -132,10 +134,11 @@ export async function getRenderableAssetUrl(asset: string): Promise<string> {
return asset;
}
const availableAsset = await resolveAvailableWallpaperPath(asset);
const resolvedAsset =
asset.startsWith("/") && !asset.startsWith("//")
? await getAssetPath(asset.replace(/^\//, ""))
: asset;
availableAsset.startsWith("/") && !availableAsset.startsWith("//")
? await getAssetPath(availableAsset.replace(/^\//, ""))
: availableAsset;
const localFilePath = toLocalFilePath(resolvedAsset);
if (!localFilePath || typeof window === "undefined" || !window.electronAPI?.readLocalFile) {
+9 -1
View File
@@ -1,6 +1,10 @@
import { describe, expect, it } from "vitest";
import { normalizeLightningRuntimePlatform, shouldPreferNativeAutoBackend } from "./backendPolicy";
import {
getDefaultLightningRenderBackend,
normalizeLightningRuntimePlatform,
shouldPreferNativeAutoBackend,
} from "./backendPolicy";
describe("backendPolicy", () => {
it("normalizes common platform hints", () => {
@@ -16,4 +20,8 @@ describe("backendPolicy", () => {
expect(shouldPreferNativeAutoBackend("darwin")).toBe(false);
expect(shouldPreferNativeAutoBackend("unknown")).toBe(false);
});
it("keeps Lightning exports on the stable WebGL renderer by default", () => {
expect(getDefaultLightningRenderBackend()).toBe("webgl");
});
});
+6
View File
@@ -1,3 +1,5 @@
import type { ExportRenderBackend } from "./types";
export type LightningRuntimePlatform = "darwin" | "win32" | "linux" | "unknown";
export function normalizeLightningRuntimePlatform(
@@ -25,3 +27,7 @@ export function normalizeLightningRuntimePlatform(
export function shouldPreferNativeAutoBackend(_platform: LightningRuntimePlatform): boolean {
return false;
}
export function getDefaultLightningRenderBackend(): ExportRenderBackend {
return "webgl";
}
+56
View File
@@ -0,0 +1,56 @@
import { describe, expect, it } from "vitest";
import { getMp4ExportBitrate, getSourceQualityBitrate } from "./exportBitrate";
describe("export bitrate policy", () => {
it("keeps the legacy source-quality bitrate unchanged", () => {
expect(getSourceQualityBitrate(1920, 1080)).toBe(30_000_000);
expect(
getMp4ExportBitrate({
width: 1920,
height: 1080,
frameRate: 30,
quality: "source",
encodingMode: "quality",
}),
).toBe(27_000_000);
});
it("caps modern native static-layout source exports to avoid bitrate inflation", () => {
expect(
getMp4ExportBitrate({
width: 1920,
height: 1080,
frameRate: 30,
quality: "source",
encodingMode: "quality",
useModernNativeStaticLayout: true,
}),
).toBe(14_000_000);
});
it("does not raise fast exports when the requested bitrate is already lower than the cap", () => {
expect(
getMp4ExportBitrate({
width: 1920,
height: 1080,
frameRate: 30,
quality: "source",
encodingMode: "fast",
useModernNativeStaticLayout: true,
}),
).toBe(3_000_000);
});
it("scales the modern native cap with output pixel rate", () => {
expect(
getMp4ExportBitrate({
width: 3840,
height: 2160,
frameRate: 30,
quality: "source",
encodingMode: "quality",
useModernNativeStaticLayout: true,
}),
).toBe(28_000_000);
});
});
+81
View File
@@ -0,0 +1,81 @@
import type { ExportEncodingMode, ExportMp4FrameRate, ExportQuality } from "./types";
const MIN_MP4_BITRATE = 2_000_000;
const REFERENCE_PIXEL_RATE = 1920 * 1080 * 30;
export function getEncodingModeBitrateMultiplier(encodingMode: ExportEncodingMode): number {
switch (encodingMode) {
case "fast":
return 0.1;
case "quality":
return 0.9;
case "balanced":
default:
return 0.5;
}
}
export function getSourceQualityBitrate(width: number, height: number): number {
const totalPixels = width * height;
if (totalPixels > 2560 * 1440) {
return 80_000_000;
}
if (totalPixels > 1920 * 1080) {
return 50_000_000;
}
return 30_000_000;
}
function getBaseMp4ExportBitrate(width: number, height: number, quality: ExportQuality): number {
if (quality === "source") {
return getSourceQualityBitrate(width, height);
}
const totalPixels = width * height;
if (totalPixels <= 1280 * 720) {
return 10_000_000;
}
if (totalPixels <= 1920 * 1080) {
return 20_000_000;
}
return 30_000_000;
}
function getModernNativeStaticLayoutBitrateCap(
width: number,
height: number,
frameRate: ExportMp4FrameRate,
quality: ExportQuality,
): number {
const referenceCap =
quality === "source" ? 14_000_000 : quality === "high" ? 12_000_000 : 8_000_000;
const pixelRateScale = Math.max((width * height * frameRate) / REFERENCE_PIXEL_RATE, 0.1);
return Math.round(referenceCap * Math.sqrt(pixelRateScale));
}
export function getMp4ExportBitrate(options: {
width: number;
height: number;
frameRate: ExportMp4FrameRate;
quality: ExportQuality;
encodingMode: ExportEncodingMode;
useModernNativeStaticLayout?: boolean;
}): number {
const requestedBitrate = Math.round(
getBaseMp4ExportBitrate(options.width, options.height, options.quality) *
getEncodingModeBitrateMultiplier(options.encodingMode),
);
const cappedBitrate = options.useModernNativeStaticLayout
? Math.min(
requestedBitrate,
getModernNativeStaticLayoutBitrateCap(
options.width,
options.height,
options.frameRate,
options.quality,
),
)
: requestedBitrate;
return Math.max(MIN_MP4_BITRATE, cappedBitrate);
}
+120 -17
View File
@@ -127,6 +127,52 @@ interface AnimationState {
y: number;
}
type ExportRenderBackend = "webgl" | "webgpu";
type PixiRendererAttempt = {
backend: ExportRenderBackend;
message: string;
};
const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000;
function isCanvasRenderer(renderer: Application): boolean {
const rendererName = renderer?.renderer?.constructor?.name?.toLowerCase();
return Boolean(rendererName && (rendererName.includes("canvasrenderer") || rendererName.includes("canvas")));
}
function toErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error ?? "Unknown renderer init error");
}
type PixiInitOptions = Parameters<Application["init"]>[0];
async function initApplicationWithTimeout(
app: Application,
options: PixiInitOptions,
backend: ExportRenderBackend,
): Promise<void> {
const timeoutErrorMessage = `Initialization timed out after ${PIXI_RENDERER_INIT_TIMEOUT_MS}ms for ${backend} renderer`;
let timeoutId: ReturnType<typeof setTimeout> | undefined;
const timeoutPromise = new Promise<never>((_, reject) => {
timeoutId = setTimeout(() => {
reject(new Error(timeoutErrorMessage));
}, PIXI_RENDERER_INIT_TIMEOUT_MS);
});
try {
await Promise.race([app.init(options), timeoutPromise]);
} finally {
if (timeoutId !== undefined) {
clearTimeout(timeoutId);
}
}
}
function summarizeRendererAttempts(attempts: readonly PixiRendererAttempt[]): string {
const details = attempts.map((attempt) => `${attempt.backend}: ${attempt.message}`).join(" | ");
return `No supported Pixi export backend was available. Attempted: ${details}`;
}
interface VideoTextureSource {
resource: VideoFrame | CanvasImageSource;
update: () => void;
@@ -233,6 +279,77 @@ export class FrameRenderer {
this.cursorFollowCamera = createCursorFollowCameraState();
}
private async createPixiApplication(
canvas: HTMLCanvasElement,
): Promise<{ app: Application; backend: ExportRenderBackend }> {
const baseOptions = {
canvas,
width: this.config.width,
height: this.config.height,
backgroundAlpha: 0,
antialias: true,
failIfMajorPerformanceCaveat: false,
resolution: 1,
autoDensity: true,
autoStart: false,
sharedTicker: false,
powerPreference: "high-performance" as const,
};
const preferredRenderBackend = this.config.preferredRenderBackend;
const backendOrder =
preferredRenderBackend === "webgpu"
? (["webgpu", "webgl"] as const)
: preferredRenderBackend === "webgl"
? (["webgl", "webgpu"] as const)
: (["webgl", "webgpu"] as const);
const failures: PixiRendererAttempt[] = [];
for (const backend of backendOrder) {
if (backend === "webgpu" && !(typeof navigator !== "undefined" && "gpu" in navigator)) {
failures.push({
backend,
message: "WebGPU runtime is unavailable in this environment.",
});
continue;
}
const app = new Application();
const initStarted = typeof performance === "undefined" ? Date.now() : performance.now();
try {
await initApplicationWithTimeout(
app,
{
...baseOptions,
preference: backend,
},
backend,
);
const elapsed = Math.round(
(typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
);
if (isCanvasRenderer(app)) {
throw new Error(
`Renderer initialized with unsupported fallback backend after ${elapsed}ms: ${app.renderer.constructor?.name ?? "unknown"}`,
);
}
return { app, backend };
} catch (error) {
const elapsed = Math.round(
(typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
);
failures.push({ backend, message: `${toErrorMessage(error)} (after ${elapsed}ms)` });
console.warn(
`[FrameRenderer] ${backend} renderer unavailable after ${elapsed}ms; trying next backend.`,
error,
);
app.destroy(true);
}
}
throw new Error(summarizeRendererAttempts(failures));
}
async initialize(): Promise<void> {
let cursorOverlayEnabled = true;
try {
@@ -261,23 +378,9 @@ export class FrameRenderer {
}
// Initialize PixiJS with optimized settings for export performance
this.app = new Application();
await this.app.init({
canvas,
width: this.config.width,
height: this.config.height,
backgroundAlpha: 0,
antialias: true,
failIfMajorPerformanceCaveat: false,
resolution: 1,
autoDensity: true,
autoStart: false,
sharedTicker: false,
powerPreference: "high-performance",
...(this.config.preferredRenderBackend
? { preference: this.config.preferredRenderBackend }
: {}),
});
const { app, backend } = await this.createPixiApplication(canvas);
this.app = app;
console.log(`[FrameRenderer] Export renderer backend: ${backend}`);
// Setup containers
this.cameraContainer = new Container();
+93 -33
View File
@@ -217,6 +217,61 @@ interface CaptionRenderState {
centerY: number;
}
type PixiRendererAttempt = {
backend: ExportRenderBackend;
message: string;
};
const CANVAS_RENDERER_NOT_IMPLEMENTED_HINT = "CanvasRenderer is not yet implemented";
const NO_RENDERER_HINT = "no available renderer";
const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000;
function isCanvasRenderer(application: Application): boolean {
const rendererName = application?.renderer?.constructor?.name?.toLowerCase();
return Boolean(rendererName && (rendererName.includes("canvasrenderer") || rendererName.includes("canvas")));
}
function toErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error ?? "Unknown renderer init error");
}
function summarizeRendererAttempts(attempts: readonly PixiRendererAttempt[]): string {
const details = attempts.map((attempt) => `${attempt.backend}: ${attempt.message}`).join(" | ");
return `No supported Pixi modern renderer was available. Attempted: ${details}`;
}
function isKnownRendererUnavailableError(error: unknown): boolean {
const message = toErrorMessage(error).toLowerCase();
return (
message.includes(CANVAS_RENDERER_NOT_IMPLEMENTED_HINT.toLowerCase()) ||
message.includes(NO_RENDERER_HINT)
);
}
type PixiInitOptions = Parameters<Application["init"]>[0];
async function initApplicationWithTimeout(
app: Application,
options: PixiInitOptions,
backend: ExportRenderBackend,
): Promise<void> {
const timeoutErrorMessage = `Initialization timed out after ${PIXI_RENDERER_INIT_TIMEOUT_MS}ms for ${backend} renderer`;
let timeoutId: ReturnType<typeof setTimeout> | undefined;
const timeoutPromise = new Promise<never>((_, reject) => {
timeoutId = setTimeout(() => {
reject(new Error(timeoutErrorMessage));
}, PIXI_RENDERER_INIT_TIMEOUT_MS);
});
try {
await Promise.race([app.init(options), timeoutPromise]);
} finally {
if (timeoutId !== undefined) {
clearTimeout(timeoutId);
}
}
}
function createAnimationState(): AnimationState {
return {
scale: 1,
@@ -550,52 +605,57 @@ export class FrameRenderer {
: typeof navigator !== "undefined" && "gpu" in navigator
? ["webgpu", "webgl"]
: ["webgl"];
let lastError: unknown = null;
const failures: PixiRendererAttempt[] = [];
for (const backend of backendOrder) {
if (backend === "webgpu") {
if (!(typeof navigator !== "undefined" && "gpu" in navigator)) {
continue;
}
const webgpuApp = new Application();
try {
await webgpuApp.init({
...baseOptions,
preference: "webgpu",
});
return { app: webgpuApp, backend: "webgpu" };
} catch (error) {
lastError = error;
console.warn(
"[FrameRenderer] WebGPU export renderer unavailable; trying next backend:",
error,
);
webgpuApp.destroy(true);
}
if (backend === "webgpu" && !(typeof navigator !== "undefined" && "gpu" in navigator)) {
failures.push({
backend,
message: "WebGPU runtime is unavailable in this environment.",
});
continue;
}
const webglApp = new Application();
const app = new Application();
const initStarted = typeof performance === "undefined" ? Date.now() : performance.now();
try {
await webglApp.init({
...baseOptions,
preference: "webgl",
});
return { app: webglApp, backend: "webgl" };
await initApplicationWithTimeout(
app,
{
...baseOptions,
preference: backend,
},
backend,
);
const elapsed = Math.round(
(typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
);
if (isCanvasRenderer(app)) {
throw new Error(
`Renderer initialized with unsupported fallback backend after ${elapsed}ms: ${app.renderer.constructor?.name ?? "unknown"}`,
);
}
return { app, backend };
} catch (error) {
lastError = error;
const elapsed = Math.round(
(typeof performance === "undefined" ? Date.now() : performance.now()) - initStarted,
);
failures.push({
backend,
message: `${toErrorMessage(error)} (after ${elapsed}ms)`,
});
const rendererMessage = isKnownRendererUnavailableError(error)
? "renderer backend unavailable in this runtime"
: "renderer init failed";
console.warn(
"[FrameRenderer] WebGL export renderer unavailable; trying next backend:",
`[FrameRenderer] ${backend} export renderer unavailable (${rendererMessage}) after ${elapsed}ms; trying next backend:`,
error,
);
webglApp.destroy(true);
app.destroy(true);
}
}
throw lastError instanceof Error
? lastError
: new Error("No supported Pixi export renderer was available");
throw new Error(summarizeRendererAttempts(failures));
}
private createShadowLayers(
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,36 @@
import { describe, expect, it } from "vitest";
import { roundNativeStaticLayoutContentSize } from "./nativeStaticLayoutGeometry";
describe("roundNativeStaticLayoutContentSize", () => {
it("keeps even dimensions inside the floating layout bounds", () => {
expect(
roundNativeStaticLayoutContentSize({
width: 1766.4,
height: 993.6,
}),
).toEqual({ width: 1764, height: 992 });
});
it("avoids independently rounding width and height into aspect drift", () => {
const rounded = roundNativeStaticLayoutContentSize({
width: 1766.4,
height: 993.6,
});
const independentAspect = 1766 / 994;
const roundedAspect = rounded.width / rounded.height;
const targetAspect = 1766.4 / 993.6;
expect(Math.abs(roundedAspect - targetAspect)).toBeLessThan(
Math.abs(independentAspect - targetAspect),
);
});
it("handles integer even layouts without changing them", () => {
expect(
roundNativeStaticLayoutContentSize({
width: 1600,
height: 900,
}),
).toEqual({ width: 1600, height: 900 });
});
});
@@ -0,0 +1,61 @@
type NativeStaticLayoutContentSize = {
width: number;
height: number;
};
function toEvenFloor(value: number) {
return Math.max(2, Math.floor(value / 2) * 2);
}
function toEvenRound(value: number) {
return Math.max(2, Math.round(value / 2) * 2);
}
function clampEven(value: number, max: number) {
const rounded = toEvenRound(value);
return rounded <= max ? rounded : toEvenFloor(max);
}
function aspectError(size: NativeStaticLayoutContentSize, aspect: number) {
return Math.abs(size.width / size.height - aspect);
}
export function roundNativeStaticLayoutContentSize(params: {
width: number;
height: number;
}): NativeStaticLayoutContentSize {
const maxWidth = toEvenFloor(params.width);
const maxHeight = toEvenFloor(params.height);
if (
!Number.isFinite(params.width) ||
!Number.isFinite(params.height) ||
params.width <= 0 ||
params.height <= 0
) {
return { width: maxWidth, height: maxHeight };
}
const aspect = params.width / params.height;
const fromWidth = {
width: maxWidth,
height: clampEven(maxWidth / aspect, maxHeight),
};
const fromHeight = {
width: clampEven(maxHeight * aspect, maxWidth),
height: maxHeight,
};
const widthError = aspectError(fromWidth, aspect);
const heightError = aspectError(fromHeight, aspect);
if (heightError < widthError) {
return fromHeight;
}
if (widthError < heightError) {
return fromWidth;
}
return fromHeight.width * fromHeight.height >= fromWidth.width * fromWidth.height
? fromHeight
: fromWidth;
}
@@ -0,0 +1,112 @@
import { describe, expect, it } from "vitest";
import { buildNativeStaticLayoutCursorTelemetry } from "./nativeStaticLayoutTelemetry";
describe("buildNativeStaticLayoutCursorTelemetry", () => {
it("filters invalid samples, clamps coordinates, and sorts by time", () => {
expect(
buildNativeStaticLayoutCursorTelemetry(
[
{ timeMs: 30, cx: 2, cy: -1 },
{ timeMs: Number.NaN, cx: 0.5, cy: 0.5 },
{ timeMs: 10, cx: 0.25, cy: 0.75 },
],
{ frameRate: 30, durationSec: 2 },
),
).toEqual([
{
timeMs: 0,
cx: 0.25,
cy: 0.75,
cursorType: "arrow",
cursorTypeIndex: 0,
bounceScale: 1,
},
{
timeMs: 2000,
cx: 1,
cy: 0,
cursorType: "arrow",
cursorTypeIndex: 0,
bounceScale: 1,
},
]);
});
it("resamples high-frequency cursor telemetry to the export frame cadence", () => {
const telemetry = Array.from({ length: 101 }, (_, index) => ({
timeMs: index * 10,
cx: index / 100,
cy: 1 - index / 100,
}));
const resampled = buildNativeStaticLayoutCursorTelemetry(telemetry, {
frameRate: 10,
durationSec: 1,
});
expect(resampled).toBeDefined();
expect(resampled).toHaveLength(11);
expect(resampled?.[0]).toEqual({
timeMs: 0,
cx: 0,
cy: 1,
cursorType: "arrow",
cursorTypeIndex: 0,
bounceScale: 1,
});
expect(resampled?.[5]).toEqual({
timeMs: 500,
cx: 0.5,
cy: 0.5,
cursorType: "arrow",
cursorTypeIndex: 0,
bounceScale: 1,
});
expect(resampled?.[10]).toEqual({
timeMs: 1000,
cx: 1,
cy: 0,
cursorType: "arrow",
cursorTypeIndex: 0,
bounceScale: 1,
});
});
it("collapses visually unchanged cursor positions", () => {
const telemetry = Array.from({ length: 101 }, (_, index) => ({
timeMs: index * 10,
cx: 0.4,
cy: 0.6,
}));
expect(
buildNativeStaticLayoutCursorTelemetry(telemetry, {
frameRate: 10,
durationSec: 1,
}),
).toEqual([
{
timeMs: 1000,
cx: 0.4,
cy: 0.6,
cursorType: "arrow",
cursorTypeIndex: 0,
bounceScale: 1,
},
]);
});
it("keeps cursor type transitions and click bounce samples for native parity", () => {
const resampled = buildNativeStaticLayoutCursorTelemetry(
[
{ timeMs: 0, cx: 0.1, cy: 0.2, cursorType: "arrow" },
{ timeMs: 100, cx: 0.1, cy: 0.2, interactionType: "click", cursorType: "pointer" },
{ timeMs: 200, cx: 0.1, cy: 0.2, interactionType: "mouseup", cursorType: "text" },
],
{ frameRate: 10, durationSec: 0.4, clickBounce: 1, clickBounceDurationMs: 350 },
);
expect(resampled?.map((sample) => sample.cursorTypeIndex)).toContain(1);
expect(resampled?.some((sample) => (sample.bounceScale ?? 1) < 1)).toBe(true);
});
});
@@ -0,0 +1,244 @@
import type { CursorTelemetryPoint } from "@/components/video-editor/types";
export type NativeStaticLayoutCursorTelemetrySample = {
timeMs: number;
cx: number;
cy: number;
cursorType?: CursorTelemetryPoint["cursorType"];
interactionType?: string;
cursorTypeIndex?: number;
bounceScale?: number;
};
export type NativeStaticLayoutCursorTelemetryOptions = {
frameRate: number;
durationSec: number;
clickBounce?: number;
clickBounceDurationMs?: number;
};
const CURSOR_POSITION_EPSILON = 0.00001;
const CURSOR_BOUNCE_EPSILON = 0.0005;
const DEFAULT_CLICK_BOUNCE = 1;
const DEFAULT_CLICK_BOUNCE_DURATION_MS = 350;
const CURSOR_TYPE_INDEX: Record<string, number> = {
arrow: 0,
text: 1,
pointer: 2,
crosshair: 3,
"open-hand": 4,
"closed-hand": 5,
"resize-ew": 6,
"resize-ns": 7,
"not-allowed": 8,
};
function clampUnit(value: number) {
return Math.min(1, Math.max(0, value));
}
function sanitizeCursorTelemetry(telemetry: NativeStaticLayoutCursorTelemetrySample[]) {
return telemetry
.filter((sample) => {
return (
Number.isFinite(sample.timeMs) &&
Number.isFinite(sample.cx) &&
Number.isFinite(sample.cy)
);
})
.map((sample) => ({
timeMs: Math.max(0, sample.timeMs),
cx: clampUnit(sample.cx),
cy: clampUnit(sample.cy),
cursorType: sample.cursorType,
interactionType: sample.interactionType,
}))
.sort((left, right) => left.timeMs - right.timeMs);
}
function interpolateCursorSample(
samples: NativeStaticLayoutCursorTelemetrySample[],
timeMs: number,
) {
if (timeMs <= samples[0].timeMs) {
return { timeMs, cx: samples[0].cx, cy: samples[0].cy };
}
const last = samples[samples.length - 1];
if (timeMs >= last.timeMs) {
return { timeMs, cx: last.cx, cy: last.cy };
}
let lo = 0;
let hi = samples.length - 1;
while (lo < hi - 1) {
const mid = (lo + hi) >> 1;
if (samples[mid].timeMs <= timeMs) {
lo = mid;
} else {
hi = mid;
}
}
const a = samples[lo];
const b = samples[hi];
const span = Math.max(1, b.timeMs - a.timeMs);
const t = (timeMs - a.timeMs) / span;
return {
timeMs,
cx: a.cx + (b.cx - a.cx) * t,
cy: a.cy + (b.cy - a.cy) * t,
};
}
function findLatestInteractionSample(
samples: NativeStaticLayoutCursorTelemetrySample[],
timeMs: number,
) {
for (let index = samples.length - 1; index >= 0; index -= 1) {
const sample = samples[index];
if (sample.timeMs > timeMs) {
continue;
}
if (
sample.interactionType === "click" ||
sample.interactionType === "double-click" ||
sample.interactionType === "right-click" ||
sample.interactionType === "middle-click"
) {
return sample;
}
}
return null;
}
function findLatestStableCursorType(
samples: NativeStaticLayoutCursorTelemetrySample[],
timeMs: number,
) {
let lo = 0;
let hi = samples.length - 1;
while (lo < hi) {
const mid = Math.ceil((lo + hi) / 2);
if (samples[mid].timeMs <= timeMs) {
lo = mid;
} else {
hi = mid - 1;
}
}
for (let index = lo; index >= 0; index -= 1) {
const sample = samples[index];
if (sample.timeMs > timeMs || !sample.cursorType) {
continue;
}
if (
sample.interactionType === "click" ||
sample.interactionType === "double-click" ||
sample.interactionType === "right-click" ||
sample.interactionType === "middle-click"
) {
continue;
}
return sample.cursorType;
}
return samples[lo]?.cursorType ?? "arrow";
}
function getCursorTypeIndex(cursorType: string | undefined) {
return CURSOR_TYPE_INDEX[cursorType ?? "arrow"] ?? CURSOR_TYPE_INDEX.arrow;
}
function getCursorBounceScale(
samples: NativeStaticLayoutCursorTelemetrySample[],
timeMs: number,
options: NativeStaticLayoutCursorTelemetryOptions,
) {
const latestClick = findLatestInteractionSample(samples, timeMs);
if (!latestClick) {
return 1;
}
const clickBounceDurationMs = Math.max(
1,
options.clickBounceDurationMs ?? DEFAULT_CLICK_BOUNCE_DURATION_MS,
);
const ageMs = Math.max(0, timeMs - latestClick.timeMs);
if (ageMs > clickBounceDurationMs) {
return 1;
}
const clickBounce = Math.max(0, options.clickBounce ?? DEFAULT_CLICK_BOUNCE);
const progress = 1 - ageMs / clickBounceDurationMs;
return Math.max(0.72, 1 - Math.sin(progress * Math.PI) * (0.08 * clickBounce));
}
function buildCursorRenderSample(
samples: NativeStaticLayoutCursorTelemetrySample[],
timeMs: number,
options: NativeStaticLayoutCursorTelemetryOptions,
): NativeStaticLayoutCursorTelemetrySample {
const position = interpolateCursorSample(samples, timeMs);
const cursorType = findLatestStableCursorType(samples, timeMs);
return {
...position,
cursorType,
cursorTypeIndex: getCursorTypeIndex(cursorType),
bounceScale: getCursorBounceScale(samples, timeMs, options),
};
}
function pushCursorSample(
samples: NativeStaticLayoutCursorTelemetrySample[],
sample: NativeStaticLayoutCursorTelemetrySample,
) {
const previous = samples[samples.length - 1];
if (
previous &&
Math.abs(previous.cx - sample.cx) <= CURSOR_POSITION_EPSILON &&
Math.abs(previous.cy - sample.cy) <= CURSOR_POSITION_EPSILON &&
previous.cursorTypeIndex === sample.cursorTypeIndex &&
Math.abs((previous.bounceScale ?? 1) - (sample.bounceScale ?? 1)) <= CURSOR_BOUNCE_EPSILON
) {
previous.timeMs = sample.timeMs;
return;
}
samples.push(sample);
}
export function buildNativeStaticLayoutCursorTelemetry(
telemetry: NativeStaticLayoutCursorTelemetrySample[],
options: NativeStaticLayoutCursorTelemetryOptions,
) {
const sanitized = sanitizeCursorTelemetry(telemetry);
if (sanitized.length === 0) {
return undefined;
}
const frameRate = Math.max(1, Math.round(options.frameRate));
const durationMs = Number.isFinite(options.durationSec)
? Math.max(0, options.durationSec * 1000)
: sanitized[sanitized.length - 1].timeMs;
const frameDurationMs = 1000 / frameRate;
const targetFrames = Math.max(1, Math.floor(durationMs / frameDurationMs) + 1);
const resampled: NativeStaticLayoutCursorTelemetrySample[] = [];
for (let frameIndex = 0; frameIndex < targetFrames; frameIndex += 1) {
const timeMs = Math.min(durationMs, frameIndex * frameDurationMs);
pushCursorSample(resampled, buildCursorRenderSample(sanitized, timeMs, options));
}
const last = resampled[resampled.length - 1];
if (!last || Math.abs(last.timeMs - durationMs) > 0.5) {
pushCursorSample(resampled, buildCursorRenderSample(sanitized, durationMs, options));
}
return resampled;
}
+57
View File
@@ -6,6 +6,7 @@ export interface ExportConfig {
codec?: string;
encodingMode?: ExportEncodingMode;
backendPreference?: ExportBackendPreference;
preferredRenderBackend?: ExportRenderBackend;
experimentalNativeExport?: boolean;
maxEncodeQueue?: number;
maxDecodeQueue?: number;
@@ -28,6 +29,7 @@ export interface ExportProgress {
renderBackend?: ExportRenderBackend;
encodeBackend?: ExportEncodeBackend;
encoderName?: string;
nativeStaticLayoutSkipReason?: string;
phase?: "extracting" | "finalizing" | "saving"; // Phase of export
renderProgress?: number; // 0-100, progress of GIF rendering phase
audioProgress?: number; // 0-1, progress of real-time audio rendering (speed/audio regions)
@@ -53,6 +55,60 @@ export interface ExportFfmpegAudioMuxBreakdown {
tempVideoBytes?: number;
tempEditedAudioBytes?: number;
muxedVideoBytes?: number;
chunkCount?: number;
chunkDurationSec?: number;
chunkExecMs?: number;
concatExecMs?: number;
staticAssetExecMs?: number;
fallbackChunkCount?: number;
videoOnlyBytes?: number;
chunks?: Array<{
index: number;
startSec: number;
durationSec: number;
backend: string;
elapsedMs: number;
outputBytes: number;
fallbackReason?: string;
windowsGpuSummary?: {
success?: boolean;
width?: number;
height?: number;
fps?: number;
seconds?: number;
mediaMs?: number;
frames?: number;
gpuDecodeSurface?: boolean;
webcamOverlay?: boolean;
cursorOverlay?: boolean;
zoomOverlay?: boolean;
surfacePoolSize?: number;
adapterIndex?: number;
adapterVendorId?: number;
adapterDeviceId?: number;
adapterDedicatedVideoMemoryMB?: number;
encoderBackend?: string;
encoderTuningApplied?: boolean;
nvencOutputBytes?: number;
initializeMs?: number;
initCoInitializeMs?: number;
initMfStartupMs?: number;
initD3DDeviceMs?: number;
initSourceReaderMs?: number;
initWebcamReaderMs?: number;
initVideoProcessorMs?: number;
initTexturesMs?: number;
initShaderPipelineMs?: number;
initSinkWriterMs?: number;
totalMs?: number;
readMs?: number;
clearMs?: number;
videoProcessMs?: number;
writeSampleMs?: number;
finalizeMs?: number;
realtimeMultiplier?: number;
};
}>;
}
export interface ExportMetrics {
@@ -75,6 +131,7 @@ export interface ExportMetrics {
encodeBackend?: ExportEncodeBackend;
encoderName?: string;
backpressureProfile?: string;
nativeStaticLayoutSkipReason?: string;
averageFrameCallbackMs?: number;
averageRenderFrameMs?: number;
averageEncodeWaitMs?: number;
+32 -2
View File
@@ -1,9 +1,10 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
BUILT_IN_WALLPAPERS,
DEFAULT_WALLPAPER_PATH,
DEFAULT_WALLPAPER_RELATIVE_PATH,
getAvailableWallpapers,
resolveAvailableWallpaperPath,
} from "./wallpapers";
describe("wallpapers", () => {
@@ -11,6 +12,10 @@ describe("wallpapers", () => {
vi.unstubAllGlobals();
});
afterEach(() => {
vi.unstubAllGlobals();
});
it("keeps the curated wallpaper list and default path aligned", () => {
expect(DEFAULT_WALLPAPER_PATH).toBe("/wallpapers/midnight-8.jpg");
expect(DEFAULT_WALLPAPER_RELATIVE_PATH).toBe("wallpapers/midnight-8.jpg");
@@ -45,4 +50,29 @@ describe("wallpapers", () => {
BUILT_IN_WALLPAPERS[24],
]);
});
});
it("falls back to the default wallpaper when a bundled wallpaper is missing", async () => {
vi.stubGlobal("window", {
electronAPI: {
listAssetDirectory: vi.fn().mockResolvedValue({
success: true,
files: ["wallpaper2.jpg"],
}),
},
});
await expect(resolveAvailableWallpaperPath("/wallpapers/midnight-8.jpg")).resolves.toBe(
DEFAULT_WALLPAPER_PATH,
);
await expect(resolveAvailableWallpaperPath("/wallpapers/wallpaper2.jpg")).resolves.toBe(
"/wallpapers/wallpaper2.jpg",
);
});
it("preserves non-bundled wallpaper values", async () => {
await expect(resolveAvailableWallpaperPath("#123456")).resolves.toBe("#123456");
await expect(resolveAvailableWallpaperPath("data:image/png;base64,abc")).resolves.toBe(
"data:image/png;base64,abc",
);
});
});
+40
View File
@@ -43,6 +43,46 @@ export const WALLPAPER_RELATIVE_PATHS = BUILT_IN_WALLPAPERS.map(
export const DEFAULT_WALLPAPER_PATH = "/wallpapers/midnight-8.jpg";
export const DEFAULT_WALLPAPER_RELATIVE_PATH = "wallpapers/midnight-8.jpg";
function safeDecodeFileName(fileName: string) {
try {
return decodeURIComponent(fileName);
} catch {
return fileName;
}
}
function getBundledWallpaperFileName(value: string) {
if (!value.startsWith("/wallpapers/")) {
return null;
}
const normalizedValue = value.split("?")[0] ?? value;
const fileName = normalizedValue.split("/").filter(Boolean).pop();
return fileName ? safeDecodeFileName(fileName) : null;
}
export async function resolveAvailableWallpaperPath(wallpaper: string): Promise<string> {
const bundledFileName = getBundledWallpaperFileName(wallpaper);
if (
!bundledFileName ||
typeof window === "undefined" ||
!window.electronAPI?.listAssetDirectory
) {
return wallpaper;
}
try {
const result = await window.electronAPI.listAssetDirectory("wallpapers");
if (!result.success || !result.files?.length) {
return wallpaper;
}
return result.files.includes(bundledFileName) ? wallpaper : DEFAULT_WALLPAPER_PATH;
} catch {
return wallpaper;
}
}
export function isVideoWallpaperSource(value: string): boolean {
if (!value) {
return false;
+1
View File
@@ -20,6 +20,7 @@ export default defineConfig({
rollupOptions: {
external: ["ffmpeg-static", "uiohook-napi"],
output: {
format: "cjs",
entryFileNames: "[name].cjs",
chunkFileNames: "[name].cjs",
},