lee/Recordly
1
0
Fork 0
mirror of https://github.com/webadderallorg/Recordly.git synced 2026-09-24 06:46:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(export): support native cropped layouts

Browse Source
This commit is contained in:
wiiiii123
2026-05-08 03:20:28 +07:00
parent eab365d115
commit 234a7b537a
15 changed files with 440 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
electron/electron-env.d.ts
Vendored
+5
View File
@@ -336,6 +336,10 @@ interface Window {
contentHeight: number;
offsetX: number;
offsetY: number;
sourceCropX?: number;
sourceCropY?: number;
sourceCropWidth?: number;
sourceCropHeight?: number;
backgroundColor: string;
backgroundImagePath?: string | null;
backgroundBlurPx?: number;
@@ -355,6 +359,7 @@ interface Window {
cy: number;
cursorTypeIndex?: number;
bounceScale?: number;
visible?: boolean;
}>;
cursorSize?: number;
cursorAtlasPngDataUrl?: string | null;
electron/ipc/export/native-video.test.ts
+51
View File
@@ -646,6 +646,32 @@ describe("buildExperimentalNvidiaCudaStaticLayoutArgs", () => {
]),
);
});
it("passes source crop coordinates to the CUDA wrapper", () => {
const args = buildExperimentalNvidiaCudaStaticLayoutArgs(
createNvidiaCudaSkipOptions({
sourceCropX: 192,
sourceCropY: 108,
sourceCropWidth: 1536,
sourceCropHeight: 864,
}),
"output.mp4",
"work",
);
expect(args).toEqual(
expect.arrayContaining([
"--source-crop-x",
"192",
"--source-crop-y",
"108",
"--source-crop-width",
"1536",
"--source-crop-height",
"864",
]),
);
});
});
describe("buildExperimentalWindowsGpuStaticLayoutArgs", () => {
@@ -660,6 +686,31 @@ describe("buildExperimentalWindowsGpuStaticLayoutArgs", () => {
expect(args).toEqual(expect.arrayContaining(["--background-blur", "36"]));
});
it("passes source crop coordinates to the D3D11 compositor", () => {
const args = buildExperimentalWindowsGpuStaticLayoutArgs(
createNvidiaCudaSkipOptions({
sourceCropX: 192,
sourceCropY: 108,
sourceCropWidth: 1536,
sourceCropHeight: 864,
}),
"output.mp4",
);
expect(args).toEqual(
expect.arrayContaining([
"--source-crop-x",
"192",
"--source-crop-y",
"108",
"--source-crop-width",
"1536",
"--source-crop-height",
"864",
]),
);
});
});
describe("buildNativeStaticLayoutTimelineSegments", () => {
electron/ipc/export/native-video.ts
+62
View File
@@ -105,6 +105,10 @@ export interface NativeStaticLayoutExportOptions {
contentHeight: number;
offsetX: number;
offsetY: number;
sourceCropX?: number;
sourceCropY?: number;
sourceCropWidth?: number;
sourceCropHeight?: number;
backgroundColor: string;
backgroundImagePath?: string | null;
backgroundBlurPx?: number;
@@ -124,6 +128,7 @@ export interface NativeStaticLayoutExportOptions {
cy: number;
cursorTypeIndex?: number;
bounceScale?: number;
visible?: boolean;
}>;
cursorTelemetryPath?: string | null;
cursorSize?: number;
@@ -1655,6 +1660,15 @@ function hasNativeStaticLayoutTimeline(options: NativeStaticLayoutExportOptions)
return (options.timelineSegments?.length ?? 0) > 0;
}
function hasNativeStaticLayoutSourceCrop(options: NativeStaticLayoutExportOptions) {
return Boolean(
Number.isFinite(options.sourceCropWidth) &&
Number.isFinite(options.sourceCropHeight) &&
(options.sourceCropWidth ?? 0) >= 2 &&
(options.sourceCropHeight ?? 0) >= 2,
);
}
export function buildNativeStaticLayoutTimelineSegments(
segments: NativeVideoExportEditedTrackSegment[],
): NativeStaticLayoutTimelineSegment[] {
@@ -2149,6 +2163,25 @@ export function buildExperimentalWindowsGpuStaticLayoutArgs(
if (options.backgroundImagePath) {
args.push("--background-image", options.backgroundImagePath);
}
if (
Number.isFinite(options.sourceCropX) &&
Number.isFinite(options.sourceCropY) &&
Number.isFinite(options.sourceCropWidth) &&
Number.isFinite(options.sourceCropHeight) &&
(options.sourceCropWidth ?? 0) >= 2 &&
(options.sourceCropHeight ?? 0) >= 2
) {
args.push(
"--source-crop-x",
String(Math.round(options.sourceCropX ?? 0)),
"--source-crop-y",
String(Math.round(options.sourceCropY ?? 0)),
"--source-crop-width",
String(Math.round(options.sourceCropWidth ?? 0)),
"--source-crop-height",
String(Math.round(options.sourceCropHeight ?? 0)),
);
}
if (backgroundBlurPx > 0) {
args.push("--background-blur", formatCliNumber(backgroundBlurPx));
}
@@ -2226,12 +2259,14 @@ async function prepareWindowsGpuCursorTelemetry(
Math.min(8, Math.round(sample.cursorTypeIndex ?? 0)),
);
const bounceScale = Math.min(2, Math.max(0.1, sample.bounceScale ?? 1));
const visible = sample.visible !== false ? 1 : 0;
return [
formatCliNumber(timeMs),
formatCliNumber(cx),
formatCliNumber(cy),
String(cursorTypeIndex),
formatCliNumber(bounceScale),
String(visible),
].join(",");
});
@@ -2318,6 +2353,7 @@ async function prepareNvidiaCudaCursorTelemetry(
cy: Math.min(1, Math.max(0, sample.cy)),
cursorTypeIndex: Math.max(0, Math.min(8, Math.round(sample.cursorTypeIndex ?? 0))),
bounceScale: Math.min(2, Math.max(0.1, sample.bounceScale ?? 1)),
visible: sample.visible !== false,
}));
if (samples.length === 0) {
return null;
@@ -2587,6 +2623,25 @@ export function buildExperimentalNvidiaCudaStaticLayoutArgs(
if (options.backgroundImagePath) {
args.push("--background-image", options.backgroundImagePath);
}
if (
Number.isFinite(options.sourceCropX) &&
Number.isFinite(options.sourceCropY) &&
Number.isFinite(options.sourceCropWidth) &&
Number.isFinite(options.sourceCropHeight) &&
(options.sourceCropWidth ?? 0) >= 2 &&
(options.sourceCropHeight ?? 0) >= 2
) {
args.push(
"--source-crop-x",
String(Math.round(options.sourceCropX ?? 0)),
"--source-crop-y",
String(Math.round(options.sourceCropY ?? 0)),
"--source-crop-width",
String(Math.round(options.sourceCropWidth ?? 0)),
"--source-crop-height",
String(Math.round(options.sourceCropHeight ?? 0)),
);
}
if (backgroundBlurPx > 0) {
args.push("--background-blur", formatCliNumber(backgroundBlurPx));
}
@@ -3158,6 +3213,10 @@ export async function exportNativeStaticLayoutVideo(
contentHeight: options.contentHeight,
offsetX: options.offsetX,
offsetY: options.offsetY,
sourceCropX: options.sourceCropX,
sourceCropY: options.sourceCropY,
sourceCropWidth: options.sourceCropWidth,
sourceCropHeight: options.sourceCropHeight,
backgroundColor: options.backgroundColor,
backgroundImagePath: options.backgroundImagePath,
backgroundBlurPx: options.backgroundBlurPx,
@@ -3463,6 +3522,9 @@ export async function exportNativeStaticLayoutVideo(
if (!didRenderVideo && hasNativeStaticLayoutTimeline(options)) {
throw new Error("Native timeline-map export requires a GPU compositor backend");
}
if (!didRenderVideo && hasNativeStaticLayoutSourceCrop(options)) {
throw new Error("Native crop export requires a GPU compositor backend");
}
if (!didRenderVideo && usePrecompositedLayout) {
const maskPath = path.join(chunkDirectory, "layout-mask.pgm");
electron/ipc/nativeVideoExport.ts
+4
View File
@@ -76,6 +76,10 @@ export interface NativeStaticLayoutExportArgsConfig {
contentHeight: number;
offsetX: number;
offsetY: number;
sourceCropX?: number;
sourceCropY?: number;
sourceCropWidth?: number;
sourceCropHeight?: number;
backgroundColor: string;
backgroundImagePath?: string | null;
backgroundBlurPx?: number;
electron/native/bin/win32-x64/helpers-manifest.json
+6 -6
View File
@@ -19,17 +19,17 @@
},
"recordly-gpu-export": {
"binaryName": "recordly-gpu-export.exe",
"binarySha256": "f147901f01dd5b410f67a5c0feae5064ea7556a878b333aa493e799498abb5e5",
"binarySha256": "4cb3a293fd36f718af55906820d9b3fd78babc855888c2e248f0b918ec1aff3c",
"sourceDir": "electron/native/gpu-export-probe",
"sourceFingerprint": "056a4c41113c471379d0ec6095249c46a79b05fac5fc0e52ce0bfdf6de4d91e1",
"updatedAt": "2026-05-07T15:21:50.702Z"
"sourceFingerprint": "743b386a5f1bbcc99cec5465c3de228d2b045061dead31dfcbf25cf6a1e61de5",
"updatedAt": "2026-05-07T20:13:48.585Z"
},
"recordly-nvidia-cuda-compositor": {
"binaryName": "recordly-nvidia-cuda-compositor.exe",
"binarySha256": "c02a3326b1f840df788a206d288164d428156177ce64aa1331d87e7913826b77",
"binarySha256": "a787531c07142de7c292d1726e0339c97dbce5073d9a0853d539a725265fd945",
"sourceDir": "electron/native/nvidia-cuda-compositor",
"sourceFingerprint": "5388602559cbfb98ab18dd3ad96ce55897c0a548febaeb9304d26a0201b296b0",
"updatedAt": "2026-05-07T15:22:12.014Z"
"sourceFingerprint": "528b599e9d576d81ec087d0d4dc93a79af1bbf30fdb969f44f773bef90146135",
"updatedAt": "2026-05-07T20:14:13.794Z"
}
}
}
electron/native/bin/win32-x64/recordly-gpu-export.exe
BIN
View File
Binary file not shown.
electron/native/bin/win32-x64/recordly-nvidia-cuda-compositor.exe
BIN
View File
Binary file not shown.
electron/native/gpu-export-probe/src/main.cpp
+56 -12
View File
@@ -57,6 +57,10 @@ struct Options {
LONG contentTop = -1;
LONG contentWidth = 0;
LONG contentHeight = 0;
LONG sourceCropLeft = 0;
LONG sourceCropTop = 0;
LONG sourceCropWidth = 0;
LONG sourceCropHeight = 0;
float backgroundR = 0.035f;
float backgroundG = 0.035f;
float backgroundB = 0.045f;
@@ -140,6 +144,7 @@ struct CursorSample {
float cy = 0.0f;
int cursorTypeIndex = 0;
float bounceScale = 1.0f;
bool visible = true;
};
struct CursorAtlasEntry {
@@ -388,6 +393,10 @@ Options parseOptions(int argc, wchar_t** argv) {
options.contentTop = parseLongArg(args, L"--content-top", options.contentTop);
options.contentWidth = parseLongArg(args, L"--content-width", options.contentWidth);
options.contentHeight = parseLongArg(args, L"--content-height", options.contentHeight);
options.sourceCropLeft = parseLongArg(args, L"--source-crop-x", options.sourceCropLeft);
options.sourceCropTop = parseLongArg(args, L"--source-crop-y", options.sourceCropTop);
options.sourceCropWidth = parseLongArg(args, L"--source-crop-width", options.sourceCropWidth);
options.sourceCropHeight = parseLongArg(args, L"--source-crop-height", options.sourceCropHeight);
const auto backgroundColor = getArgValue(args, L"--background-color");
if (!backgroundColor.empty()) {
parseHexColor(
@@ -875,12 +884,7 @@ private:
return false;
}
RECT rect = {
0,
0,
static_cast<LONG>(sourceWidth_),
static_cast<LONG>(sourceHeight_),
};
RECT rect = getSourceCropRect();
RECT outputRect = {
0,
0,
@@ -2024,14 +2028,16 @@ float4 main(PSIn input) : SV_Target {
float cy = 0.0f;
int cursorTypeIndex = 0;
float bounceScale = 1.0f;
int visible = 1;
const int parsed = sscanf_s(
line,
"%lf,%f,%f,%d,%f",
"%lf,%f,%f,%d,%f,%d",
&timeMs,
&cx,
&cy,
&cursorTypeIndex,
&bounceScale);
&bounceScale,
&visible);
if (parsed < 3) {
continue;
}
@@ -2046,6 +2052,7 @@ float4 main(PSIn input) : SV_Target {
std::isfinite(bounceScale)
? std::min(2.0f, std::max(0.1f, bounceScale))
: 1.0f,
parsed >= 6 ? visible != 0 : true,
});
}
std::fclose(file);
@@ -2083,6 +2090,7 @@ float4 main(PSIn input) : SV_Target {
a.cy + (b.cy - a.cy) * t,
a.cursorTypeIndex,
a.bounceScale + (b.bounceScale - a.bounceScale) * t,
a.visible && b.visible,
};
}
@@ -2188,6 +2196,41 @@ float4 main(PSIn input) : SV_Target {
return {left, top, left + safeSize, top + safeSize};
}
RECT getSourceCropRect() const {
if (
options_.sourceCropWidth >= 2 &&
options_.sourceCropHeight >= 2 &&
sourceWidth_ >= 2 &&
sourceHeight_ >= 2
) {
const LONG left = std::min<LONG>(
std::max<LONG>(0, options_.sourceCropLeft),
static_cast<LONG>(sourceWidth_) - 2);
const LONG top = std::min<LONG>(
std::max<LONG>(0, options_.sourceCropTop),
static_cast<LONG>(sourceHeight_) - 2);
const LONG width = (std::min<LONG>(
options_.sourceCropWidth & ~1L,
static_cast<LONG>(sourceWidth_) - left)) & ~1L;
const LONG height = (std::min<LONG>(
options_.sourceCropHeight & ~1L,
static_cast<LONG>(sourceHeight_) - top)) & ~1L;
return {
left,
top,
left + std::max<LONG>(2, width),
top + std::max<LONG>(2, height),
};
}
return {
0,
0,
static_cast<LONG>(sourceWidth_),
static_cast<LONG>(sourceHeight_),
};
}
RECT getContentRect() const {
if (
options_.contentLeft >= 0 &&
@@ -2730,15 +2773,16 @@ float4 main(PSIn input) : SV_Target {
const CursorSample cursor = cursorEnabled
? getCursorSampleAt(static_cast<double>(outputTimestamp) / 10'000.0)
: CursorSample{};
const CursorAtlasEntry* cursorAtlasEntry = cursorEnabled
const bool cursorVisible = cursorEnabled && cursor.visible;
const CursorAtlasEntry* cursorAtlasEntry = cursorVisible
? getCursorAtlasEntry(cursor.cursorTypeIndex)
: nullptr;
const bool cursorAtlasEnabled = cursorAtlasEntry != nullptr;
const float cursorX = cursorEnabled
const float cursorX = cursorVisible
? static_cast<float>(contentRect.left) +
cursor.cx * static_cast<float>(contentRect.right - contentRect.left)
: 0.0f;
const float cursorY = cursorEnabled
const float cursorY = cursorVisible
? static_cast<float>(contentRect.top) +
cursor.cy * static_cast<float>(contentRect.bottom - contentRect.top)
: 0.0f;
@@ -2773,7 +2817,7 @@ float4 main(PSIn input) : SV_Target {
webcamEnabled ? options_.webcamShadow : 0.0f,
webcamEnabled ? 0.42f : 0.0f,
options_.webcamMirror ? 1.0f : 0.0f,
cursorEnabled ? 1.0f : 0.0f,
cursorVisible ? 1.0f : 0.0f,
cursorX,
cursorY,
options_.cursorSize,
electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs
+17
View File
@@ -505,6 +505,7 @@ function writeCursorSamples(cursorPayload, outputPath) {
? Math.max(0, Math.min(8, Math.round(sample.cursorTypeIndex)))
: 0),
Number(bounceScale.toFixed(4)),
sample.visible === false ? 0 : 1,
].join("\t");
})
.filter(Boolean)
@@ -951,6 +952,10 @@ const contentX = Math.round(getNonNegativeNumberArg("--content-x", 0));
const contentY = Math.round(getNonNegativeNumberArg("--content-y", 0));
const contentWidth = Math.round(getNumberArg("--content-width", 0));
const contentHeight = Math.round(getNumberArg("--content-height", 0));
const sourceCropX = Math.round(getNonNegativeNumberArg("--source-crop-x", 0));
const sourceCropY = Math.round(getNonNegativeNumberArg("--source-crop-y", 0));
const sourceCropWidth = Math.round(getNumberArg("--source-crop-width", 0));
const sourceCropHeight = Math.round(getNumberArg("--source-crop-height", 0));
const radius = Math.round(getNonNegativeNumberArg("--radius", 0));
const backgroundY = Math.round(getNonNegativeNumberArg("--background-y", 16));
const backgroundU = Math.round(getNonNegativeNumberArg("--background-u", 128));
@@ -1322,6 +1327,18 @@ if (contentWidth > 0 && contentHeight > 0) {
if (backgroundNv12Path) {
encodeArgs.push("--background-nv12", backgroundNv12Path);
}
if (sourceCropWidth >= 2 && sourceCropHeight >= 2) {
encodeArgs.push(
"--source-crop-x",
String(sourceCropX),
"--source-crop-y",
String(sourceCropY),
"--source-crop-width",
String(sourceCropWidth),
"--source-crop-height",
String(sourceCropHeight),
);
}
if (!shouldBakeStaticShadow && shadowOffsetY > 0 && shadowIntensityPct > 0) {
encodeArgs.push(
"--shadow-offset-y",
electron/native/nvidia-cuda-compositor/src/main.cu
+98 -20
View File
@@ -54,6 +54,10 @@ struct Options {
int contentY = 0;
int contentWidth = 0;
int contentHeight = 0;
int sourceCropX = 0;
int sourceCropY = 0;
int sourceCropWidth = 0;
int sourceCropHeight = 0;
int radius = 0;
int backgroundY = 16;
int backgroundU = 128;
@@ -201,6 +205,14 @@ Options parseOptions(int argc, char** argv) {
options.contentWidth = parsePositiveInt(requireValue("--content-width"), "--content-width");
} else if (arg == "--content-height") {
options.contentHeight = parsePositiveInt(requireValue("--content-height"), "--content-height");
} else if (arg == "--source-crop-x") {
options.sourceCropX = parseNonNegativeInt(requireValue("--source-crop-x"), "--source-crop-x");
} else if (arg == "--source-crop-y") {
options.sourceCropY = parseNonNegativeInt(requireValue("--source-crop-y"), "--source-crop-y");
} else if (arg == "--source-crop-width") {
options.sourceCropWidth = parsePositiveInt(requireValue("--source-crop-width"), "--source-crop-width");
} else if (arg == "--source-crop-height") {
options.sourceCropHeight = parsePositiveInt(requireValue("--source-crop-height"), "--source-crop-height");
} else if (arg == "--radius") {
options.radius = parseNonNegativeInt(requireValue("--radius"), "--radius");
} else if (arg == "--background-y") {
@@ -623,6 +635,7 @@ struct CursorSample {
double cy = 0.0;
int typeIndex = 0;
double bounceScale = 1.0;
bool visible = true;
};
struct CursorPosition {
@@ -641,10 +654,22 @@ struct CursorTrack {
return {};
}
if (timeMs <= samples.front().timeMs) {
return {true, samples.front().cx, samples.front().cy, samples.front().typeIndex, samples.front().bounceScale};
return {
samples.front().visible,
samples.front().cx,
samples.front().cy,
samples.front().typeIndex,
samples.front().bounceScale,
};
}
if (timeMs >= samples.back().timeMs) {
return {true, samples.back().cx, samples.back().cy, samples.back().typeIndex, samples.back().bounceScale};
return {
samples.back().visible,
samples.back().cx,
samples.back().cy,
samples.back().typeIndex,
samples.back().bounceScale,
};
}
int low = 0;
@@ -662,12 +687,12 @@ struct CursorTrack {
const CursorSample& right = samples[high];
const double span = right.timeMs - left.timeMs;
if (span <= 0.0) {
return {true, left.cx, left.cy, left.typeIndex, left.bounceScale};
return {left.visible, left.cx, left.cy, left.typeIndex, left.bounceScale};
}
const double t = (timeMs - left.timeMs) / span;
return {
true,
left.visible && right.visible,
left.cx + (right.cx - left.cx) * t,
left.cy + (right.cy - left.cy) * t,
t < 0.5 ? left.typeIndex : right.typeIndex,
@@ -706,6 +731,10 @@ std::unique_ptr<CursorTrack> loadCursorTrack(const Options& options) {
if (!(row >> sample.bounceScale)) {
sample.bounceScale = 1.0;
}
int visible = 1;
if (row >> visible) {
sample.visible = visible != 0;
}
if (sample.cx < -1.0 || sample.cx > 2.0 || sample.cy < -1.0 || sample.cy > 2.0) {
continue;
}
@@ -1046,7 +1075,11 @@ __global__ void overlayContentRectNv12Kernel(
int contentX,
int contentY,
int contentWidth,
int contentHeight) {
int contentHeight,
int sourceCropX,
int sourceCropY,
int sourceCropWidth,
int sourceCropHeight) {
const int localX = blockIdx.x * blockDim.x + threadIdx.x;
const int localY = blockIdx.y * blockDim.y + threadIdx.y;
if (localX >= contentWidth || localY >= contentHeight) {
@@ -1059,15 +1092,19 @@ __global__ void overlayContentRectNv12Kernel(
return;
}
const int srcX = min(srcWidth - 1, (localX * srcWidth) / contentWidth);
const int srcY = min(srcHeight - 1, (localY * srcHeight) / contentHeight);
const int cropWidth = max(1, min(sourceCropWidth > 0 ? sourceCropWidth : srcWidth, srcWidth - sourceCropX));
const int cropHeight = max(1, min(sourceCropHeight > 0 ? sourceCropHeight : srcHeight, srcHeight - sourceCropY));
const int cropX = max(0, min(sourceCropX, srcWidth - 1));
const int cropY = max(0, min(sourceCropY, srcHeight - 1));
const int srcX = min(srcWidth - 1, cropX + (localX * cropWidth) / contentWidth);
const int srcY = min(srcHeight - 1, cropY + (localY * cropHeight) / contentHeight);
dst[y * dstPitch + x] = src[srcY * srcPitch + srcX];
if ((x % 2) == 0 && (y % 2) == 0) {
const int localUvX = max(0, min(contentWidth - 1, localX + 1));
const int localUvY = max(0, min(contentHeight - 1, localY + 1));
const int srcUvX = min(srcWidth - 2, ((localUvX * srcWidth) / contentWidth) & ~1);
const int srcUvY = min((srcHeight / 2) - 1, ((localUvY * srcHeight) / contentHeight) / 2);
const int srcUvX = min(srcWidth - 2, (cropX + ((localUvX * cropWidth) / contentWidth)) & ~1);
const int srcUvY = min((srcHeight / 2) - 1, (cropY + ((localUvY * cropHeight) / contentHeight)) / 2);
const unsigned char* srcUv = src + srcPitch * srcSurfaceHeight + srcUvY * srcPitch + srcUvX;
unsigned char* dstUv = dst + dstChromaOffset + (y / 2) * dstPitch + x;
dstUv[0] = srcUv[0];
@@ -1099,6 +1136,8 @@ __global__ void overlayContentTransformNv12Kernel(
float invZoomScale,
float srcScaleX,
float srcScaleY,
int sourceCropX,
int sourceCropY,
float zoomX,
float zoomY) {
const int localX = blockIdx.x * blockDim.x + threadIdx.x;
@@ -1125,8 +1164,10 @@ __global__ void overlayContentTransformNv12Kernel(
fminf(static_cast<float>(contentWidth - 1), fmaxf(0.0f, layoutXf - contentX));
const float localContentY =
fminf(static_cast<float>(contentHeight - 1), fmaxf(0.0f, layoutYf - contentY));
const int sx = min(srcWidth - 1, __float2int_rd(localContentX * srcScaleX));
const int sy = min(srcHeight - 1, __float2int_rd(localContentY * srcScaleY));
const int cropX = max(0, min(sourceCropX, srcWidth - 1));
const int cropY = max(0, min(sourceCropY, srcHeight - 1));
const int sx = min(srcWidth - 1, cropX + __float2int_rd(localContentX * srcScaleX));
const int sy = min(srcHeight - 1, cropY + __float2int_rd(localContentY * srcScaleY));
dst[y * dstPitch + x] = src[sy * srcPitch + sx];
if ((x % 2) == 0 && (y % 2) == 0 && x + 1 < dstWidth && y + 1 < dstHeight) {
@@ -1147,9 +1188,9 @@ __global__ void overlayContentTransformNv12Kernel(
const float uvLocalContentY =
fminf(static_cast<float>(contentHeight - 1), fmaxf(0.0f, uvLayoutYf - contentY));
const int suvX =
min(srcWidth - 2, __float2int_rd(uvLocalContentX * srcScaleX) & ~1);
min(srcWidth - 2, (cropX + __float2int_rd(uvLocalContentX * srcScaleX)) & ~1);
const int suvY =
min((srcHeight / 2) - 1, __float2int_rd(uvLocalContentY * srcScaleY) / 2);
min((srcHeight / 2) - 1, (cropY + __float2int_rd(uvLocalContentY * srcScaleY)) / 2);
const unsigned char* srcUv = src + srcPitch * srcSurfaceHeight + suvY * srcPitch + suvX;
unsigned char* dstUv = dst + dstChromaOffset + (y / 2) * dstPitch + x;
dstUv[0] = srcUv[0];
@@ -1438,6 +1479,10 @@ __global__ void compositeStaticNv12Kernel(
int contentY,
int contentWidth,
int contentHeight,
int sourceCropX,
int sourceCropY,
int sourceCropWidth,
int sourceCropHeight,
int radius,
unsigned char backgroundY,
unsigned char backgroundU,
@@ -1482,13 +1527,17 @@ __global__ void compositeStaticNv12Kernel(
const int layoutX = static_cast<int>(floorf(layoutXf));
const int layoutY = static_cast<int>(floorf(layoutYf));
const int cropX = max(0, min(sourceCropX, srcWidth - 1));
const int cropY = max(0, min(sourceCropY, srcHeight - 1));
const int cropWidth = max(1, min(sourceCropWidth > 0 ? sourceCropWidth : srcWidth, srcWidth - cropX));
const int cropHeight = max(1, min(sourceCropHeight > 0 ? sourceCropHeight : srcHeight, srcHeight - cropY));
const bool inside = isInsideRoundedRect(layoutX, layoutY, contentX, contentY, contentWidth, contentHeight, radius);
unsigned char outY = background ? background[y * dstWidth + x] : backgroundY;
if (inside) {
const float localX = fminf(static_cast<float>(contentWidth - 1), fmaxf(0.0f, layoutXf - contentX));
const float localY = fminf(static_cast<float>(contentHeight - 1), fmaxf(0.0f, layoutYf - contentY));
const int sx = min(srcWidth - 1, static_cast<int>((localX * srcWidth) / contentWidth));
const int sy = min(srcHeight - 1, static_cast<int>((localY * srcHeight) / contentHeight));
const int sx = min(srcWidth - 1, cropX + static_cast<int>((localX * cropWidth) / contentWidth));
const int sy = min(srcHeight - 1, cropY + static_cast<int>((localY * cropHeight) / contentHeight));
outY = src[sy * srcPitch + sx];
} else {
const bool shadowInside =
@@ -1590,8 +1639,10 @@ __global__ void compositeStaticNv12Kernel(
if (uvInside) {
const float localX = fminf(static_cast<float>(contentWidth - 1), fmaxf(0.0f, uvLayoutXf - contentX));
const float localY = fminf(static_cast<float>(contentHeight - 1), fmaxf(0.0f, uvLayoutYf - contentY));
const int suvX = min(srcWidth - 2, (static_cast<int>((localX * srcWidth) / contentWidth)) & ~1);
const int suvY = min((srcHeight / 2) - 1, static_cast<int>(localY * srcHeight / contentHeight) / 2);
const int suvX =
min(srcWidth - 2, (cropX + static_cast<int>((localX * cropWidth) / contentWidth)) & ~1);
const int suvY =
min((srcHeight / 2) - 1, (cropY + static_cast<int>(localY * cropHeight / contentHeight)) / 2);
const unsigned char* srcUv = src + srcPitch * srcSurfaceHeight + suvY * srcPitch + suvX;
dstUv[0] = srcUv[0];
dstUv[1] = srcUv[1];
@@ -2050,6 +2101,21 @@ public:
const int cursorY = cursorPosition.visible
? static_cast<int>(std::round(cursorHotspotOutputY)) - cursorHotspotY
: 0;
const bool hasSourceCrop =
layoutOptions_.sourceCropWidth >= 2 &&
layoutOptions_.sourceCropHeight >= 2;
const int sourceCropX = hasSourceCrop
? std::max(0, std::min(layoutOptions_.sourceCropX, srcWidth - 2)) & ~1
: 0;
const int sourceCropY = hasSourceCrop
? std::max(0, std::min(layoutOptions_.sourceCropY, srcHeight - 2)) & ~1
: 0;
const int sourceCropWidth = hasSourceCrop
? std::max(2, std::min(layoutOptions_.sourceCropWidth, srcWidth - sourceCropX)) & ~1
: srcWidth;
const int sourceCropHeight = hasSourceCrop
? std::max(2, std::min(layoutOptions_.sourceCropHeight, srcHeight - sourceCropY)) & ~1
: srcHeight;
const bool zoomChangesLayout =
zoomTrack_ &&
(std::abs(zoomSample.scale - 1.0) > 0.001 ||
@@ -2190,9 +2256,9 @@ public:
const float safeZoomScale = std::max(0.01f, static_cast<float>(zoomSample.scale));
const float invZoomScale = 1.0f / safeZoomScale;
const float srcScaleX =
static_cast<float>(srcWidth) / static_cast<float>(std::max(1, layoutOptions_.contentWidth));
static_cast<float>(sourceCropWidth) / static_cast<float>(std::max(1, layoutOptions_.contentWidth));
const float srcScaleY =
static_cast<float>(srcHeight) / static_cast<float>(std::max(1, layoutOptions_.contentHeight));
static_cast<float>(sourceCropHeight) / static_cast<float>(std::max(1, layoutOptions_.contentHeight));
const int transformedContentX = zoomChangesLayout
? static_cast<int>(std::floor(layoutOptions_.contentX * safeZoomScale + zoomSample.x))
: layoutOptions_.contentX;
@@ -2248,6 +2314,8 @@ public:
invZoomScale,
srcScaleX,
srcScaleY,
sourceCropX,
sourceCropY,
static_cast<float>(zoomSample.x),
static_cast<float>(zoomSample.y));
checkCuda(cudaGetLastError(), "overlayContentTransformNv12Kernel");
@@ -2266,7 +2334,11 @@ public:
layoutOptions_.contentX,
layoutOptions_.contentY,
layoutOptions_.contentWidth,
layoutOptions_.contentHeight);
layoutOptions_.contentHeight,
sourceCropX,
sourceCropY,
sourceCropWidth,
sourceCropHeight);
checkCuda(cudaGetLastError(), "overlayContentRectNv12Kernel");
const int cornerRadius = std::min(
@@ -2388,6 +2460,10 @@ public:
layoutOptions_.contentY,
layoutOptions_.contentWidth,
layoutOptions_.contentHeight,
sourceCropX,
sourceCropY,
sourceCropWidth,
sourceCropHeight,
layoutOptions_.radius,
clampByte(layoutOptions_.backgroundY),
clampByte(layoutOptions_.backgroundU),
@@ -2518,6 +2594,8 @@ private:
layoutOptions_.radius == 0 &&
layoutOptions_.shadowIntensityPct == 0 &&
backgroundDevice_ == nullptr &&
layoutOptions_.sourceCropWidth <= 0 &&
layoutOptions_.sourceCropHeight <= 0 &&
!zoomChangesLayout;
}
electron/preload.ts
+5
View File
@@ -213,6 +213,10 @@ contextBridge.exposeInMainWorld("electronAPI", {
contentHeight: number;
offsetX: number;
offsetY: number;
sourceCropX?: number;
sourceCropY?: number;
sourceCropWidth?: number;
sourceCropHeight?: number;
backgroundColor: string;
backgroundImagePath?: string | null;
backgroundBlurPx?: number;
@@ -232,6 +236,7 @@ contextBridge.exposeInMainWorld("electronAPI", {
cy: number;
cursorTypeIndex?: number;
bounceScale?: number;
visible?: boolean;
}>;
cursorSize?: number;
cursorAtlasPngDataUrl?: string | null;
src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts
+40 -2
View File
@@ -58,6 +58,12 @@ function createExporter(overrides: Record<string, unknown> = {}) {
videoInfo: DecodedVideoInfo,
effectiveDurationSec: number,
) => string[];
getNativeStaticLayoutSourceCrop: (videoInfo: DecodedVideoInfo) => {
x: number;
y: number;
width: number;
height: number;
};
resolveNativeStaticLayoutBackground: () => Promise<unknown>;
createNativeStaticLayoutGradient: (
ctx: CanvasRenderingContext2D,
@@ -243,6 +249,29 @@ describe("ModernVideoExporter native static-layout eligibility", () => {
).toBeNull();
});
it("allows non-default crop when native source crop coordinates are valid", () => {
const exporter = createExporter({
cropRegion: { x: 0.1, y: 0.1, width: 0.8, height: 0.8 },
});
expect(
exporter.getNativeStaticLayoutSkipReason(
{
audioMode: "copy-source",
audioSourcePath: "recording.mp4",
},
videoInfo,
60,
),
).toBeNull();
expect(exporter.getNativeStaticLayoutSourceCrop(videoInfo)).toEqual({
x: 192,
y: 108,
width: 1536,
height: 864,
});
});
it("uses the default wallpaper for native static-layout when the project has no wallpaper", async () => {
const exporter = createExporter({ wallpaper: "" });
const electronAPI = window.electronAPI as typeof window.electronAPI & {
@@ -288,7 +317,7 @@ describe("ModernVideoExporter native static-layout eligibility", () => {
autoCaptions: [{ id: "caption-1", text: "hello", startMs: 0, endMs: 1_000 }],
webcam: { enabled: true },
frame: "macbook",
cropRegion: { top: 0.1, bottom: 0, left: 0, right: 0 },
cropRegion: { x: 0.1, y: 0, width: 0.9, height: 1 },
});
expect(
@@ -307,10 +336,19 @@ describe("ModernVideoExporter native static-layout eligibility", () => {
"unsupported-caption-overlay",
"unsupported-webcam-source",
"unsupported-frame-overlay",
"non-default-crop",
]);
});
it("reports invalid crop geometry instead of passing native export bad coordinates", () => {
const exporter = createExporter({
cropRegion: { x: 0, y: 0, width: 0, height: 1 },
});
expect(exporter.getNativeStaticLayoutSkipReason({ audioMode: "none" }, videoInfo, 60)).toBe(
"invalid-crop-region",
);
});
it("materializes uploaded data-url image backgrounds for native static-layout", async () => {
const jpegBytes = new Uint8Array([0xff, 0xd8, 0xff, 0xd9]);
const dataUrl = `data:image/jpeg;base64,${Buffer.from(jpegBytes).toString("base64")}`;
src/lib/exporter/modernVideoExporter.ts
+49 -10
View File
@@ -1237,6 +1237,33 @@ export class ModernVideoExporter {
);
}
private getNativeStaticLayoutSourceCrop(videoInfo: DecodedVideoInfo) {
const crop = this.config.cropRegion;
const sourceWidth = Math.max(2, Math.round(videoInfo.width));
const sourceHeight = Math.max(2, Math.round(videoInfo.height));
const cropX = Math.min(1, Math.max(0, crop.x));
const cropY = Math.min(1, Math.max(0, crop.y));
const cropRight = Math.min(1, Math.max(cropX, crop.x + crop.width));
const cropBottom = Math.min(1, Math.max(cropY, crop.y + crop.height));
const left = Math.min(sourceWidth - 2, Math.max(0, Math.floor(cropX * sourceWidth))) & ~1;
const top = Math.min(sourceHeight - 2, Math.max(0, Math.floor(cropY * sourceHeight))) & ~1;
const right = Math.min(sourceWidth, Math.max(left + 2, Math.ceil(cropRight * sourceWidth)));
const bottom = Math.min(
sourceHeight,
Math.max(top + 2, Math.ceil(cropBottom * sourceHeight)),
);
const width = Math.max(2, right - left) & ~1;
const height = Math.max(2, bottom - top) & ~1;
return {
x: left,
y: top,
width: Math.min(width, sourceWidth - left),
height: Math.min(height, sourceHeight - top),
};
}
private canUseNativeStaticTailTrim(
videoInfo: DecodedVideoInfo,
effectiveDurationSec: number,
@@ -1343,8 +1370,16 @@ export class ModernVideoExporter {
reasons.push("unsupported-frame-overlay");
}
if (!this.isDefaultCropRegion()) {
reasons.push("non-default-crop");
const crop = this.config.cropRegion;
if (
!Number.isFinite(crop.x) ||
!Number.isFinite(crop.y) ||
!Number.isFinite(crop.width) ||
!Number.isFinite(crop.height) ||
crop.width <= 0 ||
crop.height <= 0
) {
reasons.push("invalid-crop-region");
}
return reasons;
@@ -1836,6 +1871,7 @@ export class ModernVideoExporter {
durationSec: this.effectiveDurationSec || 0,
clickBounce: this.config.cursorClickBounce,
clickBounceDurationMs: this.config.cursorClickBounceDuration,
sourceCrop: this.config.cropRegion,
});
}
@@ -2079,8 +2115,8 @@ export class ModernVideoExporter {
videoHeight: videoInfo.height,
});
const contentSize = roundNativeStaticLayoutContentSize({
width: layout.fullVideoDisplayWidth,
height: layout.fullVideoDisplayHeight,
width: layout.croppedDisplayWidth,
height: layout.croppedDisplayHeight,
});
const contentWidth = contentSize.width;
const contentHeight = contentSize.height;
@@ -2096,12 +2132,11 @@ export class ModernVideoExporter {
return null;
}
const offsetX = Math.round(
layout.centerOffsetX + layout.croppedDisplayWidth / 2 - contentWidth / 2,
);
const offsetY = Math.round(
layout.centerOffsetY + layout.croppedDisplayHeight / 2 - contentHeight / 2,
);
const offsetX = Math.round(layout.centerOffsetX);
const offsetY = Math.round(layout.centerOffsetY);
const sourceCrop = this.isDefaultCropRegion()
? null
: this.getNativeStaticLayoutSourceCrop(videoInfo);
const previewWidth = this.config.previewWidth || 1920;
const previewHeight = this.config.previewHeight || 1080;
const canvasScaleFactor = Math.min(
@@ -2274,6 +2309,10 @@ export class ModernVideoExporter {
contentHeight,
offsetX,
offsetY,
sourceCropX: sourceCrop?.x,
sourceCropY: sourceCrop?.y,
sourceCropWidth: sourceCrop?.width,
sourceCropHeight: sourceCrop?.height,
backgroundColor: background.backgroundColor,
backgroundImagePath: background.backgroundImagePath ?? null,
backgroundBlurPx: Math.max(0, (this.config.backgroundBlur ?? 0) * 3),
src/lib/exporter/nativeStaticLayoutTelemetry.test.ts
+36
View File
@@ -109,4 +109,40 @@ describe("buildNativeStaticLayoutCursorTelemetry", () => {
expect(resampled?.map((sample) => sample.cursorTypeIndex)).toContain(1);
expect(resampled?.some((sample) => (sample.bounceScale ?? 1) < 1)).toBe(true);
});
it("projects cursor samples into the cropped viewport and marks out-of-crop samples hidden", () => {
const resampled = buildNativeStaticLayoutCursorTelemetry(
[
{ timeMs: 0, cx: 0.25, cy: 0.5 },
{ timeMs: 1000, cx: 0.75, cy: 0.5 },
],
{
frameRate: 1,
durationSec: 1,
sourceCrop: { x: 0.25, y: 0.25, width: 0.5, height: 0.5 },
},
);
expect(resampled?.[0]).toMatchObject({ timeMs: 0, cx: 0, cy: 0.5, visible: true });
expect(resampled?.[1]).toMatchObject({ timeMs: 1000, cx: 1, cy: 0.5, visible: true });
const hidden = buildNativeStaticLayoutCursorTelemetry(
[
{ timeMs: 0, cx: 0.1, cy: 0.5 },
{ timeMs: 1000, cx: 0.1, cy: 0.5 },
],
{
frameRate: 1,
durationSec: 1,
sourceCrop: { x: 0.25, y: 0.25, width: 0.5, height: 0.5 },
},
);
expect(hidden).toEqual([
expect.objectContaining({
timeMs: 1000,
visible: false,
}),
]);
});
});
src/lib/exporter/nativeStaticLayoutTelemetry.ts
+11 -2
View File
@@ -1,4 +1,5 @@
import type { CursorTelemetryPoint } from "@/components/video-editor/types";
import type { CropRegion, CursorTelemetryPoint } from "@/components/video-editor/types";
import { projectCursorPositionToViewport } from "@/components/video-editor/videoPlayback/cursorViewport";
export type NativeStaticLayoutCursorTelemetrySample = {
timeMs: number;
@@ -8,6 +9,7 @@ export type NativeStaticLayoutCursorTelemetrySample = {
interactionType?: string;
cursorTypeIndex?: number;
bounceScale?: number;
visible?: boolean;
};
export type NativeStaticLayoutCursorTelemetryOptions = {
@@ -15,6 +17,7 @@ export type NativeStaticLayoutCursorTelemetryOptions = {
durationSec: number;
clickBounce?: number;
clickBounceDurationMs?: number;
sourceCrop?: CropRegion;
};
const CURSOR_POSITION_EPSILON = 0.00001;
@@ -186,12 +189,16 @@ function buildCursorRenderSample(
): NativeStaticLayoutCursorTelemetrySample {
const position = interpolateCursorSample(samples, timeMs);
const cursorType = findLatestStableCursorType(samples, timeMs);
const projectedPosition = projectCursorPositionToViewport(position, options.sourceCrop);
return {
...position,
cx: projectedPosition.cx,
cy: projectedPosition.cy,
cursorType,
cursorTypeIndex: getCursorTypeIndex(cursorType),
bounceScale: getCursorBounceScale(samples, timeMs, options),
...(options.sourceCrop ? { visible: projectedPosition.visible } : {}),
};
}
@@ -205,7 +212,9 @@ function pushCursorSample(
Math.abs(previous.cx - sample.cx) <= CURSOR_POSITION_EPSILON &&
Math.abs(previous.cy - sample.cy) <= CURSOR_POSITION_EPSILON &&
previous.cursorTypeIndex === sample.cursorTypeIndex &&
Math.abs((previous.bounceScale ?? 1) - (sample.bounceScale ?? 1)) <= CURSOR_BOUNCE_EPSILON
Math.abs((previous.bounceScale ?? 1) - (sample.bounceScale ?? 1)) <=
CURSOR_BOUNCE_EPSILON &&
(previous.visible ?? true) === (sample.visible ?? true)
) {
previous.timeMs = sample.timeMs;
return;