lee/Recordly
1
0
Fork 0
mirror of https://github.com/webadderallorg/Recordly.git synced 2026-09-25 23:35:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Implement native Windows Graphics Capture monitor utilities and introduce video editor UI components.

Browse Source
This commit is contained in:
Mahdy Arief
2026-03-27 17:22:55 +07:00
parent 7d7b61b7f5
commit ced02c9a8f
10 changed files with 245 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
electron-builder.json5
+4 -3
View File
@@ -9,6 +9,7 @@
"electron/native/**"
],
"productName": "Recordly",
"copyright": "Copyright (c) 2026 webadderall",
"npmRebuild": true,
"buildDependenciesFromSource": true,
"compression": "normal",
@@ -62,9 +63,9 @@
"target": [
"nsis"
],
"icon": "icons/icons/win/icon.ico"
,
"artifactName": "Recordly.exe"
"icon": "icons/icons/win/icon.ico",
"executableName": "Recordly",
"artifactName": "${productName}-Setup-${version}.${ext}"
}
}
electron/electron-env.d.ts
Vendored
+3 -2
View File
@@ -180,7 +180,7 @@ interface Window {
language?: string;
}) => Promise<{
success: boolean;
cues?: AutoCaptionCue[];
cues?: CaptionCue[];
message?: string;
error?: string;
}>;
@@ -302,6 +302,7 @@ interface Window {
startCountdown: (seconds: number) => Promise<{ success: boolean; cancelled?: boolean }>;
cancelCountdown: () => Promise<{ success: boolean }>;
getActiveCountdown: () => Promise<{ success: boolean; seconds: number | null }>;
onAutoCaptionProgress: (callback: (payload: { progress: number }) => void) => () => void;
onCountdownTick: (callback: (seconds: number) => void) => () => void;
};
}
@@ -343,7 +344,7 @@ interface SystemCursorAsset {
height: number;
}
interface AutoCaptionCue {
interface CaptionCue {
id: string;
startMs: number;
endMs: number;
electron/ipc/handlers.ts
+79 -14
View File
@@ -999,6 +999,43 @@ function getFfmpegBinaryPath() {
return ffmpegStatic
}
function runWhisperWithProgress(
executablePath: string,
args: string[],
onProgress: (progress: number) => void,
): Promise<void> {
return new Promise((resolve, reject) => {
const proc = spawn(executablePath, args);
let output = "";
proc.stdout?.on("data", (data) => {
output += data.toString();
});
proc.stderr?.on("data", (data) => {
const text = data.toString();
output += text;
// whisper.cpp variants use this pattern on stderr
const match = text.match(/progress\s*=\s*(\d+)%/i);
if (match) {
onProgress(parseInt(match[1], 10));
}
});
proc.on("close", (code) => {
if (code === 0) {
resolve();
} else {
reject(new Error(output.trim() || `Whisper exited with code ${code}`));
}
});
proc.on("error", (err) => {
reject(err);
});
});
}
function sendWhisperModelDownloadProgress(
webContents: Electron.WebContents,
payload: {
@@ -1449,11 +1486,13 @@ async function extractCaptionAudioSource(options: {
for (const candidate of candidates) {
try {
await ensureReadableFile(candidate.path, 'video file')
console.log('[auto-captions] Extracting audio from:', candidate.path)
await execFileAsync(
options.ffmpegPath,
['-y', '-i', candidate.path, '-map', '0:a:0', '-vn', '-ac', '1', '-ar', '16000', '-c:a', 'pcm_s16le', options.wavPath],
{ timeout: 5 * 60 * 1000, maxBuffer: 20 * 1024 * 1024 },
)
console.log('[auto-captions] Audio extracted successfully to:', options.wavPath)
attemptedCandidates.push({ ...candidate, readable: true, extractedAudio: true })
return candidate
} catch (error) {
@@ -1472,12 +1511,15 @@ async function extractCaptionAudioSource(options: {
throw new Error('No audio was found to transcribe in the saved recording file. Captions need an audio track. If this recording should have contained sound, the recording was saved without an audio stream.')
}
async function generateAutoCaptionsFromVideo(options: {
videoPath: string
whisperExecutablePath?: string
whisperModelPath: string
language?: string
}) {
async function generateAutoCaptionsFromVideo(
webContents: Electron.WebContents,
options: {
videoPath: string;
whisperExecutablePath?: string;
whisperModelPath: string;
language?: string;
},
) {
const ffmpegPath = getFfmpegBinaryPath()
const normalizedVideoPath = normalizeVideoSourcePath(options.videoPath)
if (!normalizedVideoPath) {
@@ -1489,6 +1531,12 @@ async function generateAutoCaptionsFromVideo(options: {
await ensureReadableFile(whisperExecutablePath, 'whisper executable')
await ensureReadableFile(whisperModelPath, 'whisper model')
console.log('[auto-captions] Starting caption generation sequence')
console.log('[auto-captions] Video:', normalizedVideoPath)
console.log('[auto-captions] Runtime:', whisperExecutablePath)
console.log('[auto-captions] Model:', whisperModelPath)
console.log('[auto-captions] Language:', options.language || 'auto')
const tempBase = path.join(app.getPath('temp'), `recordly-captions-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`)
const wavPath = `${tempBase}.wav`
const outputBase = `${tempBase}-whisper`
@@ -1514,10 +1562,11 @@ async function generateAutoCaptionsFromVideo(options: {
let jsonEnabled = true
try {
await execFileAsync(whisperExecutablePath, [...whisperBaseArgs, '-ojf'], {
timeout: 30 * 60 * 1000,
maxBuffer: 20 * 1024 * 1024,
console.log('[auto-captions] Running Whisper with JSON output...')
await runWhisperWithProgress(whisperExecutablePath, [...whisperBaseArgs, '-ojf'], (progress) => {
webContents.send('auto-caption-progress', { progress })
})
console.log('[auto-captions] Whisper JSON output generated.')
} catch (error) {
if (!shouldRetryWhisperWithoutJson(error)) {
throw error
@@ -1525,10 +1574,11 @@ async function generateAutoCaptionsFromVideo(options: {
jsonEnabled = false
console.warn('[auto-captions] Whisper runtime does not support JSON full output, retrying with SRT only:', error)
await execFileAsync(whisperExecutablePath, whisperBaseArgs, {
timeout: 30 * 60 * 1000,
maxBuffer: 20 * 1024 * 1024,
console.log('[auto-captions] Running Whisper with SRT output...')
await runWhisperWithProgress(whisperExecutablePath, whisperBaseArgs, (progress) => {
webContents.send('auto-caption-progress', { progress })
})
console.log('[auto-captions] Whisper SRT output generated.')
}
const timedCues = jsonEnabled
@@ -1538,9 +1588,12 @@ async function generateAutoCaptionsFromVideo(options: {
? timedCues
: parseSrtCues(await fs.readFile(srtPath, 'utf-8'))
if (cues.length === 0) {
console.error('[auto-captions] No cues were parsed from Whisper output.')
throw new Error('Whisper completed, but no caption cues were produced.')
}
console.log(`[auto-captions] Successfully generated ${cues.length} cues.`)
return {
cues,
audioSourceLabel: audioSource.label,
@@ -3138,6 +3191,18 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
config.displayId = Number.isFinite(screenId) && screenId > 0
? screenId
: Number(getScreen().getPrimaryDisplay().id)
// Include display bounds for more robust matching in the native helper
const allDisplays = getScreen().getAllDisplays()
const display = allDisplays.find((d) => String(d.id) === String(config.displayId))
|| getScreen().getPrimaryDisplay()
if (display) {
config.displayX = Math.round(display.bounds.x)
config.displayY = Math.round(display.bounds.y)
config.displayW = Math.round(display.bounds.width)
config.displayH = Math.round(display.bounds.height)
}
}
windowsCaptureOutputBuffer = ''
@@ -4080,14 +4145,14 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
}
})
ipcMain.handle('generate-auto-captions', async (_, options: {
ipcMain.handle('generate-auto-captions', async (event, options: {
videoPath: string
whisperExecutablePath: string
whisperModelPath: string
language?: string
}) => {
try {
const result = await generateAutoCaptionsFromVideo(options)
const result = await generateAutoCaptionsFromVideo(event.sender, options)
return {
success: true,
cues: result.cues,
electron/native/wgc-capture/src/main.cpp
+23
View File
@@ -34,6 +34,11 @@ struct CaptureConfig {
int fps = 60;
int width = 0;
int height = 0;
int displayX = 0;
int displayY = 0;
int displayW = 0;
int displayH = 0;
bool hasDisplayBounds = false;
bool captureSystemAudio = false;
bool captureMic = false;
};
@@ -129,6 +134,18 @@ static bool parseSimpleJson(const std::string& json, CaptureConfig& config) {
config.captureSystemAudio = findBool("captureSystemAudio");
config.captureMic = findBool("captureMic");
int dx = findInt("displayX");
int dy = findInt("displayY");
int dw = findInt("displayW");
int dh = findInt("displayH");
if (dw > 0 && dh > 0) {
config.displayX = dx;
config.displayY = dy;
config.displayW = dw;
config.displayH = dh;
config.hasDisplayBounds = true;
}
return true;
}
@@ -200,6 +217,12 @@ int main(int argc, char* argv[]) {
}
} else {
HMONITOR monitor = findMonitorByDisplayId(config.displayId);
if (!monitor && config.hasDisplayBounds) {
std::cerr << "Monitor ID match failed, attempting coordinate-based match: "
<< config.displayX << "," << config.displayY << " " << config.displayW << "x" << config.displayH << std::endl;
monitor = findMonitorByBounds(config.displayX, config.displayY, config.displayW, config.displayH);
}
if (!monitor) {
std::cerr << "ERROR: Could not find monitor for displayId " << config.displayId << std::endl;
return 1;
electron/native/wgc-capture/src/monitor_utils.cpp
+66
View File
@@ -1,5 +1,6 @@
#include "monitor_utils.h"
#include <ShellScalingApi.h>
#include <iostream>
static BOOL CALLBACK enumMonitorCallback(HMONITOR hMonitor, HDC, LPRECT, LPARAM lParam) {
auto* monitors = reinterpret_cast<std::vector<MonitorInfo>*>(lParam);
@@ -26,16 +27,81 @@ std::vector<MonitorInfo> enumerateMonitors() {
return monitors;
}
static std::string wstringToString(const std::wstring& wstr) {
if (wstr.empty()) return "";
int size_needed = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), (int)wstr.size(), NULL, 0, NULL, NULL);
std::string str(size_needed, 0);
WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), (int)wstr.size(), &str[0], size_needed, NULL, NULL);
return str;
}
// Electron uses the HMONITOR handle value cast to a number as the display ID.
HMONITOR findMonitorByDisplayId(int64_t displayId) {
auto monitors = enumerateMonitors();
// Try exact match first
for (const auto& m : monitors) {
if (static_cast<int64_t>(reinterpret_cast<intptr_t>(m.handle)) == displayId) {
return m.handle;
}
}
// Try matching lower 32-bits (HMONITOR handles on Windows are often 32-bit values zero-extended or sign-extended to 64-bit)
for (const auto& m : monitors) {
if ((static_cast<int64_t>(reinterpret_cast<intptr_t>(m.handle)) & 0xFFFFFFFF) == (displayId & 0xFFFFFFFF)) {
std::cerr << "WARNING: Found monitor via 32-bit partial match. Target: " << displayId
<< ", Found: " << static_cast<int64_t>(reinterpret_cast<intptr_t>(m.handle)) << std::endl;
return m.handle;
}
}
// If we only have one monitor and we couldn't match by ID, just use the only one we have.
// This is a safe fallback for the most common case (single monitor setups) where Electron
// and native Windows might report different IDs for the same display.
if (monitors.size() == 1) {
std::cerr << "WARNING: Found only one monitor, using it as fallback for displayId " << displayId
<< " (Handle: " << reinterpret_cast<intptr_t>(monitors[0].handle) << ")" << std::endl;
return monitors[0].handle;
}
// Debug: Print available monitors if not found
std::cerr << "ERROR: Monitor matching failed for displayId " << displayId << ". Enumerated " << monitors.size() << " monitors:" << std::endl;
for (const auto& m : monitors) {
std::cerr << " - Handle: " << reinterpret_cast<intptr_t>(m.handle)
<< " (device: " << wstringToString(m.deviceName)
<< ", bounds: " << m.x << "," << m.y << " " << m.width << "x" << m.height << ")" << std::endl;
}
return nullptr;
}
HMONITOR findMonitorByBounds(int x, int y, int width, int height) {
auto monitors = enumerateMonitors();
// 1. Try exact bounds match
for (const auto& m : monitors) {
if (m.x == x && m.y == y && m.width == width && m.height == height) {
std::cerr << "Found monitor by exact bounds: " << x << "," << y << " " << width << "x" << height << std::endl;
return m.handle;
}
}
// 2. Try matching top-left point (sometimes size differs due to DPI scaling, but top-left is usually more stable in screen-space)
for (const auto& m : monitors) {
if (m.x == x && m.y == y) {
std::cerr << "Found monitor by top-left point match: " << x << "," << y << std::endl;
return m.handle;
}
}
// 3. Last resort: MonitorFromRect/Point (OS choice for the best matching monitor for these coordinates)
RECT rect = { x, y, x + width, y + height };
HMONITOR hMonitor = MonitorFromRect(&rect, MONITOR_DEFAULTTONULL);
if (hMonitor) {
std::cerr << "Found monitor via Windows OS MonitorFromRect fallback" << std::endl;
return hMonitor;
}
return nullptr;
}
electron/native/wgc-capture/src/monitor_utils.h
+1
View File
@@ -16,4 +16,5 @@ struct MonitorInfo {
std::vector<MonitorInfo> enumerateMonitors();
HMONITOR findMonitorByDisplayId(int64_t displayId);
HMONITOR findMonitorByBounds(int x, int y, int width, int height);
MonitorInfo getMonitorInfo(HMONITOR monitor);
electron/preload.ts
+6
View File
@@ -203,6 +203,12 @@ contextBridge.exposeInMainWorld("electronAPI", {
}) => {
return ipcRenderer.invoke("generate-auto-captions", options);
},
onAutoCaptionProgress: (callback: (payload: { progress: number }) => void) => {
const listener = (_event: Electron.IpcRendererEvent, payload: { progress: number }) =>
callback(payload);
ipcRenderer.on("auto-caption-progress", listener);
return () => ipcRenderer.removeListener("auto-caption-progress", listener);
},
setCurrentVideoPath: (path: string) => {
return ipcRenderer.invoke("set-current-video-path", path);
},
src/components/video-editor/SettingsPanel.tsx
+42 -22
View File
@@ -220,6 +220,7 @@ interface SettingsPanelProps {
onPickWhisperModel?: () => void;
onGenerateAutoCaptions?: () => void;
onClearAutoCaptions?: () => void;
autoCaptionProgress?: number;
onDownloadWhisperModel?: () => void;
onDeleteWhisperModel?: () => void;
selectedSpeedId?: string | null;
@@ -276,7 +277,7 @@ const CAPTION_LANGUAGE_OPTIONS = [
] as const;
export type WhisperModelInfo = {
value: "tiny" | "base" | "small" | "medium" | "large";
value: "tiny" | "base" | "small" | "medium" | "large" | "custom";
label: string;
size: string;
};
@@ -287,6 +288,7 @@ const WHISPER_MODEL_OPTIONS: WhisperModelInfo[] = [
{ value: "small", label: "Small", size: "466 MB" },
{ value: "medium", label: "Medium", size: "1.5 GB" },
{ value: "large", label: "Large (v3)", size: "2.9 GB" },
{ value: "custom", label: "Custom", size: "Local File" },
];
function loadPreviewImage(url: string) {
@@ -530,6 +532,7 @@ export function SettingsPanel({
onSeek,
autoCaptions = [],
onAutoCaptionsChange,
autoCaptionProgress = 0,
autoCaptionSettings = DEFAULT_AUTO_CAPTION_SETTINGS,
whisperModelPath,
whisperModelDownloadStatus = "idle",
@@ -1363,16 +1366,6 @@ export function SettingsPanel({
</div>
<div className="rounded-lg bg-white/[0.03] px-2.5 py-2 space-y-3">
<div>
<Button
type="button"
variant="outline"
onClick={onPickWhisperModel}
className="h-10 w-full rounded-xl border-white/10 bg-white/5 px-4 text-sm text-slate-200 hover:bg-white/10 hover:text-white"
>
{tSettings("captions.selectModel", "Select Model")}
</Button>
</div>
<div className="flex items-center justify-between gap-3">
<div className="text-sm font-medium text-slate-200">
{tSettings("captions.language", "Language")}
@@ -1394,9 +1387,7 @@ export function SettingsPanel({
</Select>
</div>
<div className="flex items-center justify-between gap-3">
<div className="text-sm font-medium text-slate-200">
Model
</div>
<div className="text-sm font-medium text-slate-200">Model</div>
<Select
value={autoCaptionSettings.selectedModel || "small"}
onValueChange={(value) => updateAutoCaptionSettings({ selectedModel: value as any })}
@@ -1416,6 +1407,23 @@ export function SettingsPanel({
</SelectContent>
</Select>
</div>
{autoCaptionSettings.selectedModel === "custom" && (
<div>
<Button
type="button"
variant="outline"
onClick={onPickWhisperModel}
className="h-10 w-full rounded-xl border-white/10 bg-white/5 px-4 text-sm text-slate-200 hover:bg-white/10 hover:text-white"
>
{tSettings("captions.selectModel", "Select Model")}
</Button>
{whisperModelPath && (
<p className="mt-1 truncate px-1 text-[10px] text-slate-500">
{whisperModelPath.split(/[\\/]/).pop()}
</p>
)}
</div>
)}
<div className="flex flex-wrap items-center gap-2">
<div className="grid w-full grid-cols-2 gap-2">
{whisperModelDownloadStatus === "downloading" ? (
@@ -1436,7 +1444,7 @@ export function SettingsPanel({
>
{tSettings("captions.deleteModel", "Delete Model")}
</Button>
) : (
) : autoCaptionSettings.selectedModel !== "custom" ? (
<Button
type="button"
onClick={onDownloadWhisperModel}
@@ -1444,6 +1452,10 @@ export function SettingsPanel({
>
{tSettings("captions.downloadModel", "Download Model")}
</Button>
) : (
<div className="flex h-10 w-full items-center justify-center rounded-xl bg-white/5 px-4 text-[10px] text-slate-500 italic">
No local model selected
</div>
)}
<Button
type="button"
@@ -1460,14 +1472,22 @@ export function SettingsPanel({
<Button
type="button"
onClick={onGenerateAutoCaptions}
disabled={isGeneratingCaptions || !whisperModelPath}
className="h-10 w-full rounded-xl bg-[#2563EB] px-4 text-sm font-medium text-white hover:bg-[#2563EB]/90 disabled:opacity-60"
disabled={isGeneratingCaptions}
className="relative h-10 w-full overflow-hidden rounded-xl bg-[#2563EB] px-4 text-sm font-medium text-white hover:bg-[#2563EB]/90 disabled:bg-[#2563EB]/50"
>
{isGeneratingCaptions
? tSettings("captions.generating", "Generating...")
: captionCueCount > 0
? tSettings("captions.regenerateFull", "Regenerate Captions")
: tSettings("captions.generateFull", "Generate Captions")}
{isGeneratingCaptions && (
<motion.div
className="absolute inset-y-0 left-0 bg-white/20"
initial={{ width: 0 }}
animate={{ width: `${autoCaptionProgress}%` }}
transition={{ duration: 0.3 }}
/>
)}
<span className="relative z-10">
{isGeneratingCaptions
? `${tSettings("captions.generating", "Generating...")} (${autoCaptionProgress}%)`
: tSettings("captions.generateAutoCaptions", "Generate Captions")}
</span>
</Button>
{autoCaptions.length > 0 && (
src/components/video-editor/VideoEditor.tsx
+20
View File
@@ -392,6 +392,7 @@ export default function VideoEditor() {
>(initialEditorPreferences.whisperModelPath ? "downloaded" : "idle");
const [whisperModelDownloadProgress, setWhisperModelDownloadProgress] = useState(0);
const [isGeneratingCaptions, setIsGeneratingCaptions] = useState(false);
const [autoCaptionProgress, setAutoCaptionProgress] = useState(0);
const [isExporting, setIsExporting] = useState(false);
const [exportProgress, setExportProgress] = useState<ExportProgress | null>(null);
const [exportError, setExportError] = useState<string | null>(null);
@@ -1431,6 +1432,13 @@ export default function VideoEditor() {
return () => unsubscribe?.();
}, [autoCaptionSettings.selectedModel]);
useEffect(() => {
const unlistenProgress = window.electronAPI.onAutoCaptionProgress((payload: { progress: number }) => {
setAutoCaptionProgress(payload.progress);
});
return unlistenProgress;
}, []);
useEffect(() => {
void (async () => {
const result = await window.electronAPI.getWhisperModelStatus(
@@ -1552,7 +1560,15 @@ export default function VideoEditor() {
return;
}
console.log("[VideoEditor] handleGenerateAutoCaptions: starting", {
sourcePath,
whisperExecutablePath,
whisperModelPath,
language: autoCaptionSettings.language,
});
setIsGeneratingCaptions(true);
setAutoCaptionProgress(0);
try {
const result = await window.electronAPI.generateAutoCaptions({
videoPath: sourcePath,
@@ -1561,6 +1577,8 @@ export default function VideoEditor() {
language: autoCaptionSettings.language,
});
console.log("[VideoEditor] handleGenerateAutoCaptions: result", result);
if (!result.success || !result.cues) {
toast.error(
result.message || getErrorMessage(result.error) || "Failed to generate captions",
@@ -1575,6 +1593,7 @@ export default function VideoEditor() {
toast.error(getErrorMessage(error));
} finally {
setIsGeneratingCaptions(false);
setAutoCaptionProgress(0);
}
}, [
autoCaptionSettings.language,
@@ -3479,6 +3498,7 @@ export default function VideoEditor() {
whisperModelDownloadStatus={whisperModelDownloadStatus}
whisperModelDownloadProgress={whisperModelDownloadProgress}
isGeneratingCaptions={isGeneratingCaptions}
autoCaptionProgress={autoCaptionProgress}
onAutoCaptionSettingsChange={setAutoCaptionSettings}
onPickWhisperExecutable={handlePickWhisperExecutable}
onPickWhisperModel={handlePickWhisperModel}
src/components/video-editor/types.ts
+1 -1
View File
@@ -267,7 +267,7 @@ export interface CaptionCueWord {
}
export type AutoCaptionAnimation = "none" | "fade" | "rise" | "pop";
export type WhisperModel = "tiny" | "base" | "small" | "medium" | "large";
export type WhisperModel = "tiny" | "base" | "small" | "medium" | "large" | "custom";
export interface AutoCaptionSettings {
enabled: boolean;