diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index d9910a8f..049c59a7 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,6 +1,6 @@ # These are supported funding model platforms -github: webadderall +github: # patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: webadderall diff --git a/.gitignore b/.gitignore index d3a2eb14..a817d16a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,42 +1,47 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* -lerna-debug.log* - -node_modules -dist -dist-electron -dist-ssr -*.local - -# Editor directories and files -.vscode/* -!.vscode/extensions.json -.idea -.DS_Store -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw? -release/** -*.kiro/ -# npx electron-builder --mac --win -.tmp/ -.history/ -*.tsbuildinfo -vite.config.js -vite.config.d.ts - -# Native capture build artifacts -electron/native/wgc-capture/build/ -electron/native/cursor-monitor/build/ - -# Local build tools and caches -.cache/ -.cmake_ext/ -ebcache/ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-electron +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? +release/** +*.kiro/ +# npx electron-builder --mac --win +.tmp/ +.history/ +*.tsbuildinfo +vite.config.js +vite.config.d.ts + +# Native capture build artifacts +electron/native/wgc-capture/build/ +electron/native/cursor-monitor/build/ + +# Local build tools and caches +.cache/ +.cmake_ext/ +ebcache/ +docs/ + +# Local debug helpers +tmp-*.ps1 +.tmp-*.ps1 diff --git a/electron-builder.json5 b/electron-builder.json5 index 138f2183..677d0a38 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -65,7 +65,6 @@ ], "icon": "icons/icons/win/icon.ico", "executableName": "Recordly", - "artifactName": "${productName}-Setup-${version}.${ext}" + "artifactName": "Recordly.${ext}" } -} - +} diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index f159ccad..f871ddfd 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -85,11 +85,15 @@ function getScreen() { return nodeRequire('electron').screen as typeof import('electron').screen } +function normalizeRecordingTimeOffsetMs(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) + ? Math.round(value) + : 0 +} + function broadcastSelectedSourceChange() { for (const window of BrowserWindow.getAllWindows()) { - if (!window.isDestroyed()) { - window.webContents.send('selected-source-changed', selectedSource) - } + safeSend(window.webContents, 'selected-source-changed', selectedSource) } } @@ -425,7 +429,7 @@ async function loadProjectFromPath(projectPath: string) { currentRecordingSession = { videoPath: mediaSources.videoPath, webcamPath: mediaSources.webcamPath, - timeOffsetMs: mediaSources.timeOffsetMs, + timeOffsetMs: 0, } await rememberRecentProject(normalizedPath) @@ -462,7 +466,6 @@ async function resolveProjectMediaSources(project: unknown): Promise< success: true videoPath: string webcamPath: string | null - timeOffsetMs: number } | { success: false @@ -496,9 +499,6 @@ async function resolveProjectMediaSources(project: unknown): Promise< typeof (project as { editor?: { webcam?: { sourcePath?: unknown } } }).editor?.webcam?.sourcePath === 'string' ? ((project as { editor?: { webcam?: { sourcePath?: string } } }).editor?.webcam?.sourcePath ?? null) : null - const timeOffsetMs = normalizeRecordingTimeOffsetMs( - (project as { editor?: { webcam?: { timeOffsetMs?: unknown } } }).editor?.webcam?.timeOffsetMs, - ) const normalizedWebcamPath = normalizeVideoSourcePath(rawWebcamPath) if (!normalizedWebcamPath) { @@ -506,7 +506,6 @@ async function resolveProjectMediaSources(project: unknown): Promise< success: true, videoPath: normalizedVideoPath, webcamPath: null, - timeOffsetMs, } } @@ -516,24 +515,16 @@ async function resolveProjectMediaSources(project: unknown): Promise< success: true, videoPath: normalizedVideoPath, webcamPath: normalizedWebcamPath, - timeOffsetMs, } } catch { return { success: true, videoPath: normalizedVideoPath, webcamPath: null, - timeOffsetMs, } } } -function normalizeRecordingTimeOffsetMs(value: unknown): number { - return typeof value === 'number' && Number.isFinite(value) - ? Math.round(value) - : 0 -} - function getRecordingSessionManifestPath(videoPath: string) { const extension = path.extname(videoPath) const baseName = path.basename(videoPath, extension) @@ -652,7 +643,6 @@ async function resolveRecordingSession(videoPath?: string | null): Promise { + // On Windows, the 'finish' event might fire before the OS has fully released the file handle. + // We listen for 'close' to be absolutely sure the file descriptor is closed. + fileStream.on('close', () => { onProgress(100) resolve() }) @@ -1151,35 +1151,50 @@ async function downloadWhisperModel( path: null, }); - try { - await fs.rm(tempPath, { force: true }); - await downloadFileWithProgress(model.url, tempPath, (progress) => { - sendWhisperModelDownloadProgress(webContents, { - status: "downloading", - progress, - model: modelName, - path: null, - }); - }); - await fs.rename(tempPath, modelPath); - sendWhisperModelDownloadProgress(webContents, { - status: "downloaded", - progress: 100, - model: modelName, - path: modelPath, - }); - return modelPath; - } catch (error) { - await fs.rm(tempPath, { force: true }).catch(() => undefined); - sendWhisperModelDownloadProgress(webContents, { - status: "error", - progress: 0, - model: modelName, - path: null, - error: String(error), - }); - throw error; - } + try { + await fs.rm(tempPath, { force: true }).catch(() => undefined) + await downloadFileWithProgress(model.url, tempPath, (progress) => { + sendWhisperModelDownloadProgress(webContents, { + status: 'downloading', + progress, + model: modelName, + path: null, + }) + }) + + // Robust rename logic for Windows to avoid EPERM/EBUSY + let renameRetries = 0 + const maxRetries = 5 + while (renameRetries < maxRetries) { + try { + await fs.rename(tempPath, modelPath) + break + } catch (err) { + renameRetries++ + if (renameRetries >= maxRetries) throw err + // Wait briefly between retries to allow OS to release file handles + await new Promise((resolve) => setTimeout(resolve, 100 * renameRetries)) + } + } + + sendWhisperModelDownloadProgress(webContents, { + status: 'downloaded', + progress: 100, + model: modelName, + path: modelPath, + }) + return modelPath + } catch (error) { + await fs.rm(tempPath, { force: true }).catch(() => undefined) + sendWhisperModelDownloadProgress(webContents, { + status: 'error', + progress: 0, + model: modelName, + path: null, + error: String(error), + }) + throw error + } } async function deleteWhisperModel(_event: any, modelName: string) { @@ -1488,7 +1503,7 @@ async function extractCaptionAudioSource(options: { for (const candidate of candidates) { try { await ensureReadableFile(candidate.path, 'video file') - console.log('[auto-captions] Extracting audio from:', candidate.path, options.startTime ? `at ${options.startTime}s` : '') + console.log('[auto-captions] Extracting audio from:', path.basename(candidate.path), options.startTime ? `at ${options.startTime}s` : '') const ffmpegArgs = ['-y']; if (options.startTime !== undefined) { @@ -1504,7 +1519,7 @@ async function extractCaptionAudioSource(options: { ffmpegArgs, { timeout: 5 * 60 * 1000, maxBuffer: 20 * 1024 * 1024 }, ) - console.log('[auto-captions] Audio extracted successfully to:', options.wavPath) + console.log('[auto-captions] Audio extracted successfully to temporary workspace') attemptedCandidates.push({ ...candidate, readable: true, extractedAudio: true }) return candidate } catch (error) { @@ -1518,7 +1533,7 @@ async function extractCaptionAudioSource(options: { } } - console.warn('[auto-captions] No audio source candidate could be extracted:', attemptedCandidates) + console.warn('[auto-captions] No audio source candidate could be extracted') throw new Error('No audio was found to transcribe in the saved recording file. Captions need an audio track. If this recording should have contained sound, the recording was saved without an audio stream.') } @@ -1554,15 +1569,10 @@ async function generateAutoCaptionsFromVideo( const endTimeMs = totalDurationMs > 0 ? startTimeMs + totalDurationMs : Infinity; console.log('[auto-captions] Starting segmented caption generation sequence') - console.log('[auto-captions] Video:', normalizedVideoPath) + console.log('[auto-captions] Video:', path.basename(normalizedVideoPath)) console.log('[auto-captions] Range:', `${(startTimeMs/1000).toFixed(2)}s - ${totalDurationMs ? `${((startTimeMs + totalDurationMs)/1000).toFixed(2)}s` : 'End'}`) const allCues: any[] = []; - let chunkCount = 1; - if (totalDurationMs > 0) { - chunkCount = Math.ceil(totalDurationMs / CHUNK_SIZE_MS); - } - let audioSourceLabel = 'Unknown'; for (let offsetMs = startTimeMs; offsetMs < endTimeMs; offsetMs += CHUNK_SIZE_MS) { @@ -1574,7 +1584,6 @@ async function generateAutoCaptionsFromVideo( const jsonPath = `${outputBase}.json` try { - console.log(`[auto-captions] Processing chunk ${chunkIndex + 1}/${chunkCount || '?'} at offset ${offsetMs / 1000}s`) const audioSource = await extractCaptionAudioSource({ videoPath: normalizedVideoPath, @@ -1599,9 +1608,9 @@ async function generateAutoCaptionsFromVideo( const updateChunkProgress = (progress: number) => { if (totalDurationMs > 0) { const totalProgress = (offsetMs / totalDurationMs * 100) + (progress / (totalDurationMs / CHUNK_SIZE_MS)); - webContents.send('auto-caption-progress', { progress: Math.min(99, totalProgress) }) + safeSend(webContents, 'auto-caption-progress', { progress: Math.min(99, totalProgress) }) } else { - webContents.send('auto-caption-progress', { progress }) + safeSend(webContents, 'auto-caption-progress', { progress }) } }; @@ -1639,9 +1648,8 @@ async function generateAutoCaptionsFromVideo( }); if (adjustedCues.length > 0) { - console.log(`[auto-captions] Chunk ${chunkIndex + 1} produced ${adjustedCues.length} adjusted cues.`) allCues.push(...adjustedCues); - webContents.send('auto-caption-chunk', { cues: adjustedCues }); + safeSend(webContents, 'auto-caption-chunk', { cues: adjustedCues }); } // If we don't know duration and this was a short chunk, we might be at the end @@ -1664,8 +1672,7 @@ async function generateAutoCaptionsFromVideo( } } - console.log(`[auto-captions] Generation complete. Total cues: ${allCues.length}`) - webContents.send('auto-caption-progress', { progress: 100 }) + safeSend(webContents, 'auto-caption-progress', { progress: 100 }) return { cues: allCues, audioSourceLabel, @@ -1920,6 +1927,10 @@ function waitForWindowsCaptureStop(proc: ChildProcessWithoutNullStreams) { resolve(match[1].trim()) return } + if (code === 0 && windowsCaptureTargetPath) { + resolve(windowsCaptureTargetPath) + return + } reject(new Error(windowsCaptureOutputBuffer.trim() || `Native Windows capture exited with code ${code ?? 'unknown'}`)) } @@ -1954,7 +1965,7 @@ function attachWindowsCaptureLifecycle(proc: ChildProcessWithoutNullStreams) { const sourceName = selectedSource?.name ?? 'Screen' BrowserWindow.getAllWindows().forEach((window) => { if (!window.isDestroyed()) { - window.webContents.send('recording-state-changed', { + safeSend(window.webContents, 'recording-state-changed', { recording: false, sourceName, }) @@ -2041,43 +2052,6 @@ async function muxNativeWindowsVideoWithAudio(videoPath: string, systemAudioPath } } -async function probeRecordedMediaStream(videoPath: string, streamType: 'video' | 'audio') { - const ffmpegPath = getFfmpegBinaryPath() - const args = streamType === 'video' - ? ['-v', 'error', '-i', videoPath, '-map', '0:v:0', '-frames:v', '1', '-f', 'null', '-'] - : ['-v', 'error', '-i', videoPath, '-map', '0:a:0', '-frames:a', '1', '-f', 'null', '-'] - - await execFileAsync(ffmpegPath, args, { - timeout: 30000, - maxBuffer: 4 * 1024 * 1024, - }) -} - -async function validateRecordedVideoFile(videoPath: string, options?: { requiresAudio?: boolean }) { - await fs.access(videoPath, fsConstants.R_OK) - - const stats = await fs.stat(videoPath) - if (stats.size <= 0) { - throw new Error('Recorded video file is empty') - } - - try { - await probeRecordedMediaStream(videoPath, 'video') - } catch (error) { - throw new Error(`Recorded video file is unreadable or missing a video stream: ${String(error)}`) - } - - if (!options?.requiresAudio) { - return - } - - try { - await probeRecordedMediaStream(videoPath, 'audio') - } catch (error) { - throw new Error(`Recorded video is missing the requested audio track: ${String(error)}`) - } -} - function waitForNativeCaptureStart(process: ChildProcessWithoutNullStreams) { return new Promise((resolve, reject) => { const timer = setTimeout(() => { @@ -2228,7 +2202,7 @@ async function muxNativeMacRecordingWithAudio( function emitRecordingInterrupted(reason: string, message: string) { BrowserWindow.getAllWindows().forEach((window) => { if (!window.isDestroyed()) { - window.webContents.send('recording-interrupted', { reason, message }) + safeSend(window.webContents, 'recording-interrupted', { reason, message }) } }) } @@ -2236,7 +2210,7 @@ function emitRecordingInterrupted(reason: string, message: string) { function emitCursorStateChanged(cursorType: CursorVisualType) { BrowserWindow.getAllWindows().forEach((window) => { if (!window.isDestroyed()) { - window.webContents.send('cursor-state-changed', { cursorType }) + safeSend(window.webContents, 'cursor-state-changed', { cursorType }) } }) } @@ -2272,7 +2246,7 @@ function attachNativeCaptureLifecycle(process: ChildProcessWithoutNullStreams) { const sourceName = selectedSource?.name ?? 'Screen' BrowserWindow.getAllWindows().forEach((window) => { if (!window.isDestroyed()) { - window.webContents.send('recording-state-changed', { + safeSend(window.webContents, 'recording-state-changed', { recording: false, sourceName, }) @@ -2676,7 +2650,6 @@ function snapshotCursorTelemetryForPersistence() { } async function finalizeStoredVideo(videoPath: string) { - await validateRecordedVideoFile(videoPath) snapshotCursorTelemetryForPersistence() currentVideoPath = videoPath currentProjectPath = null @@ -3239,9 +3212,6 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh} const micPath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`) config.captureMic = true config.micOutputPath = micPath - if (options.microphoneDeviceId) { - config.micDeviceId = options.microphoneDeviceId - } if (options.microphoneLabel) { config.micDeviceName = options.microphoneLabel } @@ -3454,7 +3424,6 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh} await moveFileWithOverwrite(tempVideoPath, finalVideoPath) } - await validateRecordedVideoFile(finalVideoPath) windowsPendingVideoPath = finalVideoPath return { success: true, path: finalVideoPath } } catch (error) { @@ -3521,13 +3490,14 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh} await moveFileWithOverwrite(tempVideoPath, finalVideoPath) } - const requiresAudio = Boolean(preferredSystemAudioPath || preferredMicrophonePath) - - if (requiresAudio) { - await muxNativeMacRecordingWithAudio(finalVideoPath, preferredSystemAudioPath, preferredMicrophonePath) + if (preferredSystemAudioPath || preferredMicrophonePath) { + try { + await muxNativeMacRecordingWithAudio(finalVideoPath, preferredSystemAudioPath, preferredMicrophonePath) + } catch (error) { + console.warn('Failed to mux native macOS audio into capture:', error) + } } - await validateRecordedVideoFile(finalVideoPath, { requiresAudio }) return await finalizeStoredVideo(finalVideoPath) } catch (error) { console.error('Failed to stop native ScreenCaptureKit recording:', error) @@ -3661,21 +3631,22 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh} } try { - const requiresAudio = Boolean(windowsSystemAudioPath || windowsMicAudioPath) - - if (requiresAudio) { + if (windowsSystemAudioPath || windowsMicAudioPath) { await muxNativeWindowsVideoWithAudio(videoPath, windowsSystemAudioPath, windowsMicAudioPath) windowsSystemAudioPath = null windowsMicAudioPath = null } - await validateRecordedVideoFile(videoPath, { requiresAudio }) return await finalizeStoredVideo(videoPath) } catch (error) { console.error('Failed to mux native Windows recording:', error) windowsSystemAudioPath = null windowsMicAudioPath = null - return { success: false, message: 'Failed to mux native Windows recording', error: String(error) } + try { + return await finalizeStoredVideo(videoPath) + } catch { + return { success: false, message: 'Failed to mux native Windows recording', error: String(error) } + } } }) @@ -3823,7 +3794,7 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh} const source = selectedSource || { name: 'Screen' } BrowserWindow.getAllWindows().forEach((window) => { if (!window.isDestroyed()) { - window.webContents.send('recording-state-changed', { + safeSend(window.webContents, 'recording-state-changed', { recording, sourceName: source.name, }) @@ -4489,6 +4460,7 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh} ?? { videoPath: currentVideoPath, webcamPath: null, + timeOffsetMs: 0, } currentRecordingSession = resolvedSession @@ -4554,6 +4526,10 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh} } }); + ipcMain.handle('app:getVersion', () => { + return app.getVersion() + }) + ipcMain.handle('get-platform', () => { return process.platform; }); @@ -4635,7 +4611,7 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh} let remaining = seconds countdownRemaining = remaining - countdownWin.webContents.send('countdown-tick', remaining) + safeSend(countdownWin.webContents, 'countdown-tick', remaining) countdownTimer = setInterval(() => { if (countdownCancelled) { @@ -4664,9 +4640,11 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh} resolve({ success: true }) } else { const win = getCountdownWindow() - if (win && !win.isDestroyed()) { - win.webContents.send('countdown-tick', remaining) - } + try { + if (win && !win.isDestroyed()) { + safeSend(win.webContents, 'countdown-tick', remaining) + } + } catch {} } }, 1000) }) @@ -4690,9 +4668,5 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh} seconds: countdownInProgress ? countdownRemaining : null, } }) - - ipcMain.handle('app:getVersion', () => { - return app.getVersion() - }) } diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index 6cd60315..8d69d1c5 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -29,7 +29,6 @@ struct CaptureConfig { std::string outputPath; std::string audioOutputPath; std::string micOutputPath; - std::string micDeviceId; std::string micDeviceName; int fps = 60; int width = 0; @@ -119,7 +118,6 @@ static bool parseSimpleJson(const std::string& json, CaptureConfig& config) { config.audioOutputPath = findString("audioOutputPath"); config.micOutputPath = findString("micOutputPath"); - config.micDeviceId = findString("micDeviceId"); config.micDeviceName = findString("micDeviceName"); auto findBool = [&](const std::string& key) -> bool { @@ -251,7 +249,6 @@ int main(int argc, char* argv[]) { // Set up frame callback std::atomic frameCount{0}; - std::atomic frameWriteFailed{false}; session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) { g_lastFrameTimestampHns = timestampHns; if (g_stopRequested) return; @@ -273,12 +270,7 @@ int main(int argc, char* argv[]) { if (encoder.writeFrame(texture, adjustedTimestampHns)) { frameCount++; - return; } - - frameWriteFailed = true; - g_stopRequested = true; - g_stopCv.notify_all(); }); // Start stdin listener @@ -301,10 +293,7 @@ int main(int argc, char* argv[]) { } if (config.captureMic && !config.micOutputPath.empty()) { - micInitialized = micCapture.initializeMic( - config.micOutputPath, - config.micDeviceId, - config.micDeviceName); + micInitialized = micCapture.initializeMic(config.micOutputPath, config.micDeviceName); if (!micInitialized) { std::cerr << "WARNING: Failed to initialize WASAPI mic capture" << std::endl; } @@ -344,20 +333,7 @@ int main(int argc, char* argv[]) { session.stopCapture(); if (audioActive) loopback.stop(); if (micActive) micCapture.stop(); - if (frameWriteFailed.load()) { - std::cerr << "ERROR: Failed to encode one or more video frames" << std::endl; - return 1; - } - - if (frameCount.load() <= 0) { - std::cerr << "ERROR: No video frames were written" << std::endl; - return 1; - } - - if (!encoder.finalize()) { - std::cerr << "ERROR: Failed to finalize Media Foundation encoder" << std::endl; - return 1; - } + encoder.finalize(); std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl; if (audioActive) { diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp index 1adef1b6..a1474c20 100644 --- a/electron/native/wgc-capture/src/mf_encoder.cpp +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -124,7 +124,6 @@ bool MFEncoder::initialize(const std::wstring& outputPath, int width, int height } bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns) { - std::lock_guard lock(writeMutex_); if (!initialized_ || !sinkWriter_) return false; context_->CopyResource(stagingTexture_.Get(), texture); @@ -191,7 +190,6 @@ bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns) { } bool MFEncoder::finalize() { - std::lock_guard lock(writeMutex_); if (!initialized_) return false; initialized_ = false; diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h index 11b9c774..7eb8e642 100644 --- a/electron/native/wgc-capture/src/mf_encoder.h +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -8,7 +8,6 @@ #include #include #include -#include using Microsoft::WRL::ComPtr; @@ -33,5 +32,4 @@ private: int height_ = 0; int fps_ = 60; bool initialized_ = false; - std::mutex writeMutex_; }; diff --git a/electron/native/wgc-capture/src/wasapi_loopback.cpp b/electron/native/wgc-capture/src/wasapi_loopback.cpp index b7b1cc06..8ae15ad1 100644 --- a/electron/native/wgc-capture/src/wasapi_loopback.cpp +++ b/electron/native/wgc-capture/src/wasapi_loopback.cpp @@ -29,36 +29,6 @@ static std::wstring utf8ToWide(const std::string& str) { return wstr; } -IMMDevice* WasapiCapture::findCaptureDeviceById(const std::wstring& targetId) { - IMMDeviceCollection* collection = nullptr; - HRESULT hr = enumerator_->EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE, &collection); - if (FAILED(hr)) return nullptr; - - UINT count = 0; - collection->GetCount(&count); - - for (UINT i = 0; i < count; i++) { - IMMDevice* dev = nullptr; - collection->Item(i, &dev); - - LPWSTR deviceId = nullptr; - hr = dev->GetId(&deviceId); - if (SUCCEEDED(hr) && deviceId) { - const bool matches = targetId == deviceId; - CoTaskMemFree(deviceId); - if (matches) { - collection->Release(); - return dev; - } - } - - dev->Release(); - } - - collection->Release(); - return nullptr; -} - IMMDevice* WasapiCapture::findCaptureDeviceByName(const std::wstring& targetName) { IMMDeviceCollection* collection = nullptr; HRESULT hr = enumerator_->EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE, &collection); @@ -67,8 +37,6 @@ IMMDevice* WasapiCapture::findCaptureDeviceByName(const std::wstring& targetName UINT count = 0; collection->GetCount(&count); - IMMDevice* partialMatch = nullptr; - for (UINT i = 0; i < count; i++) { IMMDevice* dev = nullptr; collection->Item(i, &dev); @@ -82,21 +50,15 @@ IMMDevice* WasapiCapture::findCaptureDeviceByName(const std::wstring& targetName PropVariantClear(&pv); store->Release(); - if (name == targetName) { + if (name.find(targetName) != std::wstring::npos || targetName.find(name) != std::wstring::npos) { collection->Release(); return dev; } - - if (!partialMatch && (name.find(targetName) != std::wstring::npos || targetName.find(name) != std::wstring::npos)) { - partialMatch = dev; - continue; - } - dev->Release(); } collection->Release(); - return partialMatch; + return nullptr; } bool WasapiCapture::initializeLoopback(const std::string& outputPath) { @@ -114,10 +76,7 @@ bool WasapiCapture::initializeLoopback(const std::string& outputPath) { return initializeCommon(); } -bool WasapiCapture::initializeMic( - const std::string& outputPath, - const std::string& deviceId, - const std::string& deviceName) { +bool WasapiCapture::initializeMic(const std::string& outputPath, const std::string& deviceName) { outputPath_ = outputPath; streamFlags_ = 0; @@ -126,10 +85,7 @@ bool WasapiCapture::initializeMic( IID_IMMDeviceEnumerator_, reinterpret_cast(&enumerator_)); if (FAILED(hr)) return false; - if (!deviceId.empty()) { - device_ = findCaptureDeviceById(utf8ToWide(deviceId)); - } - if (!device_ && !deviceName.empty()) { + if (!deviceName.empty()) { device_ = findCaptureDeviceByName(utf8ToWide(deviceName)); } if (!device_) { diff --git a/electron/native/wgc-capture/src/wasapi_loopback.h b/electron/native/wgc-capture/src/wasapi_loopback.h index e9b9c21a..a4facf13 100644 --- a/electron/native/wgc-capture/src/wasapi_loopback.h +++ b/electron/native/wgc-capture/src/wasapi_loopback.h @@ -14,10 +14,7 @@ public: ~WasapiCapture(); bool initializeLoopback(const std::string& outputPath); - bool initializeMic( - const std::string& outputPath, - const std::string& deviceId = "", - const std::string& deviceName = ""); + bool initializeMic(const std::string& outputPath, const std::string& deviceName = ""); bool start(); bool pause(); bool resume(); @@ -27,7 +24,6 @@ private: bool initializeCommon(); void captureThread(); bool writeWavHeader(HANDLE file, DWORD dataSize); - IMMDevice* findCaptureDeviceById(const std::wstring& id); IMMDevice* findCaptureDeviceByName(const std::wstring& name); std::string outputPath_; diff --git a/electron/native/windows-capture/CMakeLists.txt b/electron/native/windows-capture/CMakeLists.txt new file mode 100644 index 00000000..773452c4 --- /dev/null +++ b/electron/native/windows-capture/CMakeLists.txt @@ -0,0 +1,27 @@ +cmake_minimum_required(VERSION 3.20) +project(windows-capture LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +add_executable(windows-capture + src/main.cpp + src/dxgi_session.cpp + src/mf_encoder.cpp + src/monitor_utils.cpp + src/wasapi_loopback.cpp +) + +target_compile_options(windows-capture PRIVATE /EHsc /W3 /utf-8) + +target_link_libraries(windows-capture PRIVATE + d3d11 + dxgi + dwmapi + mfplat + mfreadwrite + mf + mfuuid + ole32 + shcore +) diff --git a/electron/native/windows-capture/src/dxgi_session.cpp b/electron/native/windows-capture/src/dxgi_session.cpp new file mode 100644 index 00000000..76c91402 --- /dev/null +++ b/electron/native/windows-capture/src/dxgi_session.cpp @@ -0,0 +1,343 @@ +#include "dxgi_session.h" + +#include + +#include +#include +#include + +namespace { + +bool intersectRectChecked(const RECT& lhs, const RECT& rhs, RECT& result) { + return IntersectRect(&result, &lhs, &rhs) != FALSE; +} + +int evenFloor(int value) { + return value > 1 ? (value & ~1) : value; +} + +RECT getExtendedWindowBounds(HWND hwnd) { + RECT bounds{}; + if (SUCCEEDED(DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, &bounds, sizeof(bounds)))) { + return bounds; + } + + GetWindowRect(hwnd, &bounds); + return bounds; +} + +} // namespace + +DxgiSession::DxgiSession() {} + +DxgiSession::~DxgiSession() { + stopCapture(); +} + +bool DxgiSession::findOutputForMonitor(HMONITOR monitor, ComPtr& adapter, ComPtr& output) { + ComPtr factory; + HRESULT hr = CreateDXGIFactory1(IID_PPV_ARGS(&factory)); + if (FAILED(hr)) { + std::cerr << "ERROR: CreateDXGIFactory1 failed: 0x" << std::hex << hr << std::endl; + return false; + } + + for (UINT adapterIndex = 0;; ++adapterIndex) { + ComPtr candidateAdapter; + hr = factory->EnumAdapters1(adapterIndex, &candidateAdapter); + if (hr == DXGI_ERROR_NOT_FOUND) { + break; + } + if (FAILED(hr)) { + continue; + } + + for (UINT outputIndex = 0;; ++outputIndex) { + ComPtr candidateOutput; + hr = candidateAdapter->EnumOutputs(outputIndex, &candidateOutput); + if (hr == DXGI_ERROR_NOT_FOUND) { + break; + } + if (FAILED(hr)) { + continue; + } + + DXGI_OUTPUT_DESC desc{}; + if (FAILED(candidateOutput->GetDesc(&desc))) { + continue; + } + + if (!desc.AttachedToDesktop || desc.Monitor != monitor) { + continue; + } + + ComPtr output1; + hr = candidateOutput.As(&output1); + if (FAILED(hr)) { + continue; + } + + adapter = candidateAdapter; + output = output1; + outputDesc_ = desc; + return true; + } + } + + return false; +} + +bool DxgiSession::createD3DDevice(IDXGIAdapter1* adapter) { + UINT creationFlags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; + D3D_FEATURE_LEVEL featureLevels[] = { + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + + HRESULT hr = D3D11CreateDevice( + adapter, + D3D_DRIVER_TYPE_UNKNOWN, + nullptr, + creationFlags, + featureLevels, + ARRAYSIZE(featureLevels), + D3D11_SDK_VERSION, + &d3dDevice_, + nullptr, + &d3dContext_); + + if (FAILED(hr)) { + std::cerr << "ERROR: D3D11CreateDevice failed: 0x" << std::hex << hr << std::endl; + return false; + } + + return true; +} + +bool DxgiSession::createDuplication(IDXGIOutput1* output) { + HRESULT hr = output->DuplicateOutput(d3dDevice_.Get(), &duplication_); + if (FAILED(hr)) { + std::cerr << "ERROR: DuplicateOutput failed: 0x" << std::hex << hr << std::endl; + return false; + } + + return true; +} + +bool DxgiSession::createCaptureTexture() { + if (captureWidth_ <= 0 || captureHeight_ <= 0) { + return false; + } + + D3D11_TEXTURE2D_DESC desc{}; + desc.Width = static_cast(captureWidth_); + desc.Height = static_cast(captureHeight_); + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + + HRESULT hr = d3dDevice_->CreateTexture2D(&desc, nullptr, &captureTexture_); + if (FAILED(hr)) { + std::cerr << "ERROR: CreateTexture2D failed: 0x" << std::hex << hr << std::endl; + return false; + } + + return true; +} + +bool DxgiSession::initializeFullMonitorRect() { + const int width = outputDesc_.DesktopCoordinates.right - outputDesc_.DesktopCoordinates.left; + const int height = outputDesc_.DesktopCoordinates.bottom - outputDesc_.DesktopCoordinates.top; + + captureWidth_ = evenFloor(width); + captureHeight_ = evenFloor(height); + if (captureWidth_ <= 0 || captureHeight_ <= 0) { + return false; + } + + captureRect_.left = 0; + captureRect_.top = 0; + captureRect_.right = captureWidth_; + captureRect_.bottom = captureHeight_; + return true; +} + +bool DxgiSession::updateWindowCaptureRect() { + if (!captureWindow_ || !windowHandle_) { + return false; + } + + RECT windowRect = getExtendedWindowBounds(windowHandle_); + RECT monitorRect = outputDesc_.DesktopCoordinates; + RECT clippedRect{}; + if (!intersectRectChecked(windowRect, monitorRect, clippedRect)) { + return false; + } + + int left = clippedRect.left - monitorRect.left; + int top = clippedRect.top - monitorRect.top; + const int monitorWidth = monitorRect.right - monitorRect.left; + const int monitorHeight = monitorRect.bottom - monitorRect.top; + + left = std::clamp(left, 0, std::max(0, monitorWidth - captureWidth_)); + top = std::clamp(top, 0, std::max(0, monitorHeight - captureHeight_)); + + captureRect_.left = left; + captureRect_.top = top; + captureRect_.right = left + captureWidth_; + captureRect_.bottom = top + captureHeight_; + return true; +} + +bool DxgiSession::initializeWindowRect(HWND hwnd) { + windowHandle_ = hwnd; + captureWindow_ = true; + + RECT windowRect = getExtendedWindowBounds(hwnd); + RECT monitorRect = outputDesc_.DesktopCoordinates; + RECT clippedRect{}; + if (!intersectRectChecked(windowRect, monitorRect, clippedRect)) { + return false; + } + + captureWidth_ = evenFloor(clippedRect.right - clippedRect.left); + captureHeight_ = evenFloor(clippedRect.bottom - clippedRect.top); + if (captureWidth_ <= 0 || captureHeight_ <= 0) { + return false; + } + + return updateWindowCaptureRect(); +} + +bool DxgiSession::initializeForMonitor(HMONITOR monitor, int fps) { + fps_ = fps; + frameIntervalHns_ = 10000000LL / fps_; + + ComPtr adapter; + ComPtr output; + if (!findOutputForMonitor(monitor, adapter, output)) { + std::cerr << "ERROR: Failed to find DXGI output for monitor" << std::endl; + return false; + } + + if (!createD3DDevice(adapter.Get())) { + return false; + } + + if (!createDuplication(output.Get())) { + return false; + } + + return true; +} + +bool DxgiSession::initialize(HMONITOR monitor, int fps) { + captureWindow_ = false; + windowHandle_ = nullptr; + + if (!initializeForMonitor(monitor, fps)) { + return false; + } + + return initializeFullMonitorRect() && createCaptureTexture(); +} + +bool DxgiSession::initialize(HWND hwnd, int fps) { + HMONITOR monitor = MonitorFromWindow(hwnd, MONITOR_DEFAULTTONEAREST); + if (!monitor) { + std::cerr << "ERROR: MonitorFromWindow failed" << std::endl; + return false; + } + + if (!initializeForMonitor(monitor, fps)) { + return false; + } + + return initializeWindowRect(hwnd) && createCaptureTexture(); +} + +void DxgiSession::setFrameCallback(FrameCallback callback) { + frameCallback_ = std::move(callback); +} + +bool DxgiSession::startCapture() { + if (!duplication_ || !captureTexture_) { + return false; + } + + capturing_ = true; + lastFrameTimeHns_ = 0; + captureThread_ = std::thread(&DxgiSession::captureLoop, this); + return true; +} + +void DxgiSession::stopCapture() { + capturing_ = false; + + if (captureThread_.joinable()) { + captureThread_.join(); + } + + duplication_.Reset(); + captureTexture_.Reset(); +} + +int64_t DxgiSession::nowHns() const { + return std::chrono::duration_cast>>( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +void DxgiSession::captureLoop() { + while (capturing_) { + DXGI_OUTDUPL_FRAME_INFO frameInfo{}; + ComPtr desktopResource; + HRESULT hr = duplication_->AcquireNextFrame(100, &frameInfo, &desktopResource); + + if (hr == DXGI_ERROR_WAIT_TIMEOUT) { + continue; + } + + if (FAILED(hr)) { + if (hr != DXGI_ERROR_ACCESS_LOST) { + std::cerr << "ERROR: AcquireNextFrame failed: 0x" << std::hex << hr << std::endl; + } + break; + } + + ComPtr sourceTexture; + hr = desktopResource.As(&sourceTexture); + if (SUCCEEDED(hr) && sourceTexture) { + if (!captureWindow_ || updateWindowCaptureRect()) { + const int64_t timestampHns = nowHns(); + if (lastFrameTimeHns_ == 0 || (timestampHns - lastFrameTimeHns_) >= (frameIntervalHns_ * 7 / 10)) { + D3D11_BOX sourceBox{}; + sourceBox.left = static_cast(captureRect_.left); + sourceBox.top = static_cast(captureRect_.top); + sourceBox.front = 0; + sourceBox.right = static_cast(captureRect_.right); + sourceBox.bottom = static_cast(captureRect_.bottom); + sourceBox.back = 1; + + d3dContext_->CopySubresourceRegion( + captureTexture_.Get(), + 0, + 0, + 0, + 0, + sourceTexture.Get(), + 0, + &sourceBox); + + if (frameCallback_) { + lastFrameTimeHns_ = timestampHns; + frameCallback_(captureTexture_.Get(), timestampHns); + } + } + } + } + + duplication_->ReleaseFrame(); + } +} \ No newline at end of file diff --git a/electron/native/windows-capture/src/dxgi_session.h b/electron/native/windows-capture/src/dxgi_session.h new file mode 100644 index 00000000..37cd04b7 --- /dev/null +++ b/electron/native/windows-capture/src/dxgi_session.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +using Microsoft::WRL::ComPtr; + +class DxgiSession { +public: + using FrameCallback = std::function; + + DxgiSession(); + ~DxgiSession(); + + bool initialize(HMONITOR monitor, int fps); + bool initialize(HWND hwnd, int fps); + void setFrameCallback(FrameCallback callback); + bool startCapture(); + void stopCapture(); + + int captureWidth() const { return captureWidth_; } + int captureHeight() const { return captureHeight_; } + ID3D11Device* device() const { return d3dDevice_.Get(); } + ID3D11DeviceContext* context() const { return d3dContext_.Get(); } + +private: + ComPtr d3dDevice_; + ComPtr d3dContext_; + ComPtr duplication_; + ComPtr captureTexture_; + + FrameCallback frameCallback_; + std::atomic capturing_{false}; + std::thread captureThread_; + + DXGI_OUTPUT_DESC outputDesc_{}; + RECT captureRect_{}; + HWND windowHandle_ = nullptr; + bool captureWindow_ = false; + int fps_ = 60; + int captureWidth_ = 0; + int captureHeight_ = 0; + int64_t frameIntervalHns_ = 0; + int64_t lastFrameTimeHns_ = 0; + + bool initializeForMonitor(HMONITOR monitor, int fps); + bool findOutputForMonitor(HMONITOR monitor, ComPtr& adapter, ComPtr& output); + bool createD3DDevice(IDXGIAdapter1* adapter); + bool createDuplication(IDXGIOutput1* output); + bool createCaptureTexture(); + bool initializeFullMonitorRect(); + bool initializeWindowRect(HWND hwnd); + bool updateWindowCaptureRect(); + void captureLoop(); + int64_t nowHns() const; +}; \ No newline at end of file diff --git a/electron/native/windows-capture/src/main.cpp b/electron/native/windows-capture/src/main.cpp new file mode 100644 index 00000000..2aac86a7 --- /dev/null +++ b/electron/native/windows-capture/src/main.cpp @@ -0,0 +1,324 @@ +#include "dxgi_session.h" +#include "mf_encoder.h" +#include "monitor_utils.h" +#include "wasapi_loopback.h" + +#include +#include +#include +#include +#include +#include +#include + +static std::atomic g_stopRequested{false}; +static std::atomic g_pauseRequested{false}; +static std::atomic g_resumePending{false}; +static std::atomic g_lastFrameTimestampHns{0}; +static std::atomic g_pauseStartTimestampHns{0}; +static std::atomic g_accumulatedPausedHns{0}; +static std::mutex g_stopMutex; +static std::condition_variable g_stopCv; + +struct CaptureConfig { + int64_t displayId = 0; + int64_t windowHandle = 0; + std::string outputPath; + std::string audioOutputPath; + std::string micOutputPath; + std::string micDeviceName; + int fps = 60; + int width = 0; + int height = 0; + bool captureSystemAudio = false; + bool captureMic = false; +}; + +static bool parseSimpleJson(const std::string& json, CaptureConfig& config) { + auto findInt = [&](const std::string& key) -> int { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return -1; + pos = json.find(':', pos); + if (pos == std::string::npos) return -1; + pos++; + while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) pos++; + try { + return std::stoi(json.substr(pos)); + } catch (...) { + return -1; + } + }; + + auto findInt64 = [&](const std::string& key) -> int64_t { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return -1; + pos = json.find(':', pos); + if (pos == std::string::npos) return -1; + pos++; + while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) pos++; + try { + return std::stoll(json.substr(pos)); + } catch (...) { + return -1; + } + }; + + auto findString = [&](const std::string& key) -> std::string { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return ""; + pos = json.find(':', pos); + if (pos == std::string::npos) return ""; + pos++; + while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) pos++; + if (pos >= json.size() || json[pos] != '"') return ""; + pos++; + std::string result; + while (pos < json.size() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.size()) { + pos++; + if (json[pos] == 'n') result += '\n'; + else if (json[pos] == 't') result += '\t'; + else if (json[pos] == '\\') result += '\\'; + else if (json[pos] == '"') result += '"'; + else if (json[pos] == '/') result += '/'; + else result += json[pos]; + } else { + result += json[pos]; + } + pos++; + } + return result; + }; + + config.outputPath = findString("outputPath"); + if (config.outputPath.empty()) return false; + + int64_t displayId = findInt64("displayId"); + if (displayId >= 0) config.displayId = displayId; + + int64_t windowHandle = findInt64("windowHandle"); + if (windowHandle > 0) config.windowHandle = windowHandle; + + int fps = findInt("fps"); + if (fps > 0) config.fps = fps; + + int width = findInt("width"); + if (width > 0) config.width = width; + + int height = findInt("height"); + if (height > 0) config.height = height; + + config.audioOutputPath = findString("audioOutputPath"); + config.micOutputPath = findString("micOutputPath"); + config.micDeviceName = findString("micDeviceName"); + + auto findBool = [&](const std::string& key) -> bool { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return false; + auto colonPos = json.find(':', pos); + if (colonPos == std::string::npos) return false; + auto valStart = json.find_first_not_of(" \t", colonPos + 1); + return valStart != std::string::npos && json.substr(valStart, 4) == "true"; + }; + + config.captureSystemAudio = findBool("captureSystemAudio"); + config.captureMic = findBool("captureMic"); + + return true; +} + +static std::wstring utf8ToWide(const std::string& str) { + if (str.empty()) return L""; + int len = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), static_cast(str.size()), nullptr, 0); + std::wstring wstr(len, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, str.c_str(), static_cast(str.size()), &wstr[0], len); + return wstr; +} + +static void stdinListenerThread() { + std::string line; + while (std::getline(std::cin, line)) { + // Trim whitespace + while (!line.empty() && (line.back() == '\r' || line.back() == '\n' || line.back() == ' ')) { + line.pop_back(); + } + + if (line == "pause") { + g_pauseRequested = true; + g_pauseStartTimestampHns = g_lastFrameTimestampHns.load(); + continue; + } + + if (line == "resume") { + g_pauseRequested = false; + g_resumePending = true; + continue; + } + + if (line == "stop") { + g_stopRequested = true; + g_stopCv.notify_all(); + return; + } + } + + // stdin closed (parent process died) + g_stopRequested = true; + g_stopCv.notify_all(); +} + +int main(int argc, char* argv[]) { + if (argc < 2) { + std::cerr << "ERROR: Missing JSON config argument" << std::endl; + return 1; + } + + CaptureConfig config; + if (!parseSimpleJson(argv[1], config)) { + std::cerr << "ERROR: Failed to parse config JSON" << std::endl; + return 1; + } + + DxgiSession session; + + if (config.windowHandle > 0) { + HWND hwnd = reinterpret_cast(static_cast(config.windowHandle)); + if (!IsWindow(hwnd)) { + std::cerr << "ERROR: Invalid window handle " << config.windowHandle << std::endl; + return 1; + } + if (!session.initialize(hwnd, config.fps)) { + std::cerr << "ERROR: Failed to initialize DXGI window capture session" << std::endl; + return 1; + } + } else { + HMONITOR monitor = findMonitorByDisplayId(config.displayId); + if (!monitor) { + std::cerr << "ERROR: Could not find monitor for displayId " << config.displayId << std::endl; + return 1; + } + if (!session.initialize(monitor, config.fps)) { + std::cerr << "ERROR: Failed to initialize DXGI capture session" << std::endl; + return 1; + } + } + + int captureWidth = config.width > 0 ? config.width : session.captureWidth(); + int captureHeight = config.height > 0 ? config.height : session.captureHeight(); + + // Ensure even dimensions for H.264 + captureWidth = (captureWidth / 2) * 2; + captureHeight = (captureHeight / 2) * 2; + + // Initialize encoder + MFEncoder encoder; + std::wstring outputPathW = utf8ToWide(config.outputPath); + if (!encoder.initialize(outputPathW, captureWidth, captureHeight, config.fps, + session.device(), session.context())) { + std::cerr << "ERROR: Failed to initialize Media Foundation encoder" << std::endl; + return 1; + } + + // Set up frame callback + std::atomic frameCount{0}; + session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) { + g_lastFrameTimestampHns = timestampHns; + if (g_stopRequested) return; + + if (g_pauseRequested) return; + + int64_t adjustedTimestampHns = timestampHns; + if (g_resumePending.exchange(false)) { + const int64_t pauseStart = g_pauseStartTimestampHns.load(); + if (pauseStart > 0 && timestampHns > pauseStart) { + g_accumulatedPausedHns += (timestampHns - pauseStart); + } + } + + adjustedTimestampHns -= g_accumulatedPausedHns.load(); + if (adjustedTimestampHns < 0) { + adjustedTimestampHns = 0; + } + + if (encoder.writeFrame(texture, adjustedTimestampHns)) { + frameCount++; + } + }); + + // Start stdin listener + std::thread stdinThread(stdinListenerThread); + stdinThread.detach(); + + // Initialize WASAPI captures (but don't start yet) + WasapiCapture loopback; + WasapiCapture micCapture; + bool audioActive = false; + bool audioInitialized = false; + bool micActive = false; + bool micInitialized = false; + + if (config.captureSystemAudio && !config.audioOutputPath.empty()) { + audioInitialized = loopback.initializeLoopback(config.audioOutputPath); + if (!audioInitialized) { + std::cerr << "WARNING: Failed to initialize WASAPI loopback" << std::endl; + } + } + + if (config.captureMic && !config.micOutputPath.empty()) { + micInitialized = micCapture.initializeMic(config.micOutputPath, config.micDeviceName); + if (!micInitialized) { + std::cerr << "WARNING: Failed to initialize WASAPI mic capture" << std::endl; + } + } + + // Start video capture, then audio immediately after for sync + if (!session.startCapture()) { + std::cerr << "ERROR: Failed to start DXGI capture" << std::endl; + return 1; + } + + if (audioInitialized) { + audioActive = loopback.start(); + } + if (micInitialized) { + micActive = micCapture.start(); + } + + std::cout << "Recording started" << std::endl; + std::cout.flush(); + + // Wait for stop signal + while (!g_stopRequested) { + if (g_pauseRequested) { + if (audioActive) loopback.pause(); + if (micActive) micCapture.pause(); + } else { + if (audioActive) loopback.resume(); + if (micActive) micCapture.resume(); + } + + std::unique_lock lock(g_stopMutex); + g_stopCv.wait_for(lock, std::chrono::milliseconds(20), [] { return g_stopRequested.load(); }); + } + + // Stop capture and finalize + session.stopCapture(); + if (audioActive) loopback.stop(); + if (micActive) micCapture.stop(); + encoder.finalize(); + + std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl; + if (audioActive) { + std::cout << "Audio path: " << config.audioOutputPath << std::endl; + } + if (micActive) { + std::cout << "Mic path: " << config.micOutputPath << std::endl; + } + std::cout.flush(); + + // Allow pipe buffers to drain before forceful exit + Sleep(100); + + // Fast exit to avoid WinRT/COM teardown crashes during apartment cleanup + ExitProcess(0); +} diff --git a/electron/native/windows-capture/src/mf_encoder.cpp b/electron/native/windows-capture/src/mf_encoder.cpp new file mode 100644 index 00000000..a1474c20 --- /dev/null +++ b/electron/native/windows-capture/src/mf_encoder.cpp @@ -0,0 +1,205 @@ +#include "mf_encoder.h" +#include +#include +#include +#include +#include + +#pragma comment(lib, "mfplat.lib") +#pragma comment(lib, "mfreadwrite.lib") +#pragma comment(lib, "mf.lib") +#pragma comment(lib, "mfuuid.lib") + +static int clampByte(int v) { + return v < 0 ? 0 : (v > 255 ? 255 : v); +} + +MFEncoder::MFEncoder() {} + +MFEncoder::~MFEncoder() { + finalize(); +} + +bool MFEncoder::initialize(const std::wstring& outputPath, int width, int height, int fps, + ID3D11Device* device, ID3D11DeviceContext* context) { + if (initialized_) return false; + + if (width % 2 != 0 || height % 2 != 0) { + std::cerr << "ERROR: Encoder dimensions must be even, got " << width << "x" << height << std::endl; + return false; + } + + width_ = width; + height_ = height; + fps_ = fps; + device_ = device; + context_ = context; + + HRESULT hr = MFStartup(MF_VERSION); + if (FAILED(hr)) { + std::cerr << "ERROR: MFStartup failed: 0x" << std::hex << hr << std::endl; + return false; + } + + // Output media type (H.264) + ComPtr outputType; + hr = MFCreateMediaType(&outputType); + if (FAILED(hr)) return false; + + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + outputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); + outputType->SetUINT32(MF_MT_AVG_BITRATE, 20000000); + MFSetAttributeSize(outputType.Get(), MF_MT_FRAME_SIZE, width_, height_); + MFSetAttributeRatio(outputType.Get(), MF_MT_FRAME_RATE, fps_, 1); + MFSetAttributeRatio(outputType.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + outputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + + // Input media type (NV12) + ComPtr inputType; + hr = MFCreateMediaType(&inputType); + if (FAILED(hr)) return false; + + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); + MFSetAttributeSize(inputType.Get(), MF_MT_FRAME_SIZE, width_, height_); + MFSetAttributeRatio(inputType.Get(), MF_MT_FRAME_RATE, fps_, 1); + MFSetAttributeRatio(inputType.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + + // Create SinkWriter with MPEG4 container + ComPtr writerAttrs; + hr = MFCreateAttributes(&writerAttrs, 1); + if (FAILED(hr)) return false; + + writerAttrs->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE); + + hr = MFCreateSinkWriterFromURL(outputPath.c_str(), nullptr, writerAttrs.Get(), &sinkWriter_); + if (FAILED(hr)) { + std::cerr << "ERROR: MFCreateSinkWriterFromURL failed: 0x" << std::hex << hr << std::endl; + return false; + } + + hr = sinkWriter_->AddStream(outputType.Get(), &streamIndex_); + if (FAILED(hr)) { + std::cerr << "ERROR: AddStream failed: 0x" << std::hex << hr << std::endl; + return false; + } + + hr = sinkWriter_->SetInputMediaType(streamIndex_, inputType.Get(), nullptr); + if (FAILED(hr)) { + std::cerr << "ERROR: SetInputMediaType failed: 0x" << std::hex << hr << std::endl; + return false; + } + + hr = sinkWriter_->BeginWriting(); + if (FAILED(hr)) { + std::cerr << "ERROR: BeginWriting failed: 0x" << std::hex << hr << std::endl; + return false; + } + + // Pre-allocate staging texture + D3D11_TEXTURE2D_DESC stagingDesc = {}; + stagingDesc.Width = width_; + stagingDesc.Height = height_; + stagingDesc.MipLevels = 1; + stagingDesc.ArraySize = 1; + stagingDesc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + stagingDesc.SampleDesc.Count = 1; + stagingDesc.Usage = D3D11_USAGE_STAGING; + stagingDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + + hr = device_->CreateTexture2D(&stagingDesc, nullptr, &stagingTexture_); + if (FAILED(hr)) { + std::cerr << "ERROR: Failed to create staging texture: 0x" << std::hex << hr << std::endl; + return false; + } + + // Pre-allocate NV12 buffer + const int ySize = width_ * height_; + const int uvSize = (width_ / 2) * (height_ / 2) * 2; + nv12Buffer_.resize(ySize + uvSize); + + initialized_ = true; + return true; +} + +bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns) { + if (!initialized_ || !sinkWriter_) return false; + + context_->CopyResource(stagingTexture_.Get(), texture); + + D3D11_MAPPED_SUBRESOURCE mapped; + HRESULT hr = context_->Map(stagingTexture_.Get(), 0, D3D11_MAP_READ, 0, &mapped); + if (FAILED(hr)) return false; + + // Convert BGRA → NV12 + const uint8_t* bgra = static_cast(mapped.pData); + const int bgraPitch = static_cast(mapped.RowPitch); + + // Y plane + for (int y = 0; y < height_; y++) { + for (int x = 0; x < width_; x++) { + const uint8_t* pixel = bgra + y * bgraPitch + x * 4; + uint8_t b = pixel[0], g = pixel[1], r = pixel[2]; + int yVal = ((66 * r + 129 * g + 25 * b + 128) >> 8) + 16; + nv12Buffer_[y * width_ + x] = static_cast(clampByte(yVal)); + } + } + + // UV plane (interleaved, subsampled 2x2) + const int ySize = width_ * height_; + uint8_t* uvPlane = nv12Buffer_.data() + ySize; + for (int y = 0; y < height_; y += 2) { + for (int x = 0; x < width_; x += 2) { + const uint8_t* pixel = bgra + y * bgraPitch + x * 4; + uint8_t b = pixel[0], g = pixel[1], r = pixel[2]; + int u = ((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128; + int v = ((112 * r - 94 * g - 18 * b + 128) >> 8) + 128; + int uvIdx = (y / 2) * width_ + (x / 2) * 2; + uvPlane[uvIdx] = static_cast(clampByte(u)); + uvPlane[uvIdx + 1] = static_cast(clampByte(v)); + } + } + + context_->Unmap(stagingTexture_.Get(), 0); + + // Create MF sample + DWORD bufferSize = static_cast(nv12Buffer_.size()); + ComPtr buffer; + hr = MFCreateMemoryBuffer(bufferSize, &buffer); + if (FAILED(hr)) return false; + + BYTE* bufferData = nullptr; + hr = buffer->Lock(&bufferData, nullptr, nullptr); + if (FAILED(hr)) return false; + + std::memcpy(bufferData, nv12Buffer_.data(), bufferSize); + buffer->Unlock(); + buffer->SetCurrentLength(bufferSize); + + ComPtr sample; + hr = MFCreateSample(&sample); + if (FAILED(hr)) return false; + + sample->AddBuffer(buffer.Get()); + sample->SetSampleTime(timestampHns); + sample->SetSampleDuration(10000000LL / fps_); + + hr = sinkWriter_->WriteSample(streamIndex_, sample.Get()); + return SUCCEEDED(hr); +} + +bool MFEncoder::finalize() { + if (!initialized_) return false; + initialized_ = false; + + stagingTexture_.Reset(); + nv12Buffer_.clear(); + nv12Buffer_.shrink_to_fit(); + + if (!sinkWriter_) return false; + HRESULT hr = sinkWriter_->Finalize(); + sinkWriter_.Reset(); + MFShutdown(); + return SUCCEEDED(hr); +} diff --git a/electron/native/windows-capture/src/mf_encoder.h b/electron/native/windows-capture/src/mf_encoder.h new file mode 100644 index 00000000..7eb8e642 --- /dev/null +++ b/electron/native/windows-capture/src/mf_encoder.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +using Microsoft::WRL::ComPtr; + +class MFEncoder { +public: + MFEncoder(); + ~MFEncoder(); + + bool initialize(const std::wstring& outputPath, int width, int height, int fps, + ID3D11Device* device, ID3D11DeviceContext* context); + bool writeFrame(ID3D11Texture2D* texture, int64_t timestampHns); + bool finalize(); + +private: + ComPtr sinkWriter_; + ID3D11Device* device_ = nullptr; + ID3D11DeviceContext* context_ = nullptr; + ComPtr stagingTexture_; + std::vector nv12Buffer_; + DWORD streamIndex_ = 0; + int width_ = 0; + int height_ = 0; + int fps_ = 60; + bool initialized_ = false; +}; diff --git a/electron/native/windows-capture/src/monitor_utils.cpp b/electron/native/windows-capture/src/monitor_utils.cpp new file mode 100644 index 00000000..56962aeb --- /dev/null +++ b/electron/native/windows-capture/src/monitor_utils.cpp @@ -0,0 +1,61 @@ +#include "monitor_utils.h" +#include + +static BOOL CALLBACK enumMonitorCallback(HMONITOR hMonitor, HDC, LPRECT, LPARAM lParam) { + auto* monitors = reinterpret_cast*>(lParam); + + MONITORINFOEXW mi = {}; + mi.cbSize = sizeof(mi); + if (GetMonitorInfoW(hMonitor, &mi)) { + MonitorInfo info; + info.handle = hMonitor; + info.x = mi.rcMonitor.left; + info.y = mi.rcMonitor.top; + info.width = mi.rcMonitor.right - mi.rcMonitor.left; + info.height = mi.rcMonitor.bottom - mi.rcMonitor.top; + info.deviceName = mi.szDevice; + monitors->push_back(info); + } + + return TRUE; +} + +std::vector enumerateMonitors() { + std::vector monitors; + EnumDisplayMonitors(nullptr, nullptr, enumMonitorCallback, reinterpret_cast(&monitors)); + return monitors; +} + +// Electron uses the HMONITOR handle value cast to a number as the display ID. +HMONITOR findMonitorByDisplayId(int64_t displayId) { + auto monitors = enumerateMonitors(); + + for (const auto& m : monitors) { + if (static_cast(reinterpret_cast(m.handle)) == displayId) { + return m.handle; + } + } + + if (!monitors.empty()) { + return monitors[0].handle; + } + + return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY); +} + +MonitorInfo getMonitorInfo(HMONITOR monitor) { + MonitorInfo info; + info.handle = monitor; + + MONITORINFOEXW mi = {}; + mi.cbSize = sizeof(mi); + if (GetMonitorInfoW(monitor, &mi)) { + info.x = mi.rcMonitor.left; + info.y = mi.rcMonitor.top; + info.width = mi.rcMonitor.right - mi.rcMonitor.left; + info.height = mi.rcMonitor.bottom - mi.rcMonitor.top; + info.deviceName = mi.szDevice; + } + + return info; +} diff --git a/electron/native/windows-capture/src/monitor_utils.h b/electron/native/windows-capture/src/monitor_utils.h new file mode 100644 index 00000000..8889410d --- /dev/null +++ b/electron/native/windows-capture/src/monitor_utils.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include +#include +#include + +struct MonitorInfo { + HMONITOR handle; + int x; + int y; + int width; + int height; + std::wstring deviceName; +}; + +std::vector enumerateMonitors(); +HMONITOR findMonitorByDisplayId(int64_t displayId); +MonitorInfo getMonitorInfo(HMONITOR monitor); diff --git a/electron/native/windows-capture/src/wasapi_loopback.cpp b/electron/native/windows-capture/src/wasapi_loopback.cpp new file mode 100644 index 00000000..8ae15ad1 --- /dev/null +++ b/electron/native/windows-capture/src/wasapi_loopback.cpp @@ -0,0 +1,274 @@ +#include "wasapi_loopback.h" +#include +#include +#include + +#pragma comment(lib, "ole32.lib") + +static const CLSID CLSID_MMDeviceEnumerator_ = __uuidof(MMDeviceEnumerator); +static const IID IID_IMMDeviceEnumerator_ = __uuidof(IMMDeviceEnumerator); +static const IID IID_IAudioClient_ = __uuidof(IAudioClient); +static const IID IID_IAudioCaptureClient_ = __uuidof(IAudioCaptureClient); + +WasapiCapture::WasapiCapture() {} + +WasapiCapture::~WasapiCapture() { + stop(); + if (mixFormat_) CoTaskMemFree(mixFormat_); + if (captureClient_) captureClient_->Release(); + if (audioClient_) audioClient_->Release(); + if (device_) device_->Release(); + if (enumerator_) enumerator_->Release(); +} + +static std::wstring utf8ToWide(const std::string& str) { + if (str.empty()) return L""; + int len = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), static_cast(str.size()), nullptr, 0); + std::wstring wstr(len, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, str.c_str(), static_cast(str.size()), &wstr[0], len); + return wstr; +} + +IMMDevice* WasapiCapture::findCaptureDeviceByName(const std::wstring& targetName) { + IMMDeviceCollection* collection = nullptr; + HRESULT hr = enumerator_->EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE, &collection); + if (FAILED(hr)) return nullptr; + + UINT count = 0; + collection->GetCount(&count); + + for (UINT i = 0; i < count; i++) { + IMMDevice* dev = nullptr; + collection->Item(i, &dev); + + IPropertyStore* store = nullptr; + dev->OpenPropertyStore(STGM_READ, &store); + PROPVARIANT pv; + PropVariantInit(&pv); + store->GetValue(PKEY_Device_FriendlyName, &pv); + std::wstring name = pv.pwszVal ? pv.pwszVal : L""; + PropVariantClear(&pv); + store->Release(); + + if (name.find(targetName) != std::wstring::npos || targetName.find(name) != std::wstring::npos) { + collection->Release(); + return dev; + } + dev->Release(); + } + + collection->Release(); + return nullptr; +} + +bool WasapiCapture::initializeLoopback(const std::string& outputPath) { + outputPath_ = outputPath; + streamFlags_ = AUDCLNT_STREAMFLAGS_LOOPBACK; + + HRESULT hr = CoCreateInstance( + CLSID_MMDeviceEnumerator_, nullptr, CLSCTX_ALL, + IID_IMMDeviceEnumerator_, reinterpret_cast(&enumerator_)); + if (FAILED(hr)) return false; + + hr = enumerator_->GetDefaultAudioEndpoint(eRender, eConsole, &device_); + if (FAILED(hr)) return false; + + return initializeCommon(); +} + +bool WasapiCapture::initializeMic(const std::string& outputPath, const std::string& deviceName) { + outputPath_ = outputPath; + streamFlags_ = 0; + + HRESULT hr = CoCreateInstance( + CLSID_MMDeviceEnumerator_, nullptr, CLSCTX_ALL, + IID_IMMDeviceEnumerator_, reinterpret_cast(&enumerator_)); + if (FAILED(hr)) return false; + + if (!deviceName.empty()) { + device_ = findCaptureDeviceByName(utf8ToWide(deviceName)); + } + if (!device_) { + hr = enumerator_->GetDefaultAudioEndpoint(eCapture, eCommunications, &device_); + if (FAILED(hr)) { + hr = enumerator_->GetDefaultAudioEndpoint(eCapture, eConsole, &device_); + if (FAILED(hr)) return false; + } + } + + return initializeCommon(); +} + +bool WasapiCapture::initializeCommon() { + HRESULT hr = device_->Activate(IID_IAudioClient_, CLSCTX_ALL, nullptr, + reinterpret_cast(&audioClient_)); + if (FAILED(hr)) return false; + + hr = audioClient_->GetMixFormat(&mixFormat_); + if (FAILED(hr)) return false; + + REFERENCE_TIME bufferDuration = 200000; // 20ms + hr = audioClient_->Initialize( + AUDCLNT_SHAREMODE_SHARED, + streamFlags_, + bufferDuration, 0, mixFormat_, nullptr); + if (FAILED(hr)) return false; + + hr = audioClient_->GetBufferSize(&bufferFrameCount_); + if (FAILED(hr)) return false; + + hr = audioClient_->GetService(IID_IAudioCaptureClient_, + reinterpret_cast(&captureClient_)); + if (FAILED(hr)) return false; + + return true; +} + +bool WasapiCapture::start() { + if (capturing_) return true; + + outputFile_ = CreateFileA( + outputPath_.c_str(), GENERIC_WRITE, 0, nullptr, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + + if (outputFile_ == INVALID_HANDLE_VALUE) { + std::cerr << "ERROR: Cannot create audio output file" << std::endl; + return false; + } + + totalDataBytes_ = 0; + writeWavHeader(outputFile_, 0); + + HRESULT hr = audioClient_->Start(); + if (FAILED(hr)) { + CloseHandle(outputFile_); + outputFile_ = INVALID_HANDLE_VALUE; + return false; + } + + capturing_ = true; + paused_ = false; + thread_ = std::thread(&WasapiCapture::captureThread, this); + return true; +} + +bool WasapiCapture::pause() { + if (!capturing_ || paused_) return true; + paused_ = true; + return audioClient_ ? SUCCEEDED(audioClient_->Stop()) : false; +} + +bool WasapiCapture::resume() { + if (!capturing_ || !paused_) return true; + HRESULT hr = audioClient_ ? audioClient_->Start() : E_FAIL; + if (FAILED(hr)) return false; + paused_ = false; + return true; +} + +void WasapiCapture::stop() { + if (!capturing_) return; + capturing_ = false; + if (thread_.joinable()) thread_.join(); + if (audioClient_) audioClient_->Stop(); + + if (outputFile_ != INVALID_HANDLE_VALUE) { + SetFilePointer(outputFile_, 0, nullptr, FILE_BEGIN); + writeWavHeader(outputFile_, totalDataBytes_); + CloseHandle(outputFile_); + outputFile_ = INVALID_HANDLE_VALUE; + } + + paused_ = false; +} + +static int16_t floatToInt16(float v) { + v = v < -1.0f ? -1.0f : (v > 1.0f ? 1.0f : v); + return static_cast(v * 32767.0f); +} + +bool WasapiCapture::writeWavHeader(HANDLE file, DWORD dataSize) { + WORD channels = static_cast(mixFormat_->nChannels); + DWORD sampleRate = mixFormat_->nSamplesPerSec; + WORD bitsPerSample = 16; + WORD blockAlign = channels * (bitsPerSample / 8); + DWORD byteRate = sampleRate * blockAlign; + + DWORD written; + WriteFile(file, "RIFF", 4, &written, nullptr); + DWORD chunkSize = 36 + dataSize; + WriteFile(file, &chunkSize, 4, &written, nullptr); + WriteFile(file, "WAVE", 4, &written, nullptr); + WriteFile(file, "fmt ", 4, &written, nullptr); + DWORD fmtSize = 16; + WriteFile(file, &fmtSize, 4, &written, nullptr); + WORD audioFormat = 1; + WriteFile(file, &audioFormat, 2, &written, nullptr); + WriteFile(file, &channels, 2, &written, nullptr); + WriteFile(file, &sampleRate, 4, &written, nullptr); + WriteFile(file, &byteRate, 4, &written, nullptr); + WriteFile(file, &blockAlign, 2, &written, nullptr); + WriteFile(file, &bitsPerSample, 2, &written, nullptr); + WriteFile(file, "data", 4, &written, nullptr); + WriteFile(file, &dataSize, 4, &written, nullptr); + return true; +} + +void WasapiCapture::captureThread() { + WORD channels = static_cast(mixFormat_->nChannels); + bool isFloat = (mixFormat_->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) || + (mixFormat_->wFormatTag == WAVE_FORMAT_EXTENSIBLE && + reinterpret_cast(mixFormat_)->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT); + + std::vector pcmBuffer; + + DWORD sleepMs = static_cast((static_cast(bufferFrameCount_) / mixFormat_->nSamplesPerSec) * 500.0); + if (sleepMs < 5) sleepMs = 5; + + while (capturing_) { + if (paused_) { + Sleep(10); + continue; + } + + Sleep(sleepMs); + + UINT32 packetLength = 0; + HRESULT hr = captureClient_->GetNextPacketSize(&packetLength); + if (FAILED(hr)) break; + + while (packetLength > 0) { + BYTE* data = nullptr; + UINT32 numFrames = 0; + DWORD flags = 0; + + hr = captureClient_->GetBuffer(&data, &numFrames, &flags, nullptr, nullptr); + if (FAILED(hr)) break; + + UINT32 totalSamples = numFrames * channels; + + if (flags & AUDCLNT_BUFFERFLAGS_SILENT) { + pcmBuffer.assign(totalSamples, 0); + } else if (isFloat) { + pcmBuffer.resize(totalSamples); + const float* src = reinterpret_cast(data); + for (UINT32 i = 0; i < totalSamples; i++) { + pcmBuffer[i] = floatToInt16(src[i]); + } + } else { + pcmBuffer.resize(totalSamples); + std::memcpy(pcmBuffer.data(), data, totalSamples * sizeof(int16_t)); + } + + captureClient_->ReleaseBuffer(numFrames); + + DWORD bytesToWrite = totalSamples * sizeof(int16_t); + DWORD written; + WriteFile(outputFile_, pcmBuffer.data(), bytesToWrite, &written, nullptr); + totalDataBytes_ += written; + + hr = captureClient_->GetNextPacketSize(&packetLength); + if (FAILED(hr)) break; + } + } +} diff --git a/electron/native/windows-capture/src/wasapi_loopback.h b/electron/native/windows-capture/src/wasapi_loopback.h new file mode 100644 index 00000000..a4facf13 --- /dev/null +++ b/electron/native/windows-capture/src/wasapi_loopback.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +class WasapiCapture { +public: + WasapiCapture(); + ~WasapiCapture(); + + bool initializeLoopback(const std::string& outputPath); + bool initializeMic(const std::string& outputPath, const std::string& deviceName = ""); + bool start(); + bool pause(); + bool resume(); + void stop(); + +private: + bool initializeCommon(); + void captureThread(); + bool writeWavHeader(HANDLE file, DWORD dataSize); + IMMDevice* findCaptureDeviceByName(const std::wstring& name); + + std::string outputPath_; + std::thread thread_; + std::atomic capturing_{false}; + std::atomic paused_{false}; + HANDLE outputFile_ = INVALID_HANDLE_VALUE; + DWORD totalDataBytes_ = 0; + + IMMDeviceEnumerator* enumerator_ = nullptr; + IMMDevice* device_ = nullptr; + IAudioClient* audioClient_ = nullptr; + IAudioCaptureClient* captureClient_ = nullptr; + WAVEFORMATEX* mixFormat_ = nullptr; + DWORD streamFlags_ = 0; + + UINT32 bufferFrameCount_ = 0; +}; diff --git a/electron/windows.ts b/electron/windows.ts index cb265237..a9904667 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -246,7 +246,9 @@ export function createHudOverlayWindow(): BrowserWindow { } win.webContents.on("did-finish-load", () => { - win?.webContents.send("main-process-message", new Date().toLocaleString()); + if (!win.isDestroyed()) { + win.webContents.send("main-process-message", new Date().toLocaleString()); + } setTimeout(() => { if (!win.isDestroyed()) { win.show(); @@ -310,7 +312,9 @@ export function createEditorWindow(): BrowserWindow { }); win.webContents.on("did-finish-load", () => { - win?.webContents.send("main-process-message", new Date().toLocaleString()); + if (!win.isDestroyed()) { + win.webContents.send("main-process-message", new Date().toLocaleString()); + } }); if (VITE_DEV_SERVER_URL) { diff --git a/package.json b/package.json index 86fd82b9..124d9a16 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "type": "module", "scripts": { "dev": "vite", - "postinstall": "npm run rebuild:native && npm run build:platform-native-helpers", + "postinstall": "node scripts/postinstall.mjs", "build": "npm run build:platform-native-helpers && tsc && vite build && electron-builder", "lint": "biome check .", "lint:fix": "biome check --write .", diff --git a/scripts/build-cursor-monitor.mjs b/scripts/build-cursor-monitor.mjs index 03b0c9d6..df973241 100644 --- a/scripts/build-cursor-monitor.mjs +++ b/scripts/build-cursor-monitor.mjs @@ -1,5 +1,5 @@ import { execSync } from "node:child_process"; -import { existsSync, mkdirSync } from "node:fs"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; import path from "node:path"; const projectRoot = process.cwd(); @@ -31,29 +31,47 @@ function findCmake() { return `"${localCmake}"`; } - // VS 2022 bundled CMake - const vsEditions = ["Community", "Professional", "Enterprise", "BuildTools"]; - for (const edition of vsEditions) { - const cmakePath = path.join( - "C:", - "Program Files", - "Microsoft Visual Studio", - "2022", - edition, - "Common7", - "IDE", - "CommonExtensions", - "Microsoft", - "CMake", - "CMake", - "bin", - "cmake.exe", - ); + // Standalone CMake paths + const standaloneCmakePaths = [ + path.join("C:", "Program Files", "CMake", "bin", "cmake.exe"), + path.join("C:", "Program Files (x86)", "CMake", "bin", "cmake.exe"), + ]; + for (const cmakePath of standaloneCmakePaths) { if (existsSync(cmakePath)) { return `"${cmakePath}"`; } } + // VS 2022/2019 bundled CMake + const vsRoots = [ + path.join("C:", "Program Files", "Microsoft Visual Studio"), + path.join("C:", "Program Files (x86)", "Microsoft Visual Studio"), + ]; + const vsEditions = ["Community", "Professional", "Enterprise", "BuildTools"]; + const vsVersions = ["2022", "2019"]; + for (const root of vsRoots) { + for (const version of vsVersions) { + for (const edition of vsEditions) { + const cmakePath = path.join( + root, + version, + edition, + "Common7", + "IDE", + "CommonExtensions", + "Microsoft", + "CMake", + "CMake", + "bin", + "cmake.exe", + ); + if (existsSync(cmakePath)) { + return `"${cmakePath}"`; + } + } + } + } + return null; } @@ -66,9 +84,17 @@ if (!cmake) { } mkdirSync(buildDir, { recursive: true }); +const cacheFile = path.join(buildDir, "CMakeCache.txt"); +const cacheDir = path.join(buildDir, "CMakeFiles"); + +function clearCmakeCache() { + rmSync(cacheFile, { force: true }); + rmSync(cacheDir, { recursive: true, force: true }); +} console.log("[build-cursor-monitor] Configuring CMake..."); try { + clearCmakeCache(); execSync(`${cmake} .. -G "Visual Studio 17 2022" -A x64`, { cwd: buildDir, stdio: "inherit", @@ -77,6 +103,7 @@ try { } catch { console.log("[build-cursor-monitor] VS 2022 generator not found, trying VS 2019..."); try { + clearCmakeCache(); execSync(`${cmake} .. -G "Visual Studio 16 2019" -A x64`, { cwd: buildDir, stdio: "inherit", diff --git a/scripts/build-whisper-runtime.mjs b/scripts/build-whisper-runtime.mjs index bd720d28..1c284a4a 100644 --- a/scripts/build-whisper-runtime.mjs +++ b/scripts/build-whisper-runtime.mjs @@ -10,6 +10,11 @@ const nativeRoot = path.join(projectRoot, "electron", "native"); const cacheRoot = path.join(projectRoot, ".tmp", "whisper-runtime"); const archivePath = path.join(cacheRoot, `${whisperVersion}.tar.gz`); const extractRoot = path.join(cacheRoot, `src-${whisperVersion}`); + +function getHostArch() { + return process.arch === "arm64" ? "arm64" : "x64"; +} + function getNativeArchTag(platform, arch) { if (platform === "darwin") { return arch === "arm64" ? "darwin-arm64" : "darwin-x64"; @@ -26,29 +31,66 @@ function getNativeArchTag(platform, arch) { throw new Error(`[build-whisper-runtime] Unsupported platform: ${platform}/${arch}`); } -function getTargetConfigs() { - if (process.platform === "darwin") { - return [ - { - platform: "darwin", - arch: "arm64", - archTag: getNativeArchTag("darwin", "arm64"), - buildRoot: path.join(cacheRoot, "build-darwin-arm64"), - outputDir: path.join(nativeRoot, "bin", getNativeArchTag("darwin", "arm64")), - configureArgs: ["-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_OSX_ARCHITECTURES=arm64"], - }, - { - platform: "darwin", - arch: "x64", - archTag: getNativeArchTag("darwin", "x64"), - buildRoot: path.join(cacheRoot, "build-darwin-x64"), - outputDir: path.join(nativeRoot, "bin", getNativeArchTag("darwin", "x64")), - configureArgs: ["-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_OSX_ARCHITECTURES=x86_64"], - }, - ]; +function getRequestedArchitectures(platform) { + const hostArch = getHostArch(); + const configured = process.env.WHISPER_RUNTIME_ARCHS?.trim(); + + if (!configured) { + return [hostArch]; } - const arch = process.arch === "arm64" ? "arm64" : "x64"; + if (configured === "all") { + return ["arm64", "x64"]; + } + + const supported = new Set(["arm64", "x64"]); + const requested = configured + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean); + + if (requested.length === 0) { + return [hostArch]; + } + + const invalid = requested.filter((arch) => !supported.has(arch)); + if (invalid.length > 0) { + throw new Error( + `[build-whisper-runtime] Unsupported ${platform} target architecture request: ${invalid.join(", ")}`, + ); + } + + return [...new Set(requested)]; +} + +function createDarwinTarget(arch) { + const targetArch = arch === "arm64" ? "arm64" : "x64"; + const isCrossCompile = targetArch !== getHostArch(); + const configureArgs = [ + "-DCMAKE_BUILD_TYPE=Release", + `-DCMAKE_OSX_ARCHITECTURES=${targetArch === "arm64" ? "arm64" : "x86_64"}`, + ]; + + if (isCrossCompile) { + configureArgs.push("-DGGML_NATIVE=OFF"); + } + + return { + platform: "darwin", + arch: targetArch, + archTag: getNativeArchTag("darwin", targetArch), + buildRoot: path.join(cacheRoot, `build-darwin-${targetArch}`), + outputDir: path.join(nativeRoot, "bin", getNativeArchTag("darwin", targetArch)), + configureArgs, + }; +} + +function getTargetConfigs() { + if (process.platform === "darwin") { + return getRequestedArchitectures("darwin").map((arch) => createDarwinTarget(arch)); + } + + const arch = getHostArch(); const archTag = getNativeArchTag(process.platform, arch); if (process.platform === "win32") { @@ -326,8 +368,13 @@ async function main() { } const sourceDir = await ensureSourceTree(); + const targets = getTargetConfigs(); - for (const target of getTargetConfigs()) { + console.log( + `[build-whisper-runtime] Target architectures for ${process.platform}: ${targets.map((target) => target.archTag).join(", ")}`, + ); + + for (const target of targets) { if (await shouldSkipBuild(target)) { console.log( `[build-whisper-runtime] Whisper runtime ${whisperVersion} already staged for ${target.archTag}.`, diff --git a/scripts/build-windows-capture.mjs b/scripts/build-windows-capture.mjs index 32ee91b2..8df2c6e3 100644 --- a/scripts/build-windows-capture.mjs +++ b/scripts/build-windows-capture.mjs @@ -1,96 +1,136 @@ -import { execSync } from 'node:child_process'; -import { mkdirSync, existsSync } from 'node:fs'; -import path from 'node:path'; +import { execSync } from "node:child_process"; +import { mkdirSync, existsSync, rmSync } from "node:fs"; +import path from "node:path"; const projectRoot = process.cwd(); -const sourceDir = path.join(projectRoot, 'electron', 'native', 'wgc-capture'); -const buildDir = path.join(sourceDir, 'build'); +const sourceDir = path.join(projectRoot, "electron", "native", "wgc-capture"); +const buildDir = path.join(sourceDir, "build"); -if (process.platform !== 'win32') { - console.log('[build-windows-capture] Skipping native Windows capture build: host platform is not Windows.'); - process.exit(0); +if (process.platform !== "win32") { + console.log("[build-windows-capture] Skipping native Windows capture build: host platform is not Windows."); + process.exit(0); } -if (!existsSync(path.join(sourceDir, 'CMakeLists.txt'))) { - console.error('[build-windows-capture] CMakeLists.txt not found at', sourceDir); - process.exit(1); +if (!existsSync(path.join(sourceDir, "CMakeLists.txt"))) { + console.error("[build-windows-capture] CMakeLists.txt not found at", sourceDir); + process.exit(1); } function findCmake() { - // Check PATH first - try { - execSync('cmake --version', { stdio: 'pipe' }); - return 'cmake'; - } catch { - // not on PATH - } + // Check PATH first + try { + execSync("cmake --version", { stdio: "pipe" }); + return "cmake"; + } catch { + // not on PATH + } - // Local .cmake_ext path - const localCmake = path.join(projectRoot, '.cmake_ext', 'cmake-4.3.0-windows-x86_64', 'bin', 'cmake.exe'); - if (existsSync(localCmake)) { - return `"${localCmake}"`; - } + // Local .cmake_ext path + const localCmake = path.join(projectRoot, ".cmake_ext", "cmake-4.3.0-windows-x86_64", "bin", "cmake.exe"); + if (existsSync(localCmake)) { + return `"${localCmake}"`; + } - // VS 2022 bundled CMake - const vsEditions = ['Community', 'Professional', 'Enterprise', 'BuildTools']; - for (const edition of vsEditions) { - const cmakePath = path.join( - 'C:', 'Program Files', 'Microsoft Visual Studio', '2022', edition, - 'Common7', 'IDE', 'CommonExtensions', 'Microsoft', 'CMake', 'CMake', 'bin', 'cmake.exe' - ); - if (existsSync(cmakePath)) { - return `"${cmakePath}"`; - } - } + // Standalone CMake paths + const standaloneCmakePaths = [ + path.join("C:", "Program Files", "CMake", "bin", "cmake.exe"), + path.join("C:", "Program Files (x86)", "CMake", "bin", "cmake.exe"), + ]; + for (const cmakePath of standaloneCmakePaths) { + if (existsSync(cmakePath)) { + return `"${cmakePath}"`; + } + } - return null; + // VS 2022/2019 bundled CMake + const vsRoots = [ + path.join("C:", "Program Files", "Microsoft Visual Studio"), + path.join("C:", "Program Files (x86)", "Microsoft Visual Studio"), + ]; + const vsEditions = ["Community", "Professional", "Enterprise", "BuildTools"]; + const vsVersions = ["2022", "2019"]; + for (const root of vsRoots) { + for (const version of vsVersions) { + for (const edition of vsEditions) { + const cmakePath = path.join( + root, + version, + edition, + "Common7", + "IDE", + "CommonExtensions", + "Microsoft", + "CMake", + "CMake", + "bin", + "cmake.exe", + ); + if (existsSync(cmakePath)) { + return `"${cmakePath}"`; + } + } + } + } + + return null; } const cmake = findCmake(); if (!cmake) { - console.error('[build-windows-capture] CMake not found. Install Visual Studio with C++ CMake tools or standalone CMake.'); - process.exit(1); + console.error( + "[build-windows-capture] CMake not found. Install Visual Studio with C++ CMake tools or standalone CMake.", + ); + process.exit(1); } mkdirSync(buildDir, { recursive: true }); +const cacheFile = path.join(buildDir, "CMakeCache.txt"); +const cacheDir = path.join(buildDir, "CMakeFiles"); -console.log('[build-windows-capture] Configuring CMake...'); +function clearCmakeCache() { + rmSync(cacheFile, { force: true }); + rmSync(cacheDir, { recursive: true, force: true }); +} + +console.log("[build-windows-capture] Configuring CMake..."); try { - execSync(`${cmake} .. -G "Visual Studio 17 2022" -A x64`, { - cwd: buildDir, - stdio: 'inherit', - timeout: 120000, - }); + clearCmakeCache(); + execSync(`${cmake} .. -G "Visual Studio 17 2022" -A x64`, { + cwd: buildDir, + stdio: "inherit", + timeout: 120000, + }); } catch { - console.log('[build-windows-capture] VS 2022 generator not found, trying VS 2019...'); - try { - execSync(`${cmake} .. -G "Visual Studio 16 2019" -A x64`, { - cwd: buildDir, - stdio: 'inherit', - timeout: 120000, - }); - } catch (innerError) { - console.error('[build-windows-capture] CMake configure failed:', innerError.message); - process.exit(1); - } + console.log("[build-windows-capture] VS 2022 generator not found, trying VS 2019..."); + try { + clearCmakeCache(); + execSync(`${cmake} .. -G "Visual Studio 16 2019" -A x64`, { + cwd: buildDir, + stdio: "inherit", + timeout: 120000, + }); + } catch (innerError) { + console.error("[build-windows-capture] CMake configure failed:", innerError.message); + process.exit(1); + } } -console.log('[build-windows-capture] Building native Windows capture helper...'); +console.log("[build-windows-capture] Building native Windows capture helper..."); try { - execSync(`${cmake} --build . --config Release`, { - cwd: buildDir, - stdio: 'inherit', - timeout: 300000, - }); + execSync(`${cmake} --build . --config Release`, { + cwd: buildDir, + stdio: "inherit", + timeout: 300000, + }); } catch (error) { - console.error('[build-windows-capture] Build failed:', error.message); - process.exit(1); + console.error("[build-windows-capture] Build failed:", error.message); + process.exit(1); } -const exePath = path.join(buildDir, 'Release', 'wgc-capture.exe'); +const exePath = path.join(buildDir, "Release", "wgc-capture.exe"); if (existsSync(exePath)) { - console.log(`[build-windows-capture] Built successfully: ${exePath}`); + console.log(`[build-windows-capture] Built successfully: ${exePath}`); } else { - console.error('[build-windows-capture] Expected exe not found at', exePath); - process.exit(1); + console.error("[build-windows-capture] Expected exe not found at", exePath); + process.exit(1); } diff --git a/scripts/postinstall.mjs b/scripts/postinstall.mjs new file mode 100644 index 00000000..3a35d3db --- /dev/null +++ b/scripts/postinstall.mjs @@ -0,0 +1,53 @@ +import { spawnSync } from "node:child_process"; + +const npmExecPath = process.env.npm_execpath; +const hasNpmExecPath = typeof npmExecPath === "string" && npmExecPath.length > 0; +const npmInvoker = hasNpmExecPath + ? { + command: process.execPath, + argsPrefix: [npmExecPath], + shell: false, + } + : { + command: process.platform === "win32" ? "npm.cmd" : "npm", + argsPrefix: [], + shell: process.platform === "win32", + }; + +function runScript(scriptName) { + console.log(`[postinstall] Running npm script: ${scriptName}`); + const result = spawnSync(npmInvoker.command, [...npmInvoker.argsPrefix, "run", scriptName], { + stdio: "inherit", + env: process.env, + shell: npmInvoker.shell, + }); + + if (result.error) { + console.error( + `[postinstall] Failed to start "${scriptName}" (${result.error.message}).`, + ); + return false; + } + + if (result.signal) { + console.error(`[postinstall] "${scriptName}" was terminated by signal ${result.signal}.`); + return false; + } + + if (result.status !== 0) { + console.error( + `[postinstall] "${scriptName}" exited with code ${result.status}.`, + ); + return false; + } + + return true; +} + +if (!runScript("rebuild:native")) { + process.exit(1); +} + +if (!runScript("build:platform-native-helpers")) { + process.exit(1); +} diff --git a/src/assets/cursors/amongus/default.png b/src/assets/cursors/amongus/default.png new file mode 100644 index 00000000..9fb001e3 Binary files /dev/null and b/src/assets/cursors/amongus/default.png differ diff --git a/src/assets/cursors/amongus/pointer.png b/src/assets/cursors/amongus/pointer.png new file mode 100644 index 00000000..8d89c8e3 Binary files /dev/null and b/src/assets/cursors/amongus/pointer.png differ diff --git a/src/assets/cursors/chooper/default.png b/src/assets/cursors/chooper/default.png new file mode 100644 index 00000000..73f3e702 Binary files /dev/null and b/src/assets/cursors/chooper/default.png differ diff --git a/src/assets/cursors/chooper/pointer.png b/src/assets/cursors/chooper/pointer.png new file mode 100644 index 00000000..a821fd71 Binary files /dev/null and b/src/assets/cursors/chooper/pointer.png differ diff --git a/src/assets/cursors/lavender/default.png b/src/assets/cursors/lavender/default.png new file mode 100644 index 00000000..0f1bd7c7 Binary files /dev/null and b/src/assets/cursors/lavender/default.png differ diff --git a/src/assets/cursors/lavender/pointer.png b/src/assets/cursors/lavender/pointer.png new file mode 100644 index 00000000..3d5985a9 Binary files /dev/null and b/src/assets/cursors/lavender/pointer.png differ diff --git a/src/assets/cursors/parched/default.png b/src/assets/cursors/parched/default.png new file mode 100644 index 00000000..ba6f4a46 Binary files /dev/null and b/src/assets/cursors/parched/default.png differ diff --git a/src/assets/cursors/parched/pointer.png b/src/assets/cursors/parched/pointer.png new file mode 100644 index 00000000..54214883 Binary files /dev/null and b/src/assets/cursors/parched/pointer.png differ diff --git a/src/assets/cursors/turtle/default.png b/src/assets/cursors/turtle/default.png new file mode 100644 index 00000000..fa1672d9 Binary files /dev/null and b/src/assets/cursors/turtle/default.png differ diff --git a/src/assets/cursors/turtle/pointer.png b/src/assets/cursors/turtle/pointer.png new file mode 100644 index 00000000..8fe8e4da Binary files /dev/null and b/src/assets/cursors/turtle/pointer.png differ diff --git a/src/components/launch/LaunchWindow.module.css b/src/components/launch/LaunchWindow.module.css index c8770a92..bfa9de88 100644 --- a/src/components/launch/LaunchWindow.module.css +++ b/src/components/launch/LaunchWindow.module.css @@ -217,14 +217,6 @@ color: #6360f5; } -.ddSeparator { - width: 100%; - height: 1px; - margin: 4px 0; - background: rgba(255, 255, 255, 0.07); - flex-shrink: 0; -} - .recBtn { position: relative; width: 46px; diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index b62ca488..7b54a332 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -111,14 +111,10 @@ function DropdownItem({ ); } -function BarSeparator() { +function Separator() { return
; } -function DropdownSeparator() { - return
; -} - function MicDeviceRow({ device, selected, @@ -639,7 +635,7 @@ export function LaunchWindow() { {formatTime(elapsed)} - + : } - + - + - + - + - + {microphoneEnabled && ( } diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 604b2860..14554058 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -18,7 +18,12 @@ import type { BuiltInWallpaper } from "@/lib/wallpapers"; import { BUILT_IN_WALLPAPERS, getAvailableWallpapers } from "@/lib/wallpapers"; import { type AspectRatio } from "@/utils/aspectRatioUtils"; import minimalCursorUrl from "../../../Minimal Cursor.svg"; +import amongusCursorUrl from "../../assets/cursors/amongus/default.png"; import tahoeCursorUrl from "../../assets/cursors/Cursor=Default.svg"; +import chooperCursorUrl from "../../assets/cursors/chooper/default.png"; +import lavenderCursorUrl from "../../assets/cursors/lavender/default.png"; +import parchedCursorUrl from "../../assets/cursors/parched/default.png"; +import turtleCursorUrl from "../../assets/cursors/turtle/default.png"; import { useI18n, useScopedT } from "../../contexts/I18nContext"; import { AnnotationSettingsPanel } from "./AnnotationSettingsPanel"; import { loadEditorPreferences, saveEditorPreferences } from "./editorPreferences"; @@ -262,6 +267,11 @@ const CURSOR_STYLE_OPTIONS: Array<{ value: CursorStyle; label: string }> = [ { value: "dot", label: "Dot" }, { value: "figma", label: "Minimal" }, { value: "mono", label: "Inverted" }, + { value: "lavender", label: "Lavender" }, + { value: "parched", label: "Parched" }, + { value: "chooper", label: "Chooper" }, + { value: "amongus", label: "Among Us" }, + { value: "turtle", label: "Turtle" }, ]; const CAPTION_LANGUAGE_OPTIONS = [ @@ -465,6 +475,36 @@ function CursorStylePreview({ ); } + if (style === "lavender") { + return ( + + ); + } + + if (style === "parched") { + return ( + + ); + } + + if (style === "chooper") { + return ( + + ); + } + + if (style === "amongus") { + return ( + + ); + } + + if (style === "turtle") { + return ( + + ); + } + return ( Math.max(max, region.zIndex), 0) + 1; - applyingHistoryRef.current = false; }, [cloneSnapshot], ); @@ -1185,7 +1184,7 @@ export default function VideoEditor() { }, [currentPersistedEditorState, currentSourcePath]); const syncRecordingSessionWebcam = useCallback( - async (webcamPath: string | null, timeOffsetMs = 0) => { + async (webcamPath: string | null) => { if (!currentSourcePath || !window.electronAPI.setCurrentRecordingSession) { return; } @@ -1193,7 +1192,6 @@ export default function VideoEditor() { await window.electronAPI.setCurrentRecordingSession({ videoPath: currentSourcePath, webcamPath, - timeOffsetMs, }); }, [currentSourcePath], @@ -1224,10 +1222,9 @@ export default function VideoEditor() { ...prev, enabled: true, sourcePath: result.path ?? null, - timeOffsetMs: 0, })); - await syncRecordingSessionWebcam(result.path, 0); + await syncRecordingSessionWebcam(result.path); toast.success(t("settings.effects.webcamFootageAdded")); }, [syncRecordingSessionWebcam, t]); @@ -1236,10 +1233,9 @@ export default function VideoEditor() { ...prev, enabled: false, sourcePath: null, - timeOffsetMs: 0, })); - await syncRecordingSessionWebcam(null, 0); + await syncRecordingSessionWebcam(null); toast.success(t("settings.effects.webcamFootageRemoved")); }, [syncRecordingSessionWebcam, t]); @@ -1308,7 +1304,6 @@ export default function VideoEditor() { ...prev, enabled: Boolean(sessionResult.session?.webcamPath), sourcePath: sessionResult.session?.webcamPath ?? null, - timeOffsetMs: sessionResult.session?.timeOffsetMs ?? 0, })); return; } @@ -1324,7 +1319,6 @@ export default function VideoEditor() { ...prev, enabled: false, sourcePath: null, - timeOffsetMs: 0, })); } else { setError("No video to load. Please record or select a video."); @@ -3454,18 +3448,15 @@ export default function VideoEditor() { onAnnotationDelete={handleAnnotationDelete} selectedAnnotationId={selectedAnnotationId} onSelectAnnotation={handleSelectAnnotation} + autoCaptions={autoCaptions} + onCaptionSpanChange={handleCaptionSpanChange} + selectedCaptionId={selectedCaptionId} + onSelectCaption={handleSelectCaption} + onClearAutoCaptions={handleClearAutoCaptions} aspectRatio={aspectRatio} onAspectRatioChange={setAspectRatio} onOpenCropEditor={handleOpenCropEditor} isCropped={isCropped} - autoCaptions={autoCaptions} - onCaptionSpanChange={(id, span) => { - setAutoCaptions((prev) => - prev.map((r) => (r.id === id ? { ...r, startMs: span.start, endMs: span.end } : r)), - ); - }} - selectedCaptionId={selectedCaptionId} - onSelectCaption={setSelectedCaptionId} timeSelection={timeSelection} onTimeSelectionChange={setTimeSelection} /> diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 79949f0b..42b642fc 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -951,7 +951,7 @@ const VideoPlayback = forwardRef( } const targetTime = clampMediaTimeToDuration( - Math.max(0, currentTime - (webcam.timeOffsetMs ?? 0) / 1000), + currentTime, Number.isFinite(webcamVideo.duration) ? webcamVideo.duration : null, ); diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 206d44d8..3c871e11 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -2,30 +2,31 @@ import type { ExportFormat, ExportQuality, GifFrameRate, GifSizePreset } from "@ import { DEFAULT_WALLPAPER_PATH } from "@/lib/wallpapers"; import { ASPECT_RATIOS, type AspectRatio, isCustomAspectRatio } from "@/utils/aspectRatioUtils"; import { + type AnnotationRegion, + type AudioRegion, type AutoCaptionAnimation, type AutoCaptionSettings, type CaptionCue, type CaptionCueWord, - type AnnotationRegion, - type AudioRegion, type CropRegion, type CursorStyle, - DEFAULT_AUTO_CAPTION_SETTINGS, - getDefaultCaptionFontFamily, DEFAULT_ANNOTATION_POSITION, DEFAULT_ANNOTATION_SIZE, DEFAULT_ANNOTATION_STYLE, + DEFAULT_AUTO_CAPTION_SETTINGS, + DEFAULT_CONNECTED_ZOOM_DURATION_MS, + DEFAULT_CONNECTED_ZOOM_EASING, + DEFAULT_CONNECTED_ZOOM_GAP_MS, DEFAULT_CROP_REGION, DEFAULT_CURSOR_CLICK_BOUNCE, DEFAULT_CURSOR_CLICK_BOUNCE_DURATION, DEFAULT_CURSOR_MOTION_BLUR, DEFAULT_CURSOR_SIZE, - DEFAULT_CURSOR_STYLE, DEFAULT_CURSOR_SMOOTHING, + DEFAULT_CURSOR_STYLE, DEFAULT_CURSOR_SWAY, - DEFAULT_CONNECTED_ZOOM_DURATION_MS, - DEFAULT_CONNECTED_ZOOM_EASING, - DEFAULT_CONNECTED_ZOOM_GAP_MS, + DEFAULT_FIGURE_DATA, + DEFAULT_PLAYBACK_SPEED, DEFAULT_WEBCAM_CORNER_RADIUS, DEFAULT_WEBCAM_MARGIN, DEFAULT_WEBCAM_OVERLAY, @@ -36,8 +37,6 @@ import { DEFAULT_WEBCAM_SHADOW, DEFAULT_WEBCAM_SIZE, DEFAULT_WEBCAM_TIME_OFFSET_MS, - DEFAULT_FIGURE_DATA, - DEFAULT_PLAYBACK_SPEED, DEFAULT_ZOOM_DEPTH, DEFAULT_ZOOM_IN_DURATION_MS, DEFAULT_ZOOM_IN_EASING, @@ -45,11 +44,12 @@ import { DEFAULT_ZOOM_MOTION_BLUR, DEFAULT_ZOOM_OUT_DURATION_MS, DEFAULT_ZOOM_OUT_EASING, + getDefaultCaptionFontFamily, type SpeedRegion, type TrimRegion, type WebcamOverlaySettings, - type ZoomTransitionEasing, type ZoomRegion, + type ZoomTransitionEasing, } from "./types"; export const PROJECT_VERSION = 1; @@ -420,12 +420,16 @@ export function normalizeProjectEditor(editor: Partial): Pro const endMs = Math.max(startMs + 1, rawEnd); const words: CaptionCueWord[] | undefined = Array.isArray(cue.words) ? cue.words - .filter( - (word): word is CaptionCueWord => Boolean(word && typeof word.text === "string"), + .filter((word): word is CaptionCueWord => + Boolean(word && typeof word.text === "string"), ) .map((word) => { - const rawWordStart = isFiniteNumber(word.startMs) ? Math.round(word.startMs) : startMs; - const rawWordEnd = isFiniteNumber(word.endMs) ? Math.round(word.endMs) : rawWordStart + 1; + const rawWordStart = isFiniteNumber(word.startMs) + ? Math.round(word.startMs) + : startMs; + const rawWordEnd = isFiniteNumber(word.endMs) + ? Math.round(word.endMs) + : rawWordStart + 1; const normalizedWordStart = clamp(rawWordStart, startMs, endMs - 1); const normalizedWordEnd = clamp(rawWordEnd, normalizedWordStart + 1, endMs); @@ -463,8 +467,7 @@ export function normalizeProjectEditor(editor: Partial): Pro typeof rawAutoCaptionSettings.language === "string" && rawAutoCaptionSettings.language.trim() ? rawAutoCaptionSettings.language.trim() : DEFAULT_AUTO_CAPTION_SETTINGS.language, - fontFamily: - getDefaultCaptionFontFamily(), + fontFamily: getDefaultCaptionFontFamily(), fontSize: isFiniteNumber(rawAutoCaptionSettings.fontSize) ? clamp(rawAutoCaptionSettings.fontSize, 16, 72) : DEFAULT_AUTO_CAPTION_SETTINGS.fontSize, @@ -485,11 +488,13 @@ export function normalizeProjectEditor(editor: Partial): Pro ? clamp(rawAutoCaptionSettings.boxRadius, 0, 40) : DEFAULT_AUTO_CAPTION_SETTINGS.boxRadius, textColor: - typeof rawAutoCaptionSettings.textColor === "string" && rawAutoCaptionSettings.textColor.trim() + typeof rawAutoCaptionSettings.textColor === "string" && + rawAutoCaptionSettings.textColor.trim() ? rawAutoCaptionSettings.textColor : DEFAULT_AUTO_CAPTION_SETTINGS.textColor, inactiveTextColor: - typeof rawAutoCaptionSettings.inactiveTextColor === "string" && rawAutoCaptionSettings.inactiveTextColor.trim() + typeof rawAutoCaptionSettings.inactiveTextColor === "string" && + rawAutoCaptionSettings.inactiveTextColor.trim() ? rawAutoCaptionSettings.inactiveTextColor : DEFAULT_AUTO_CAPTION_SETTINGS.inactiveTextColor, backgroundOpacity: isFiniteNumber(rawAutoCaptionSettings.backgroundOpacity) @@ -526,10 +531,11 @@ export function normalizeProjectEditor(editor: Partial): Pro const webcam: Partial = editor.webcam && typeof editor.webcam === "object" ? editor.webcam : {}; const webcamSourcePath = typeof webcam.sourcePath === "string" ? webcam.sourcePath : null; - const legacyZoomScaleEffect = - isFiniteNumber((webcam as Partial<{ zoomScaleEffect: number }>).zoomScaleEffect) - ? (webcam as Partial<{ zoomScaleEffect: number }>).zoomScaleEffect - : null; + const legacyZoomScaleEffect = isFiniteNumber( + (webcam as Partial<{ zoomScaleEffect: number }>).zoomScaleEffect, + ) + ? (webcam as Partial<{ zoomScaleEffect: number }>).zoomScaleEffect + : null; return { wallpaper: typeof editor.wallpaper === "string" ? editor.wallpaper : DEFAULT_WALLPAPER_PATH, @@ -554,7 +560,12 @@ export function normalizeProjectEditor(editor: Partial): Pro editor.cursorStyle === "dot" || editor.cursorStyle === "figma" || editor.cursorStyle === "mono" || - editor.cursorStyle === "tahoe" + editor.cursorStyle === "tahoe" || + editor.cursorStyle === "lavender" || + editor.cursorStyle === "parched" || + editor.cursorStyle === "chooper" || + editor.cursorStyle === "amongus" || + editor.cursorStyle === "turtle" ? editor.cursorStyle : DEFAULT_CURSOR_STYLE, cursorSize: isFiniteNumber(editor.cursorSize) @@ -569,7 +580,9 @@ export function normalizeProjectEditor(editor: Partial): Pro cursorClickBounce: isFiniteNumber((editor as Partial).cursorClickBounce) ? clamp((editor as Partial).cursorClickBounce as number, 0, 5) : DEFAULT_CURSOR_CLICK_BOUNCE, - cursorClickBounceDuration: isFiniteNumber((editor as Partial).cursorClickBounceDuration) + cursorClickBounceDuration: isFiniteNumber( + (editor as Partial).cursorClickBounceDuration, + ) ? clamp((editor as Partial).cursorClickBounceDuration as number, 60, 500) : DEFAULT_CURSOR_CLICK_BOUNCE_DURATION, cursorSway: isFiniteNumber((editor as Partial).cursorSway) @@ -594,9 +607,6 @@ export function normalizeProjectEditor(editor: Partial): Pro enabled: typeof webcam.enabled === "boolean" ? webcam.enabled : DEFAULT_WEBCAM_OVERLAY.enabled, sourcePath: webcamSourcePath, - timeOffsetMs: isFiniteNumber(webcam.timeOffsetMs) - ? Math.round(clamp(webcam.timeOffsetMs, -30_000, 30_000)) - : DEFAULT_WEBCAM_TIME_OFFSET_MS, mirror: typeof webcam.mirror === "boolean" ? webcam.mirror : DEFAULT_WEBCAM_OVERLAY.mirror, positionPreset: webcam.positionPreset === "top-left" || @@ -611,11 +621,11 @@ export function normalizeProjectEditor(editor: Partial): Pro webcam.positionPreset === "custom" ? webcam.positionPreset : webcam.corner === "top-left" || - webcam.corner === "top-right" || - webcam.corner === "bottom-left" || - webcam.corner === "bottom-right" - ? webcam.corner - : DEFAULT_WEBCAM_POSITION_PRESET, + webcam.corner === "top-right" || + webcam.corner === "bottom-left" || + webcam.corner === "bottom-right" + ? webcam.corner + : DEFAULT_WEBCAM_POSITION_PRESET, positionX: isFiniteNumber(webcam.positionX) ? clamp(webcam.positionX, 0, 1) : DEFAULT_WEBCAM_POSITION_X, @@ -640,6 +650,9 @@ export function normalizeProjectEditor(editor: Partial): Pro ? clamp(webcam.cornerRadius, 0, 160) : DEFAULT_WEBCAM_CORNER_RADIUS, shadow: isFiniteNumber(webcam.shadow) ? clamp(webcam.shadow, 0, 1) : DEFAULT_WEBCAM_SHADOW, + timeOffsetMs: isFiniteNumber(webcam.timeOffsetMs) + ? Math.round(webcam.timeOffsetMs) + : DEFAULT_WEBCAM_TIME_OFFSET_MS, margin: isFiniteNumber(webcam.margin) ? clamp(webcam.margin, 0, 96) : DEFAULT_WEBCAM_MARGIN, }, aspectRatio: diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 87088464..e87272e5 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -79,6 +79,7 @@ interface TimelineEditorProps { onCaptionSpanChange?: (id: string, span: Span) => void; selectedCaptionId?: string | null; onSelectCaption?: (id: string | null) => void; + onClearAutoCaptions?: () => void; aspectRatio: AspectRatio; onAspectRatioChange: (aspectRatio: AspectRatio) => void; onOpenCropEditor?: () => void; @@ -816,6 +817,7 @@ export default function TimelineEditor({ onCaptionSpanChange, selectedCaptionId, onSelectCaption, + onClearAutoCaptions, aspectRatio, onAspectRatioChange, onOpenCropEditor, @@ -980,6 +982,7 @@ export default function TimelineEditor({ speedIds.forEach((id) => onSpeedDelete?.(id)); audioIds.forEach((id) => onAudioDelete?.(id)); + onClearAutoCaptions?.(); clearSelectedBlocks(); setSelectedKeyframeId(null); }, [ @@ -988,6 +991,7 @@ export default function TimelineEditor({ clearSelectedBlocks, onAnnotationDelete, onAudioDelete, + onClearAutoCaptions, onSpeedDelete, onTrimDelete, onZoomDelete, @@ -999,33 +1003,80 @@ export default function TimelineEditor({ const handleSelectZoom = useCallback((id: string | null) => { setSelectAllBlocksActive(false); onSelectZoom(id); - }, [onSelectZoom]); + if (id) { + onSelectTrim?.(null); + onSelectAnnotation?.(null); + onSelectSpeed?.(null); + onSelectAudio?.(null); + onSelectCaption?.(null); + onTimeSelectionChange?.(null); + } + }, [onSelectZoom, onSelectTrim, onSelectAnnotation, onSelectSpeed, onSelectAudio, onSelectCaption, onTimeSelectionChange]); const handleSelectTrim = useCallback((id: string | null) => { setSelectAllBlocksActive(false); onSelectTrim?.(id); - }, [onSelectTrim]); + if (id) { + onSelectZoom(null); + onSelectAnnotation?.(null); + onSelectSpeed?.(null); + onSelectAudio?.(null); + onSelectCaption?.(null); + onTimeSelectionChange?.(null); + } + }, [onSelectZoom, onSelectTrim, onSelectAnnotation, onSelectSpeed, onSelectAudio, onSelectCaption, onTimeSelectionChange]); const handleSelectAnnotation = useCallback((id: string | null) => { setSelectAllBlocksActive(false); onSelectAnnotation?.(id); - }, [onSelectAnnotation]); + if (id) { + onSelectZoom(null); + onSelectTrim?.(null); + onSelectSpeed?.(null); + onSelectAudio?.(null); + onSelectCaption?.(null); + onTimeSelectionChange?.(null); + } + }, [onSelectZoom, onSelectTrim, onSelectAnnotation, onSelectSpeed, onSelectAudio, onSelectCaption, onTimeSelectionChange]); const handleSelectSpeed = useCallback((id: string | null) => { setSelectAllBlocksActive(false); onSelectSpeed?.(id); - }, [onSelectSpeed]); + if (id) { + onSelectZoom(null); + onSelectTrim?.(null); + onSelectAnnotation?.(null); + onSelectAudio?.(null); + onSelectCaption?.(null); + onTimeSelectionChange?.(null); + } + }, [onSelectZoom, onSelectTrim, onSelectAnnotation, onSelectSpeed, onSelectAudio, onSelectCaption, onTimeSelectionChange]); const handleSelectAudio = useCallback((id: string | null) => { setSelectAllBlocksActive(false); onSelectAudio?.(id); - }, [onSelectAudio]); - + if (id) { + onSelectZoom(null); + onSelectTrim?.(null); + onSelectAnnotation?.(null); + onSelectSpeed?.(null); + onSelectCaption?.(null); + onTimeSelectionChange?.(null); + } + }, [onSelectZoom, onSelectTrim, onSelectAnnotation, onSelectSpeed, onSelectAudio, onSelectCaption, onTimeSelectionChange]); const handleSelectCaption = useCallback((id: string | null) => { setSelectAllBlocksActive(false); onSelectCaption?.(id); - }, [onSelectCaption]); + if (id) { + onSelectZoom(null); + onSelectTrim?.(null); + onSelectAnnotation?.(null); + onSelectSpeed?.(null); + onSelectAudio?.(null); + onTimeSelectionChange?.(null); + } + }, [onSelectZoom, onSelectTrim, onSelectAnnotation, onSelectSpeed, onSelectAudio, onSelectCaption, onTimeSelectionChange]); useEffect(() => { setRange(createInitialRange(totalMs)); diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 68b7d310..ae4ca1d4 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -1,85 +1,83 @@ export type ZoomDepth = 1 | 2 | 3 | 4 | 5 | 6; export interface ZoomFocus { - cx: number; // normalized horizontal center (0-1) - cy: number; // normalized vertical center (0-1) + cx: number; // normalized horizontal center (0-1) + cy: number; // normalized vertical center (0-1) } export interface ZoomRegion { - id: string; - startMs: number; - endMs: number; - depth: ZoomDepth; - focus: ZoomFocus; + id: string; + startMs: number; + endMs: number; + depth: ZoomDepth; + focus: ZoomFocus; } export interface CursorTelemetryPoint { - timeMs: number; - cx: number; - cy: number; - interactionType?: - | "move" - | "click" - | "double-click" - | "right-click" - | "middle-click" - | "mouseup"; - cursorType?: - | "arrow" - | "text" - | "pointer" - | "crosshair" - | "open-hand" - | "closed-hand" - | "resize-ew" - | "resize-ns" - | "not-allowed"; + timeMs: number; + cx: number; + cy: number; + interactionType?: "move" | "click" | "double-click" | "right-click" | "middle-click" | "mouseup"; + cursorType?: + | "arrow" + | "text" + | "pointer" + | "crosshair" + | "open-hand" + | "closed-hand" + | "resize-ew" + | "resize-ns" + | "not-allowed"; } export interface CursorVisualSettings { - size: number; - smoothing: number; - motionBlur: number; - clickBounce: number; - clickBounceDuration: number; - sway: number; - style: CursorStyle; + size: number; + smoothing: number; + motionBlur: number; + clickBounce: number; + clickBounceDuration: number; + sway: number; + style: CursorStyle; } -export type CursorStyle = "tahoe" | "dot" | "figma" | "mono"; +export type CursorStyle = + | "tahoe" + | "dot" + | "figma" + | "mono" + | "lavender" + | "parched" + | "chooper" + | "amongus" + | "turtle"; export const DEFAULT_CURSOR_STYLE: CursorStyle = "tahoe"; -export type ZoomTransitionEasing = - | "recordly" - | "glide" - | "smooth" - | "snappy" - | "linear"; +export type ZoomTransitionEasing = "recordly" | "glide" | "smooth" | "snappy" | "linear"; export type WebcamCorner = "top-left" | "top-right" | "bottom-left" | "bottom-right"; export type WebcamPositionPreset = - | WebcamCorner - | "top-center" - | "center-left" - | "center" - | "center-right" - | "bottom-center" - | "custom"; + | WebcamCorner + | "top-center" + | "center-left" + | "center" + | "center-right" + | "bottom-center" + | "custom"; export interface WebcamOverlaySettings { - enabled: boolean; - sourcePath: string | null; - timeOffsetMs: number; - mirror: boolean; - corner: WebcamCorner; - positionPreset: WebcamPositionPreset; - positionX: number; - positionY: number; - size: number; - reactToZoom: boolean; - cornerRadius: number; - shadow: number; - margin: number; + enabled: boolean; + sourcePath: string | null; + timeOffsetMs: number; + mirror: boolean; + corner: WebcamCorner; + positionPreset: WebcamPositionPreset; + positionX: number; + positionY: number; + size: number; + reactToZoom: boolean; + cornerRadius: number; + shadow: number; + margin: number; } export const DEFAULT_CURSOR_SIZE = 3.0; @@ -108,147 +106,147 @@ export const DEFAULT_WEBCAM_POSITION_Y = 1; export const DEFAULT_WEBCAM_TIME_OFFSET_MS = 0; export const DEFAULT_WEBCAM_OVERLAY: WebcamOverlaySettings = { - enabled: false, - sourcePath: null, - timeOffsetMs: DEFAULT_WEBCAM_TIME_OFFSET_MS, - mirror: true, - corner: "bottom-right", - positionPreset: DEFAULT_WEBCAM_POSITION_PRESET, - positionX: DEFAULT_WEBCAM_POSITION_X, - positionY: DEFAULT_WEBCAM_POSITION_Y, - size: DEFAULT_WEBCAM_SIZE, - reactToZoom: DEFAULT_WEBCAM_REACT_TO_ZOOM, - cornerRadius: DEFAULT_WEBCAM_CORNER_RADIUS, - shadow: DEFAULT_WEBCAM_SHADOW, - margin: DEFAULT_WEBCAM_MARGIN, + enabled: false, + sourcePath: null, + timeOffsetMs: DEFAULT_WEBCAM_TIME_OFFSET_MS, + mirror: true, + corner: "bottom-right", + positionPreset: DEFAULT_WEBCAM_POSITION_PRESET, + positionX: DEFAULT_WEBCAM_POSITION_X, + positionY: DEFAULT_WEBCAM_POSITION_Y, + size: DEFAULT_WEBCAM_SIZE, + reactToZoom: DEFAULT_WEBCAM_REACT_TO_ZOOM, + cornerRadius: DEFAULT_WEBCAM_CORNER_RADIUS, + shadow: DEFAULT_WEBCAM_SHADOW, + margin: DEFAULT_WEBCAM_MARGIN, }; export interface TrimRegion { - id: string; - startMs: number; - endMs: number; + id: string; + startMs: number; + endMs: number; } export type AnnotationType = "text" | "image" | "figure" | "blur"; export type ArrowDirection = - | "up" - | "down" - | "left" - | "right" - | "up-right" - | "up-left" - | "down-right" - | "down-left"; + | "up" + | "down" + | "left" + | "right" + | "up-right" + | "up-left" + | "down-right" + | "down-left"; export interface FigureData { - arrowDirection: ArrowDirection; - color: string; - strokeWidth: number; + arrowDirection: ArrowDirection; + color: string; + strokeWidth: number; } export interface AnnotationPosition { - x: number; - y: number; + x: number; + y: number; } export interface AnnotationSize { - width: number; - height: number; + width: number; + height: number; } export interface AnnotationTextStyle { - color: string; - backgroundColor: string; - fontSize: number; // pixels - fontFamily: string; - fontWeight: "normal" | "bold"; - fontStyle: "normal" | "italic"; - textDecoration: "none" | "underline"; - textAlign: "left" | "center" | "right"; + color: string; + backgroundColor: string; + fontSize: number; // pixels + fontFamily: string; + fontWeight: "normal" | "bold"; + fontStyle: "normal" | "italic"; + textDecoration: "none" | "underline"; + textAlign: "left" | "center" | "right"; } function getDefaultAnnotationFontFamily() { - if (typeof navigator !== "undefined" && /mac/i.test(navigator.platform)) { - return '"SF Pro Display", "SF Pro Text", -apple-system, BlinkMacSystemFont, sans-serif'; - } + if (typeof navigator !== "undefined" && /mac/i.test(navigator.platform)) { + return '"SF Pro Display", "SF Pro Text", -apple-system, BlinkMacSystemFont, sans-serif'; + } - return "Inter, system-ui, sans-serif"; + return "Inter, system-ui, sans-serif"; } export function getDefaultCaptionFontFamily() { - if (typeof navigator !== "undefined" && /mac/i.test(navigator.platform)) { - return '"SF Pro Text", "SF Pro Display", -apple-system, BlinkMacSystemFont, sans-serif'; - } + if (typeof navigator !== "undefined" && /mac/i.test(navigator.platform)) { + return '"SF Pro Text", "SF Pro Display", -apple-system, BlinkMacSystemFont, sans-serif'; + } - return '"Helvetica Neue", Helvetica, Arial, sans-serif'; + return '"Helvetica Neue", Helvetica, Arial, sans-serif'; } export interface AnnotationRegion { - id: string; - startMs: number; - endMs: number; - type: AnnotationType; - content: string; // Legacy - still used for current type - textContent?: string; // Separate storage for text - imageContent?: string; // Separate storage for image data URL - position: AnnotationPosition; - size: AnnotationSize; - style: AnnotationTextStyle; - zIndex: number; - figureData?: FigureData; - blurIntensity?: number; + id: string; + startMs: number; + endMs: number; + type: AnnotationType; + content: string; // Legacy - still used for current type + textContent?: string; // Separate storage for text + imageContent?: string; // Separate storage for image data URL + position: AnnotationPosition; + size: AnnotationSize; + style: AnnotationTextStyle; + zIndex: number; + figureData?: FigureData; + blurIntensity?: number; } export const DEFAULT_BLUR_INTENSITY = 12; export const DEFAULT_ANNOTATION_POSITION: AnnotationPosition = { - x: 50, - y: 50, + x: 50, + y: 50, }; export const DEFAULT_ANNOTATION_SIZE: AnnotationSize = { - width: 30, - height: 20, + width: 30, + height: 20, }; export const DEFAULT_ANNOTATION_STYLE: AnnotationTextStyle = { - color: "#ffffff", - backgroundColor: "transparent", - fontSize: 32, - fontFamily: getDefaultAnnotationFontFamily(), - fontWeight: "bold", - fontStyle: "normal", - textDecoration: "none", - textAlign: "center", + color: "#ffffff", + backgroundColor: "transparent", + fontSize: 32, + fontFamily: getDefaultAnnotationFontFamily(), + fontWeight: "bold", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", }; export const DEFAULT_FIGURE_DATA: FigureData = { - arrowDirection: "right", - color: "#2563EB", - strokeWidth: 4, + arrowDirection: "right", + color: "#2563EB", + strokeWidth: 4, }; export interface CropRegion { - x: number; - y: number; - width: number; - height: number; + x: number; + y: number; + width: number; + height: number; } export const DEFAULT_CROP_REGION: CropRegion = { - x: 0, - y: 0, - width: 1, - height: 1, + x: 0, + y: 0, + width: 1, + height: 1, }; export interface AudioRegion { - id: string; - startMs: number; - endMs: number; - audioPath: string; - volume: number; + id: string; + startMs: number; + endMs: number; + audioPath: string; + volume: number; } @@ -258,100 +256,97 @@ export interface TimeSelection { } export interface CaptionCue { - id: string; - startMs: number; - endMs: number; - text: string; - words?: CaptionCueWord[]; + id: string; + startMs: number; + endMs: number; + text: string; + words?: CaptionCueWord[]; } export interface CaptionCueWord { - text: string; - startMs: number; - endMs: number; - leadingSpace?: boolean; + text: string; + startMs: number; + endMs: number; + leadingSpace?: boolean; } export type AutoCaptionAnimation = "none" | "fade" | "rise" | "pop"; export type WhisperModel = "tiny" | "base" | "small" | "medium" | "large" | "custom"; export interface AutoCaptionSettings { - enabled: boolean; - language: string; - selectedModel: WhisperModel; - fontFamily: string; - fontSize: number; - bottomOffset: number; - maxWidth: number; - maxRows: number; - animationStyle: AutoCaptionAnimation; - boxRadius: number; - textColor: string; - inactiveTextColor: string; - backgroundOpacity: number; - generationRange: "full" | "selected"; + enabled: boolean; + language: string; + selectedModel: WhisperModel; + fontFamily: string; + fontSize: number; + bottomOffset: number; + maxWidth: number; + maxRows: number; + animationStyle: AutoCaptionAnimation; + boxRadius: number; + textColor: string; + inactiveTextColor: string; + backgroundOpacity: number; + generationRange: "full" | "selected"; } export const DEFAULT_AUTO_CAPTION_SETTINGS: AutoCaptionSettings = { - enabled: false, - language: "auto", - selectedModel: "small", - fontFamily: getDefaultCaptionFontFamily(), - fontSize: 30, - bottomOffset: 3, - maxWidth: 62, - maxRows: 1, - animationStyle: "fade", - boxRadius: 17.5, - textColor: "#FFFFFF", - inactiveTextColor: "#A3A3A3", - backgroundOpacity: 0.1, - generationRange: "full", + enabled: false, + language: "auto", + selectedModel: "small", + fontFamily: getDefaultCaptionFontFamily(), + fontSize: 30, + bottomOffset: 3, + maxWidth: 62, + maxRows: 1, + animationStyle: "fade", + boxRadius: 17.5, + textColor: "#FFFFFF", + inactiveTextColor: "#A3A3A3", + backgroundOpacity: 0.1, + generationRange: "full", }; export type PlaybackSpeed = 0.25 | 0.5 | 0.75 | 1.25 | 1.5 | 1.75 | 2; export interface SpeedRegion { - id: string; - startMs: number; - endMs: number; - speed: PlaybackSpeed; + id: string; + startMs: number; + endMs: number; + speed: PlaybackSpeed; } export const SPEED_OPTIONS: Array<{ speed: PlaybackSpeed; label: string }> = [ - { speed: 0.25, label: "0.25×" }, - { speed: 0.5, label: "0.5×" }, - { speed: 0.75, label: "0.75×" }, - { speed: 1.25, label: "1.25×" }, - { speed: 1.5, label: "1.5×" }, - { speed: 1.75, label: "1.75×" }, - { speed: 2, label: "2×" }, + { speed: 0.25, label: "0.25×" }, + { speed: 0.5, label: "0.5×" }, + { speed: 0.75, label: "0.75×" }, + { speed: 1.25, label: "1.25×" }, + { speed: 1.5, label: "1.5×" }, + { speed: 1.75, label: "1.75×" }, + { speed: 2, label: "2×" }, ]; export const DEFAULT_PLAYBACK_SPEED: PlaybackSpeed = 1.5; export const ZOOM_DEPTH_SCALES: Record = { - 1: 1.25, - 2: 1.5, - 3: 1.8, - 4: 2.2, - 5: 3.5, - 6: 5.0, + 1: 1.25, + 2: 1.5, + 3: 1.8, + 4: 2.2, + 5: 3.5, + 6: 5.0, }; export const DEFAULT_ZOOM_DEPTH: ZoomDepth = 3; -export function clampFocusToDepth( - focus: ZoomFocus, - _depth: ZoomDepth, -): ZoomFocus { - return { - cx: clamp(focus.cx, 0, 1), - cy: clamp(focus.cy, 0, 1), - }; +export function clampFocusToDepth(focus: ZoomFocus, _depth: ZoomDepth): ZoomFocus { + return { + cx: clamp(focus.cx, 0, 1), + cy: clamp(focus.cy, 0, 1), + }; } function clamp(value: number, min: number, max: number) { - if (Number.isNaN(value)) return (min + max) / 2; - return Math.min(max, Math.max(min, value)); + if (Number.isNaN(value)) return (min + max) / 2; + return Math.min(max, Math.max(min, value)); } diff --git a/src/components/video-editor/videoPlayback/cursorRenderer.ts b/src/components/video-editor/videoPlayback/cursorRenderer.ts index b729fb8c..bb5eee27 100644 --- a/src/components/video-editor/videoPlayback/cursorRenderer.ts +++ b/src/components/video-editor/videoPlayback/cursorRenderer.ts @@ -1,6 +1,16 @@ import { Assets, BlurFilter, Container, Graphics, Sprite, Texture } from "pixi.js"; import { MotionBlurFilter } from "pixi-filters/motion-blur"; import minimalCursorUrl from "../../../../Minimal Cursor.svg"; +import amongusDefaultCursorUrl from "../../../assets/cursors/amongus/default.png"; +import amongusPointerCursorUrl from "../../../assets/cursors/amongus/pointer.png"; +import chooperDefaultCursorUrl from "../../../assets/cursors/chooper/default.png"; +import chooperPointerCursorUrl from "../../../assets/cursors/chooper/pointer.png"; +import lavenderDefaultCursorUrl from "../../../assets/cursors/lavender/default.png"; +import lavenderPointerCursorUrl from "../../../assets/cursors/lavender/pointer.png"; +import parchedDefaultCursorUrl from "../../../assets/cursors/parched/default.png"; +import parchedPointerCursorUrl from "../../../assets/cursors/parched/pointer.png"; +import turtleDefaultCursorUrl from "../../../assets/cursors/turtle/default.png"; +import turtlePointerCursorUrl from "../../../assets/cursors/turtle/pointer.png"; import { type CursorStyle, type CursorTelemetryPoint, @@ -18,6 +28,10 @@ import { import { UPLOADED_CURSOR_SAMPLE_SIZE, uploadedCursorAssets } from "./uploadedCursorAssets"; type CursorAssetKey = NonNullable; +type StatefulCursorStyle = Extract; +type SingleCursorStyle = Extract; +type CursorPackStyle = Exclude; +type CursorPackVariant = "default" | "pointer"; type LoadedCursorAsset = { texture: Texture; @@ -27,6 +41,15 @@ type LoadedCursorAsset = { anchorY: number; }; +type LoadedCursorPackAssets = Record; + +type CursorPackSource = { + defaultUrl: string; + pointerUrl: string; + defaultAnchor: { x: number; y: number }; + pointerAnchor: { x: number; y: number }; +}; + /** * Configuration for cursor rendering. */ @@ -84,7 +107,9 @@ const CURSOR_SHADOW_PADDING = 12; let cursorAssetsPromise: Promise | null = null; let loadedCursorAssets: Partial> = {}; let loadedInvertedCursorAssets: Partial> = {}; -let loadedCursorStyleAssets: Partial, LoadedCursorAsset>> = {}; +let loadedCursorStyleAssets: Partial> = {}; +let loadedCursorPackAssets: Partial> = {}; +const warnedMissingCursorPackStyles = new Set(); const SUPPORTED_CURSOR_KEYS: CursorAssetKey[] = [ "arrow", "text", @@ -97,26 +122,56 @@ const SUPPORTED_CURSOR_KEYS: CursorAssetKey[] = [ "not-allowed", ]; -const CUSTOM_CURSOR_ARROW_WIDTH = 150; -const CUSTOM_CURSOR_ARROW_HEIGHT = 214; -const CUSTOM_CURSOR_ARROW_TIP_X = 14; -const CUSTOM_CURSOR_ARROW_TIP_Y = 12; +const DEFAULT_CURSOR_PACK_ANCHOR = { x: 0.08, y: 0.08 } as const; +const POINTER_CURSOR_PACK_ANCHOR = { x: 0.48, y: 0.1 } as const; +const CENTERED_CURSOR_PACK_ANCHOR = { x: 0.5, y: 0.5 } as const; +const CURSOR_PACK_POINTER_TYPES = new Set(["pointer", "open-hand", "closed-hand"]); +const CURSOR_PACK_SOURCES: Record = { + lavender: { + defaultUrl: lavenderDefaultCursorUrl, + pointerUrl: lavenderPointerCursorUrl, + defaultAnchor: DEFAULT_CURSOR_PACK_ANCHOR, + pointerAnchor: POINTER_CURSOR_PACK_ANCHOR, + }, + parched: { + defaultUrl: parchedDefaultCursorUrl, + pointerUrl: parchedPointerCursorUrl, + defaultAnchor: DEFAULT_CURSOR_PACK_ANCHOR, + pointerAnchor: POINTER_CURSOR_PACK_ANCHOR, + }, + chooper: { + defaultUrl: chooperDefaultCursorUrl, + pointerUrl: chooperPointerCursorUrl, + defaultAnchor: DEFAULT_CURSOR_PACK_ANCHOR, + pointerAnchor: POINTER_CURSOR_PACK_ANCHOR, + }, + amongus: { + defaultUrl: amongusDefaultCursorUrl, + pointerUrl: amongusPointerCursorUrl, + defaultAnchor: CENTERED_CURSOR_PACK_ANCHOR, + pointerAnchor: CENTERED_CURSOR_PACK_ANCHOR, + }, + turtle: { + defaultUrl: turtleDefaultCursorUrl, + pointerUrl: turtlePointerCursorUrl, + defaultAnchor: CENTERED_CURSOR_PACK_ANCHOR, + pointerAnchor: CENTERED_CURSOR_PACK_ANCHOR, + }, +}; -function drawArrowCursorPath(ctx: CanvasRenderingContext2D, width: number, height: number) { - ctx.beginPath(); - ctx.moveTo(width * 0.093, height * 0.056); - ctx.lineTo(width * 0.136, height * 0.78); - ctx.lineTo(width * 0.34, height * 0.618); - ctx.lineTo(width * 0.453, height * 0.967); - ctx.lineTo(width * 0.62, height * 0.906); - ctx.lineTo(width * 0.501, height * 0.57); - ctx.lineTo(width * 0.933, height * 0.57); - ctx.closePath(); +function isStatefulCursorStyle(style: CursorStyle): style is StatefulCursorStyle { + return style === "tahoe" || style === "mono"; } -async function createCursorStyleAsset( - style: Exclude, -): Promise { +function isSingleCursorStyle(style: CursorStyle): style is SingleCursorStyle { + return style === "dot" || style === "figma"; +} + +function resolveCursorPackVariant(cursorType: CursorAssetKey): CursorPackVariant { + return CURSOR_PACK_POINTER_TYPES.has(cursorType) ? "pointer" : "default"; +} + +async function createCursorStyleAsset(style: SingleCursorStyle): Promise { if (style === "figma") { const image = await loadImage(minimalCursorUrl); const sourceCanvas = document.createElement("canvas"); @@ -139,40 +194,19 @@ async function createCursorStyleAsset( } const canvas = document.createElement("canvas"); - let anchorX = 0.5; - let anchorY = 0.5; - - if (style === "dot") { - canvas.width = 112; - canvas.height = 112; - anchorX = 0.5; - anchorY = 0.5; - const ctx = canvas.getContext("2d")!; - const cx = canvas.width / 2; - const cy = canvas.height / 2; - const radius = 26; - ctx.fillStyle = "#ffffff"; - ctx.strokeStyle = "rgba(15, 23, 42, 0.88)"; - ctx.lineWidth = 10; - ctx.beginPath(); - ctx.arc(cx, cy, radius, 0, Math.PI * 2); - ctx.fill(); - ctx.stroke(); - } else { - canvas.width = CUSTOM_CURSOR_ARROW_WIDTH; - canvas.height = CUSTOM_CURSOR_ARROW_HEIGHT; - anchorX = CUSTOM_CURSOR_ARROW_TIP_X / CUSTOM_CURSOR_ARROW_WIDTH; - anchorY = CUSTOM_CURSOR_ARROW_TIP_Y / CUSTOM_CURSOR_ARROW_HEIGHT; - const ctx = canvas.getContext("2d")!; - ctx.fillStyle = "#ffffff"; - ctx.strokeStyle = "#111111"; - ctx.lineWidth = 11; - ctx.lineJoin = "round"; - ctx.lineCap = "round"; - drawArrowCursorPath(ctx, canvas.width, canvas.height); - ctx.fill(); - ctx.stroke(); - } + canvas.width = 112; + canvas.height = 112; + const ctx = canvas.getContext("2d")!; + const cx = canvas.width / 2; + const cy = canvas.height / 2; + const radius = 26; + ctx.fillStyle = "#ffffff"; + ctx.strokeStyle = "rgba(15, 23, 42, 0.88)"; + ctx.lineWidth = 10; + ctx.beginPath(); + ctx.arc(cx, cy, radius, 0, Math.PI * 2); + ctx.fill(); + ctx.stroke(); const dataUrl = canvas.toDataURL("image/png"); await Assets.load(dataUrl); @@ -183,8 +217,25 @@ async function createCursorStyleAsset( texture, image, aspectRatio: canvas.height > 0 ? canvas.width / canvas.height : 1, - anchorX, - anchorY, + anchorX: 0.5, + anchorY: 0.5, + }; +} + +async function createCursorPackAsset( + url: string, + anchor: { x: number; y: number }, +): Promise { + await Assets.load(url); + const image = await loadImage(url); + const texture = Texture.from(url); + + return { + texture, + image, + aspectRatio: image.naturalHeight > 0 ? image.naturalWidth / image.naturalHeight : 1, + anchorX: clamp(anchor.x, 0, 1), + anchorY: clamp(anchor.y, 0, 1), }; } @@ -373,7 +424,7 @@ function getAvailableCursorKeys(): CursorAssetKey[] { return loadedKeys.length > 0 ? loadedKeys : ["arrow"]; } -function getCursorStyleAsset(style: Exclude) { +function getCursorStyleAsset(style: SingleCursorStyle) { const asset = loadedCursorStyleAssets[style]; if (!asset) { throw new Error(`Missing cursor style asset for ${style}`); @@ -382,10 +433,23 @@ function getCursorStyleAsset(style: Exclude) { return asset; } -function getStatefulCursorAsset( - style: Extract, - key: CursorAssetKey, -) { +function getCursorPackStyleAsset(style: CursorPackStyle, key: CursorAssetKey) { + const styleAssets = loadedCursorPackAssets[style]; + if (!styleAssets) { + if (!warnedMissingCursorPackStyles.has(style)) { + warnedMissingCursorPackStyles.add(style); + console.warn( + `[CursorRenderer] Missing cursor pack assets for ${style}; falling back to Tahoe cursors.`, + ); + } + return getStatefulCursorAsset("tahoe", key); + } + + const variant = resolveCursorPackVariant(key); + return styleAssets[variant] ?? styleAssets.default; +} + +function getStatefulCursorAsset(style: StatefulCursorStyle, key: CursorAssetKey) { const assetMap = style === "mono" ? loadedInvertedCursorAssets : loadedCursorAssets; const asset = assetMap[key] ?? assetMap.arrow; if (!asset) { @@ -499,9 +563,33 @@ export async function preloadCursorAssets() { ); loadedCursorStyleAssets = Object.fromEntries(customStyleEntries) as Partial< - Record, LoadedCursorAsset> + Record >; + const cursorPackEntries = await Promise.all( + (Object.entries(CURSOR_PACK_SOURCES) as Array<[CursorPackStyle, CursorPackSource]>).map( + async ([style, source]) => { + try { + const [defaultAsset, pointerAsset] = await Promise.all([ + createCursorPackAsset(source.defaultUrl, source.defaultAnchor), + createCursorPackAsset(source.pointerUrl, source.pointerAnchor), + ]); + return [style, { default: defaultAsset, pointer: pointerAsset }] as const; + } catch (error) { + console.warn( + `[CursorRenderer] Failed to load cursor pack style for: ${style}`, + error, + ); + return null; + } + }, + ), + ); + + loadedCursorPackAssets = Object.fromEntries( + cursorPackEntries.filter(Boolean).map((entry) => entry!), + ) as Partial>; + if (!loadedCursorAssets.arrow) { throw new Error("Failed to initialize the fallback arrow cursor asset"); } @@ -886,7 +974,7 @@ export class PixiCursorOverlay { setStyle(style: CursorStyle) { this.config.style = style; - if (style === "tahoe" || style === "mono") { + if (isStatefulCursorStyle(style)) { for (const key of getAvailableCursorKeys()) { const asset = getStatefulCursorAsset(style, key); const shadowSprite = this.cursorShadowSprites[key]; @@ -903,7 +991,9 @@ export class PixiCursorOverlay { return; } - const asset = getCursorStyleAsset(style); + const asset = isSingleCursorStyle(style) + ? getCursorStyleAsset(style) + : getCursorPackStyleAsset(style, "arrow"); this.customCursorShadowSprite.texture = asset.texture; this.customCursorShadowSprite.anchor.set(asset.anchorX, asset.anchorY); this.customCursorSprite.texture = asset.texture; @@ -978,11 +1068,12 @@ export class PixiCursorOverlay { this.clickRingGraphics.clear(); drawClickRing(this.clickRingGraphics, px, py, h, clickProgress); - if (this.config.style === "tahoe" || this.config.style === "mono") { + const spriteKey = (cursorType in this.cursorSprites ? cursorType : "arrow") as CursorAssetKey; + + if (isStatefulCursorStyle(this.config.style)) { this.customCursorShadowSprite.visible = false; this.customCursorSprite.visible = false; - const spriteKey = (cursorType in this.cursorSprites ? cursorType : "arrow") as CursorAssetKey; const asset = getStatefulCursorAsset(this.config.style, spriteKey); const shadowSprite = this.cursorShadowSprites[spriteKey] ?? this.cursorShadowSprites.arrow!; const sprite = this.cursorSprites[spriteKey] ?? this.cursorSprites.arrow!; @@ -1022,8 +1113,12 @@ export class PixiCursorOverlay { currentSprite.visible = false; } - const asset = getCursorStyleAsset(this.config.style); + const asset = isSingleCursorStyle(this.config.style) + ? getCursorStyleAsset(this.config.style) + : getCursorPackStyleAsset(this.config.style, spriteKey); const showSeparateShadow = this.config.style !== "figma"; + this.customCursorShadowSprite.texture = asset.texture; + this.customCursorShadowSprite.anchor.set(asset.anchorX, asset.anchorY); this.customCursorShadowSprite.visible = showSeparateShadow; if (showSeparateShadow) { this.customCursorShadowSprite.height = scaledH * bounceScale; @@ -1035,6 +1130,8 @@ export class PixiCursorOverlay { this.customCursorShadowSprite.rotation = swayRotation; } + this.customCursorSprite.texture = asset.texture; + this.customCursorSprite.anchor.set(asset.anchorX, asset.anchorY); this.customCursorSprite.visible = true; this.customCursorSprite.alpha = this.config.dotAlpha; this.customCursorSprite.height = scaledH * bounceScale; @@ -1168,13 +1265,14 @@ export function drawCursorOnCanvas( timeMs, config.clickBounceDuration, ); - const asset = - config.style === "tahoe" || config.style === "mono" - ? getStatefulCursorAsset( - config.style, - (cursorType && loadedCursorAssets[cursorType] ? cursorType : "arrow") as CursorAssetKey, - ) - : getCursorStyleAsset(config.style); + const spriteKey = ( + cursorType && loadedCursorAssets[cursorType] ? cursorType : "arrow" + ) as CursorAssetKey; + const asset = isStatefulCursorStyle(config.style) + ? getStatefulCursorAsset(config.style, spriteKey) + : isSingleCursorStyle(config.style) + ? getCursorStyleAsset(config.style) + : getCursorPackStyleAsset(config.style, spriteKey); const bounceScale = Math.max( 0.72, 1 - Math.sin(clickBounceProgress * Math.PI) * (0.08 * config.clickBounce), diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index a47c41ab..af3b0b93 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -111,10 +111,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } - accumulatedPausedDurationMs.current += Math.max( - 0, - resumedAt - pauseStartedAtMs.current, - ); + accumulatedPausedDurationMs.current += Math.max(0, resumedAt - pauseStartedAtMs.current); pauseStartedAtMs.current = null; }, []); @@ -170,10 +167,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const selectMimeType = () => { const preferred = [ + "video/webm;codecs=av1", + "video/webm;codecs=h264", "video/webm;codecs=vp9", "video/webm;codecs=vp8", "video/webm", - "video/webm;codecs=av1", ]; return preferred.find((type) => MediaRecorder.isTypeSupported(type)) ?? "video/webm"; @@ -350,7 +348,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } - const duration = Math.max(0, getRecordingDurationMs(Date.now()) - webcamTimeOffsetMs.current); + const duration = Math.max( + 0, + getRecordingDurationMs(Date.now()) - webcamTimeOffsetMs.current, + ); const webcamBlob = new Blob(webcamChunks.current, { type: mimeType }); webcamChunks.current = []; const fixedBlob = await fixWebmDuration(webcamBlob, duration); @@ -410,13 +411,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (isNativeWindows) { const muxResult = await window.electronAPI.muxNativeWindowsRecording(); - if (!muxResult?.success || !muxResult.path) { - console.error("Failed to finalize native Windows recording:", muxResult?.error ?? muxResult?.message); - alert("Recording could not be finalized because the captured video file is invalid."); - return; - } - - finalPath = muxResult.path; + finalPath = muxResult?.path ?? result.path; } await finalizeRecordingSession(finalPath, webcamPath); @@ -533,8 +528,6 @@ export function useScreenRecorder(): UseScreenRecorderReturn { recordingSessionTimestamp.current = Date.now(); resetRecordingClock(recordingSessionTimestamp.current); - webcamStartTime.current = null; - webcamTimeOffsetMs.current = 0; await startWebcamRecorder(); const platform = await window.electronAPI.getPlatform(); @@ -610,10 +603,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const mainStartedAt = Date.now(); nativeScreenRecording.current = true; nativeWindowsRecording.current = useNativeWindowsCapture; + resetRecordingClock(mainStartedAt); webcamTimeOffsetMs.current = webcamStartTime.current === null ? 0 : webcamStartTime.current - mainStartedAt; - resetRecordingClock(mainStartedAt); setRecording(true); window.electronAPI?.setRecordingState(true); @@ -926,10 +919,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamRecorder.current.stop(); } webcamRecorder.current = null; - webcamStream.current?.getTracks().forEach((t) => t.stop()); - webcamStream.current = null; webcamStartTime.current = null; webcamTimeOffsetMs.current = 0; + webcamStream.current?.getTracks().forEach((t) => t.stop()); + webcamStream.current = null; pendingWebcamPathPromise.current = null; if (nativeScreenRecording.current) { diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 6784c422..ef44daa9 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -22,7 +22,12 @@ "tahoe": "Tahoe", "dot": "Dot", "figma": "Minimal", - "mono": "Inverted" + "mono": "Inverted", + "lavender": "Lavender", + "parched": "Parched", + "chooper": "Chooper", + "amongus": "Among Us", + "turtle": "Turtle" }, "backgroundBlur": "Background Blur", "zoomMotionBlur": "Zoom Motion Blur", diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index 19e141f3..8670a697 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -22,7 +22,12 @@ "tahoe": "Tahoe", "dot": "Punto", "figma": "Minimal", - "mono": "Invertido" + "mono": "Invertido", + "lavender": "Lavender", + "parched": "Parched", + "chooper": "Chooper", + "amongus": "Among Us", + "turtle": "Turtle" }, "backgroundBlur": "Desenfoque de fondo", "zoomMotionBlur": "Desenfoque de movimiento del zoom", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index ffde445f..acdaff15 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -22,7 +22,12 @@ "tahoe": "Tahoe", "dot": "圆点", "figma": "Minimal", - "mono": "反相" + "mono": "反相", + "lavender": "Lavender", + "parched": "Parched", + "chooper": "Chooper", + "amongus": "Among Us", + "turtle": "Turtle" }, "backgroundBlur": "背景模糊", "zoomMotionBlur": "缩放运动模糊", diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 3109c16b..a40c4740 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -7,6 +7,7 @@ const AUDIO_BITRATE = 128_000 const DECODE_BACKPRESSURE_LIMIT = 20 const ENCODE_BACKPRESSURE_LIMIT = 20 const MIN_SPEED_REGION_DELTA_MS = 0.0001 +const MP4_AUDIO_CODEC = 'mp4a.40.2' export class AudioProcessor { private cancelled = false @@ -74,6 +75,35 @@ export class AudioProcessor { return } + const audioStream = typeof readEndSec === 'number' + ? demuxer.read('audio', 0, readEndSec) + : demuxer.read('audio') + + await this.transcodeAudioStream( + audioStream as ReadableStream, + audioConfig, + muxer, + { + shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims), + transformAudioData: (data) => { + const timestampMs = data.timestamp / 1000 + const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims) + const adjustedTimestampUs = data.timestamp - trimOffsetMs * 1000 + return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs)) + }, + }, + ) + } + + private async transcodeAudioStream( + audioStream: ReadableStream, + audioConfig: AudioDecoderConfig, + muxer: VideoMuxer, + options: { + shouldSkipChunk?: (timestampMs: number) => boolean + transformAudioData?: (data: AudioData) => AudioData | null + } = {}, + ): Promise { const pendingFrames: AudioData[] = [] let decodeError: Error | null = null let encodeError: Error | null = null @@ -115,7 +145,7 @@ export class AudioProcessor { const sampleRate = audioConfig.sampleRate || 48_000 const channels = audioConfig.numberOfChannels || 2 const encodeConfig: AudioEncoderConfig = { - codec: 'opus', + codec: MP4_AUDIO_CODEC, sampleRate, numberOfChannels: channels, bitrate: AUDIO_BITRATE, @@ -123,7 +153,7 @@ export class AudioProcessor { const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig) if (!encodeSupport.supported) { - console.warn('[AudioProcessor] Opus encoding not supported, skipping audio') + console.warn('[AudioProcessor] AAC encoding not supported, skipping audio') return } @@ -154,12 +184,17 @@ export class AudioProcessor { return } - const timestampMs = data.timestamp / 1000 - const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims) - const adjustedTimestampUs = data.timestamp - trimOffsetMs * 1000 - const adjusted = this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs)) - data.close() - pendingFrames.push(adjusted) + const transformed = options.transformAudioData ? options.transformAudioData(data) : data + + if (transformed !== data) { + data.close() + } + + if (!transformed) { + return + } + + pendingFrames.push(transformed) }, error: (error: DOMException) => { decodeError = new Error(`[AudioProcessor] Decode error: ${error.message}`) @@ -167,10 +202,7 @@ export class AudioProcessor { }) decoder.configure(audioConfig) - const audioStream = typeof readEndSec === 'number' - ? demuxer.read('audio', 0, readEndSec) - : demuxer.read('audio') - const reader = (audioStream as ReadableStream).getReader() + const reader = audioStream.getReader() try { while (!this.cancelled) { @@ -180,7 +212,7 @@ export class AudioProcessor { if (done || !chunk) break const timestampMs = chunk.timestamp / 1000 - if (this.isInTrimRegion(timestampMs, sortedTrims)) continue + if (options.shouldSkipChunk?.(timestampMs)) continue decoder.decode(chunk) pumpEncodedFrames() @@ -434,7 +466,7 @@ export class AudioProcessor { return recordedBlob } - // Demuxes the rendered speed-adjusted blob and feeds encoded chunks into the MP4 muxer. + // Demuxes the rendered speed-adjusted blob, decodes it, and re-encodes it to AAC for MP4 output. private async muxRenderedAudioBlob(blob: Blob, muxer: VideoMuxer): Promise { if (this.cancelled) return @@ -445,27 +477,17 @@ export class AudioProcessor { try { await demuxer.load(file) const audioConfig = (await demuxer.getDecoderConfig('audio')) as AudioDecoderConfig - const reader = (demuxer.read('audio') as ReadableStream).getReader() - let isFirstChunk = true - - try { - while (!this.cancelled) { - const { done, value: chunk } = await reader.read() - if (done || !chunk) break - if (isFirstChunk) { - await muxer.addAudioChunk(chunk, { decoderConfig: audioConfig }) - isFirstChunk = false - } else { - await muxer.addAudioChunk(chunk) - } - } - } finally { - try { - await reader.cancel() - } catch { - // reader already closed - } + const codecCheck = await AudioDecoder.isConfigSupported(audioConfig) + if (!codecCheck.supported) { + console.warn('[AudioProcessor] Rendered audio codec not supported:', audioConfig.codec) + return } + + await this.transcodeAudioStream( + demuxer.read('audio') as ReadableStream, + audioConfig, + muxer, + ) } finally { try { demuxer.destroy() diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index be9d8ce3..716fd711 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -112,6 +112,19 @@ function createAnimationState(): AnimationState { }; } +function configureHighQuality2DContext( + context: CanvasRenderingContext2D | null, +): CanvasRenderingContext2D | null { + if (!context) { + return null; + } + + context.imageSmoothingEnabled = true; + context.imageSmoothingQuality = "high"; + + return context; +} + // Renders video frames with all effects (background, zoom, crop, blur, shadow) to an offscreen canvas for export. export class FrameRenderer { @@ -129,6 +142,8 @@ export class FrameRenderer { private shadowCtx: CanvasRenderingContext2D | null = null; private compositeCanvas: HTMLCanvasElement | null = null; private compositeCtx: CanvasRenderingContext2D | null = null; + private backgroundVideoElement: HTMLVideoElement | null = null; + private cleanupBackgroundSource: (() => void) | null = null; private config: FrameRenderConfig; private animationState: AnimationState; private motionBlurState: MotionBlurState; @@ -236,9 +251,11 @@ export class FrameRenderer { this.compositeCanvas = document.createElement("canvas"); this.compositeCanvas.width = this.config.width; this.compositeCanvas.height = this.config.height; - this.compositeCtx = this.compositeCanvas.getContext("2d", { - willReadFrequently: false, - }); + this.compositeCtx = configureHighQuality2DContext( + this.compositeCanvas.getContext("2d", { + willReadFrequently: false, + }), + ); if (!this.compositeCtx) { throw new Error("Failed to get 2D context for composite canvas"); @@ -249,9 +266,11 @@ export class FrameRenderer { this.shadowCanvas = document.createElement("canvas"); this.shadowCanvas.width = this.config.width; this.shadowCanvas.height = this.config.height; - this.shadowCtx = this.shadowCanvas.getContext("2d", { - willReadFrequently: false, - }); + this.shadowCtx = configureHighQuality2DContext( + this.shadowCanvas.getContext("2d", { + willReadFrequently: false, + }), + ); if (!this.shadowCtx) { throw new Error("Failed to get 2D context for shadow canvas"); @@ -276,7 +295,11 @@ export class FrameRenderer { const bgCanvas = document.createElement("canvas"); bgCanvas.width = this.config.width; bgCanvas.height = this.config.height; - const bgCtx = bgCanvas.getContext("2d")!; + const bgCtx = configureHighQuality2DContext(bgCanvas.getContext("2d")); + + if (!bgCtx) { + throw new Error("Failed to get 2D context for background canvas"); + } try { // Render background based on type @@ -842,8 +865,6 @@ export class FrameRenderer { const croppedVideoWidth = videoWidth * (cropEndX - cropStartX); const croppedVideoHeight = videoHeight * (cropEndY - cropStartY); - // Calculate scale to fit in viewport - // Padding is a percentage (0-100), where 50% ~ 0.8 scale const paddingScale = 1.0 - (padding / 100) * 0.4; const viewportWidth = width * paddingScale; const viewportHeight = height * paddingScale; @@ -867,7 +888,6 @@ export class FrameRenderer { this.videoContainer.position.set(0, 0); - // scale border radius by export/preview canvas ratio const previewWidth = this.config.previewWidth || 1920; const previewHeight = this.config.previewHeight || 1080; const canvasScaleFactor = Math.min( @@ -1024,6 +1044,8 @@ export class FrameRenderer { // Clear composite canvas ctx.clearRect(0, 0, w, h); + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = "high"; // Step 1: Draw background layer (with optional blur, not affected by zoom) if (this.backgroundSprite) { @@ -1052,6 +1074,8 @@ export class FrameRenderer { ) { const shadowCtx = this.shadowCtx; shadowCtx.clearRect(0, 0, w, h); + shadowCtx.imageSmoothingEnabled = true; + shadowCtx.imageSmoothingQuality = "high"; shadowCtx.save(); // Calculate shadow parameters based on intensity (0-1) @@ -1133,12 +1157,14 @@ export class FrameRenderer { bubbleCanvas.height = bubbleSize; } this.webcamBubbleCanvas = bubbleCanvas; - const bubbleCtx = this.webcamBubbleCtx ?? bubbleCanvas.getContext("2d"); + const bubbleCtx = this.webcamBubbleCtx ?? configureHighQuality2DContext(bubbleCanvas.getContext("2d")); if (!bubbleCtx) { return; } this.webcamBubbleCtx = bubbleCtx; bubbleCtx.clearRect(0, 0, bubbleCanvas.width, bubbleCanvas.height); + bubbleCtx.imageSmoothingEnabled = true; + bubbleCtx.imageSmoothingQuality = "high"; const canRefreshCache = hasLiveWebcamFrame && @@ -1165,7 +1191,9 @@ export class FrameRenderer { this.webcamFrameCacheCanvas = document.createElement("canvas"); this.webcamFrameCacheCanvas.width = liveFrameWidth; this.webcamFrameCacheCanvas.height = liveFrameHeight; - this.webcamFrameCacheCtx = this.webcamFrameCacheCanvas.getContext("2d"); + this.webcamFrameCacheCtx = configureHighQuality2DContext( + this.webcamFrameCacheCanvas.getContext("2d"), + ); } this.webcamFrameCacheCtx?.clearRect( @@ -1278,6 +1306,14 @@ export class FrameRenderer { this.shadowCtx = null; this.compositeCanvas = null; this.compositeCtx = null; + if (this.backgroundVideoElement) { + this.backgroundVideoElement.pause(); + this.backgroundVideoElement.src = ""; + this.backgroundVideoElement.load(); + this.backgroundVideoElement = null; + } + this.cleanupBackgroundSource?.(); + this.cleanupBackgroundSource = null; if (this.webcamVideoElement) { this.webcamVideoElement.pause(); this.webcamVideoElement.src = ""; diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.ts index a052fec0..ae942395 100644 --- a/src/lib/exporter/muxer.ts +++ b/src/lib/exporter/muxer.ts @@ -40,7 +40,7 @@ export class VideoMuxer { // Create audio source if needed if (this.hasAudio) { - this.audioSource = new EncodedAudioPacketSource('opus'); + this.audioSource = new EncodedAudioPacketSource('aac'); this.output.addAudioTrack(this.audioSource); }