From e6f36a4d14f506b6f4473623c5dde2477623aa1f Mon Sep 17 00:00:00 2001 From: tokenflow1m Date: Sat, 14 Mar 2026 21:29:23 +0900 Subject: [PATCH 1/2] Add native WGC screen capture for Windows Uses Windows.Graphics.Capture to record the screen without the OS cursor or the yellow capture border. Falls back to Electron capture on older Windows or when the helper exe isn't available. --- .gitignore | 5 +- electron/electron-env.d.ts | 2 + electron/ipc/handlers.ts | 347 ++++++++++++++++++ electron/native/wgc-capture/CMakeLists.txt | 26 ++ electron/native/wgc-capture/src/main.cpp | 196 ++++++++++ .../native/wgc-capture/src/mf_encoder.cpp | 200 ++++++++++ electron/native/wgc-capture/src/mf_encoder.h | 35 ++ .../native/wgc-capture/src/monitor_utils.cpp | 61 +++ .../native/wgc-capture/src/monitor_utils.h | 18 + .../native/wgc-capture/src/wgc_session.cpp | 196 ++++++++++ electron/native/wgc-capture/src/wgc_session.h | 59 +++ electron/preload.ts | 2 + package.json | 3 +- scripts/build-wgc-capture.mjs | 90 +++++ src/hooks/useScreenRecorder.ts | 140 ++++++- 15 files changed, 1369 insertions(+), 11 deletions(-) create mode 100644 electron/native/wgc-capture/CMakeLists.txt create mode 100644 electron/native/wgc-capture/src/main.cpp create mode 100644 electron/native/wgc-capture/src/mf_encoder.cpp create mode 100644 electron/native/wgc-capture/src/mf_encoder.h create mode 100644 electron/native/wgc-capture/src/monitor_utils.cpp create mode 100644 electron/native/wgc-capture/src/monitor_utils.h create mode 100644 electron/native/wgc-capture/src/wgc_session.cpp create mode 100644 electron/native/wgc-capture/src/wgc_session.h create mode 100644 scripts/build-wgc-capture.mjs diff --git a/.gitignore b/.gitignore index 422f77dc..43ddf77b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,7 @@ release/** *.kiro/ # npx electron-builder --mac --win .tmp/ -.history/ +.history/ + +# WGC native capture build artifacts +electron/native/wgc-capture/build/ diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 786fb75b..a4bf779f 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -78,6 +78,8 @@ interface Window { hudOverlayClose: () => void; setHasUnsavedChanges: (hasChanges: boolean) => void onRequestSaveBeforeClose: (callback: () => Promise) => () => void + isWgcAvailable: () => Promise<{ available: boolean }> + storeWgcAudio: (audioData: ArrayBuffer, type: 'system' | 'mic') => Promise<{ success: boolean; path?: string; error?: string }> /** Hide the OS cursor before browser capture starts. */ hideOsCursor: () => Promise<{ success: boolean }> } diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index d05196ac..690a1cf7 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -59,6 +59,13 @@ let nativeCaptureStopRequested = false let nativeCaptureMicrophonePath: string | null = null let nativeCursorMonitorProcess: ChildProcessWithoutNullStreams | null = null let nativeCursorMonitorOutputBuffer = '' +let wgcCaptureProcess: ChildProcessWithoutNullStreams | null = null +let wgcCaptureOutputBuffer = '' +let wgcCaptureTargetPath: string | null = null +let wgcScreenRecordingActive = false +let wgcCaptureStopRequested = false +let wgcSystemAudioPath: string | null = null +let wgcMicAudioPath: string | null = null let ffmpegScreenRecordingActive = false let ffmpegCaptureProcess: ChildProcessWithoutNullStreams | null = null let ffmpegCaptureOutputBuffer = '' @@ -678,6 +685,197 @@ async function buildFfmpegCaptureArgs(source: SelectedSource, outputPath: string throw new Error(`FFmpeg capture is not supported on ${process.platform}`) } +function getWgcCaptureExePath() { + return resolveUnpackedAppPath('electron', 'native', 'wgc-capture', 'build', 'Release', 'wgc-capture.exe') +} + +async function isWgcCaptureAvailable(): Promise { + if (process.platform !== 'win32') return false + + try { + await fs.access(getWgcCaptureExePath(), fsConstants.X_OK) + } catch { + return false + } + + // Windows 10 2004 (Build 19041) minimum for IsCursorCaptureEnabled + const os = await import('node:os') + const [major, , build] = os.release().split('.').map(Number) + return major >= 10 && build >= 19041 +} + +function waitForWgcCaptureStart(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup() + reject(new Error('Timed out waiting for WGC capture to start')) + }, 12000) + + const onStdout = (chunk: Buffer) => { + const text = chunk.toString() + if (text.includes('Recording started')) { + cleanup() + resolve() + } + } + + const onError = (error: Error) => { + cleanup() + reject(error) + } + + const onExit = (code: number | null) => { + cleanup() + reject(new Error(wgcCaptureOutputBuffer.trim() || `WGC capture exited before recording started (code ${code ?? 'unknown'})`)) + } + + const cleanup = () => { + clearTimeout(timer) + proc.stdout.off('data', onStdout) + proc.off('error', onError) + proc.off('exit', onExit) + } + + proc.stdout.on('data', onStdout) + proc.once('error', onError) + proc.once('exit', onExit) + }) +} + +function waitForWgcCaptureStop(proc: ChildProcessWithoutNullStreams) { + return new Promise((resolve, reject) => { + const onClose = (code: number | null) => { + cleanup() + const match = wgcCaptureOutputBuffer.match(/Recording stopped\. Output path: (.+)/) + if (match?.[1]) { + resolve(match[1].trim()) + return + } + if (code === 0 && wgcCaptureTargetPath) { + resolve(wgcCaptureTargetPath) + return + } + reject(new Error(wgcCaptureOutputBuffer.trim() || `WGC capture exited with code ${code ?? 'unknown'}`)) + } + + const onError = (error: Error) => { + cleanup() + reject(error) + } + + const cleanup = () => { + proc.off('close', onClose) + proc.off('error', onError) + } + + proc.once('close', onClose) + proc.once('error', onError) + }) +} + +function attachWgcCaptureLifecycle(proc: ChildProcessWithoutNullStreams) { + proc.once('close', () => { + const wasActive = wgcScreenRecordingActive + wgcCaptureProcess = null + + if (!wasActive || wgcCaptureStopRequested) { + return + } + + wgcScreenRecordingActive = false + wgcCaptureTargetPath = null + wgcCaptureStopRequested = false + + const sourceName = selectedSource?.name ?? 'Screen' + BrowserWindow.getAllWindows().forEach((window) => { + if (!window.isDestroyed()) { + window.webContents.send('recording-state-changed', { + recording: false, + sourceName, + }) + } + }) + + emitRecordingInterrupted('capture-stopped', 'Recording stopped unexpectedly.') + }) +} + +async function muxWgcVideoWithAudio(videoPath: string, systemAudioPath: string | null, micAudioPath: string | null) { + const ffmpegPath = getFfmpegBinaryPath() + const inputs: string[] = ['-i', videoPath] + const audioInputs: string[] = [] + + if (systemAudioPath) { + try { + await fs.access(systemAudioPath) + inputs.push('-i', systemAudioPath) + audioInputs.push('system') + } catch { + // system audio file not available + } + } + + if (micAudioPath) { + try { + await fs.access(micAudioPath) + inputs.push('-i', micAudioPath) + audioInputs.push('mic') + } catch { + // mic audio file not available + } + } + + if (audioInputs.length === 0) return + + const mixedOutputPath = `${videoPath}.muxed.mp4` + + if (audioInputs.length === 2) { + // Both system + mic audio: mix them + await execFileAsync( + ffmpegPath, + [ + '-y', + ...inputs, + '-filter_complex', '[1:a][2:a]amix=inputs=2:duration=longest:normalize=0[aout]', + '-map', '0:v:0', + '-map', '[aout]', + '-c:v', 'copy', + '-c:a', 'aac', + '-b:a', '192k', + '-shortest', + mixedOutputPath, + ], + { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, + ) + } else { + // Single audio track + await execFileAsync( + ffmpegPath, + [ + '-y', + ...inputs, + '-map', '0:v:0', + '-map', '1:a:0', + '-c:v', 'copy', + '-c:a', 'aac', + '-b:a', '192k', + '-shortest', + mixedOutputPath, + ], + { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, + ) + } + + await moveFileWithOverwrite(mixedOutputPath, videoPath) + + // Clean up audio files + for (const audioPath of [systemAudioPath, micAudioPath]) { + if (audioPath) { + await fs.rm(audioPath, { force: true }).catch(() => {}) + } + } +} + function waitForNativeCaptureStart(process: ChildProcessWithoutNullStreams) { return new Promise((resolve, reject) => { const timer = setTimeout(() => { @@ -1492,6 +1690,74 @@ export function registerIpcHandlers( }) ipcMain.handle('start-native-screen-recording', async (_, source: SelectedSource, options?: NativeMacRecordingOptions) => { + // Windows WGC path + if (process.platform === 'win32') { + const wgcAvailable = await isWgcCaptureAvailable() + if (!wgcAvailable) { + return { success: false, message: 'WGC capture is not available on this system.' } + } + + if (wgcCaptureProcess && !wgcScreenRecordingActive) { + try { wgcCaptureProcess.kill() } catch { /* ignore */ } + wgcCaptureProcess = null + wgcCaptureTargetPath = null + wgcCaptureStopRequested = false + } + + if (wgcCaptureProcess) { + return { success: false, message: 'A WGC screen recording is already active.' } + } + + try { + const exePath = getWgcCaptureExePath() + const outputPath = path.join(RECORDINGS_DIR, `recording-${Date.now()}.mp4`) + const screenId = Number(source?.display_id) + const displayId = Number.isFinite(screenId) && screenId > 0 + ? screenId + : Number(getScreen().getPrimaryDisplay().id) + + const config: Record = { + displayId, + outputPath, + fps: 60, + } + + wgcCaptureOutputBuffer = '' + wgcCaptureTargetPath = outputPath + wgcCaptureStopRequested = false + wgcCaptureProcess = spawn(exePath, [JSON.stringify(config)], { + cwd: RECORDINGS_DIR, + stdio: ['pipe', 'pipe', 'pipe'], + }) + attachWgcCaptureLifecycle(wgcCaptureProcess) + + wgcCaptureProcess.stdout.on('data', (chunk: Buffer) => { + wgcCaptureOutputBuffer += chunk.toString() + }) + wgcCaptureProcess.stderr.on('data', (chunk: Buffer) => { + wgcCaptureOutputBuffer += chunk.toString() + }) + + await waitForWgcCaptureStart(wgcCaptureProcess) + wgcScreenRecordingActive = true + nativeScreenRecordingActive = true + return { success: true } + } catch (error) { + console.error('Failed to start WGC capture:', error) + try { wgcCaptureProcess?.kill() } catch { /* ignore */ } + wgcScreenRecordingActive = false + nativeScreenRecordingActive = false + wgcCaptureProcess = null + wgcCaptureTargetPath = null + wgcCaptureStopRequested = false + return { + success: false, + message: 'Failed to start WGC capture', + error: String(error), + } + } + } + if (process.platform !== 'darwin') { return { success: false, message: 'Native screen recording is only available on macOS.' } } @@ -1599,6 +1865,68 @@ export function registerIpcHandlers( }) ipcMain.handle('stop-native-screen-recording', async () => { + // Windows WGC stop path + if (process.platform === 'win32' && wgcScreenRecordingActive) { + try { + if (!wgcCaptureProcess) { + throw new Error('WGC capture process is not running') + } + + const proc = wgcCaptureProcess + const preferredVideoPath = wgcCaptureTargetPath + wgcCaptureStopRequested = true + proc.stdin.write('stop\n') + const tempVideoPath = await waitForWgcCaptureStop(proc) + wgcCaptureProcess = null + wgcScreenRecordingActive = false + nativeScreenRecordingActive = false + wgcCaptureTargetPath = null + wgcCaptureStopRequested = false + + const finalVideoPath = preferredVideoPath ?? tempVideoPath + if (tempVideoPath !== finalVideoPath) { + await moveFileWithOverwrite(tempVideoPath, finalVideoPath) + } + + if (wgcSystemAudioPath || wgcMicAudioPath) { + try { + await muxWgcVideoWithAudio(finalVideoPath, wgcSystemAudioPath, wgcMicAudioPath) + } catch (muxError) { + console.warn('Failed to mux WGC audio:', muxError) + } + wgcSystemAudioPath = null + wgcMicAudioPath = null + } + + return await finalizeStoredVideo(finalVideoPath) + } catch (error) { + console.error('Failed to stop WGC capture:', error) + const fallbackPath = wgcCaptureTargetPath + wgcScreenRecordingActive = false + nativeScreenRecordingActive = false + wgcCaptureProcess = null + wgcCaptureTargetPath = null + wgcCaptureStopRequested = false + wgcSystemAudioPath = null + wgcMicAudioPath = null + + if (fallbackPath) { + try { + await fs.access(fallbackPath) + return await finalizeStoredVideo(fallbackPath) + } catch { + // File doesn't exist + } + } + + return { + success: false, + message: 'Failed to stop WGC capture', + error: String(error), + } + } + } + if (process.platform !== 'darwin') { return { success: false, message: 'Native screen recording is only available on macOS.' } } @@ -1676,6 +2004,25 @@ export function registerIpcHandlers( } }) + ipcMain.handle('is-wgc-available', async () => { + return { available: await isWgcCaptureAvailable() } + }) + + ipcMain.handle('store-wgc-audio', async (_, audioData: ArrayBuffer, type: 'system' | 'mic') => { + try { + const audioPath = path.join(RECORDINGS_DIR, `recording-${Date.now()}.${type}.webm`) + await fs.writeFile(audioPath, Buffer.from(audioData)) + if (type === 'system') { + wgcSystemAudioPath = audioPath + } else { + wgcMicAudioPath = audioPath + } + return { success: true, path: audioPath } + } catch (error) { + return { success: false, error: String(error) } + } + }) + ipcMain.handle('start-ffmpeg-recording', async (_, source: SelectedSource) => { if (ffmpegCaptureProcess) { return { success: false, message: 'An FFmpeg recording is already active.' } diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt new file mode 100644 index 00000000..418fbb8f --- /dev/null +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.20) +project(wgc-capture LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +add_executable(wgc-capture + src/main.cpp + src/wgc_session.cpp + src/mf_encoder.cpp + src/monitor_utils.cpp +) + +target_compile_options(wgc-capture PRIVATE /EHsc /W3 /utf-8) + +target_link_libraries(wgc-capture PRIVATE + windowsapp + d3d11 + dxgi + mfplat + mfreadwrite + mf + mfuuid + ole32 + shcore +) diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp new file mode 100644 index 00000000..ac2063e4 --- /dev/null +++ b/electron/native/wgc-capture/src/main.cpp @@ -0,0 +1,196 @@ +#include "wgc_session.h" +#include "mf_encoder.h" +#include "monitor_utils.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +static std::atomic g_stopRequested{false}; +static std::mutex g_stopMutex; +static std::condition_variable g_stopCv; + +struct CaptureConfig { + int displayId = 0; + std::string outputPath; + int fps = 60; + int width = 0; + int height = 0; +}; + +static bool parseSimpleJson(const std::string& json, CaptureConfig& config) { + auto findInt = [&](const std::string& key) -> int { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return -1; + pos = json.find(':', pos); + if (pos == std::string::npos) return -1; + pos++; + while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) pos++; + try { + return std::stoi(json.substr(pos)); + } catch (...) { + return -1; + } + }; + + auto findString = [&](const std::string& key) -> std::string { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return ""; + pos = json.find(':', pos); + if (pos == std::string::npos) return ""; + pos++; + while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) pos++; + if (pos >= json.size() || json[pos] != '"') return ""; + pos++; + std::string result; + while (pos < json.size() && json[pos] != '"') { + if (json[pos] == '\\' && pos + 1 < json.size()) { + pos++; + if (json[pos] == 'n') result += '\n'; + else if (json[pos] == 't') result += '\t'; + else if (json[pos] == '\\') result += '\\'; + else if (json[pos] == '"') result += '"'; + else if (json[pos] == '/') result += '/'; + else result += json[pos]; + } else { + result += json[pos]; + } + pos++; + } + return result; + }; + + config.outputPath = findString("outputPath"); + if (config.outputPath.empty()) return false; + + int displayId = findInt("displayId"); + if (displayId >= 0) config.displayId = displayId; + + int fps = findInt("fps"); + if (fps > 0) config.fps = fps; + + int width = findInt("width"); + if (width > 0) config.width = width; + + int height = findInt("height"); + if (height > 0) config.height = height; + + return true; +} + +static std::wstring utf8ToWide(const std::string& str) { + if (str.empty()) return L""; + int len = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), static_cast(str.size()), nullptr, 0); + std::wstring wstr(len, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, str.c_str(), static_cast(str.size()), &wstr[0], len); + return wstr; +} + +static void stdinListenerThread() { + std::string line; + while (std::getline(std::cin, line)) { + // Trim whitespace + while (!line.empty() && (line.back() == '\r' || line.back() == '\n' || line.back() == ' ')) { + line.pop_back(); + } + + if (line == "stop") { + g_stopRequested = true; + g_stopCv.notify_all(); + return; + } + } + + // stdin closed (parent process died) + g_stopRequested = true; + g_stopCv.notify_all(); +} + +int main(int argc, char* argv[]) { + if (argc < 2) { + std::cerr << "ERROR: Missing JSON config argument" << std::endl; + return 1; + } + + winrt::init_apartment(winrt::apartment_type::multi_threaded); + + CaptureConfig config; + if (!parseSimpleJson(argv[1], config)) { + std::cerr << "ERROR: Failed to parse config JSON" << std::endl; + return 1; + } + + // Resolve monitor + HMONITOR monitor = findMonitorByDisplayId(config.displayId); + if (!monitor) { + std::cerr << "ERROR: Could not find monitor for displayId " << config.displayId << std::endl; + return 1; + } + + // Initialize WGC session + WgcSession session; + if (!session.initialize(monitor, config.fps)) { + std::cerr << "ERROR: Failed to initialize WGC capture session" << std::endl; + return 1; + } + + int captureWidth = config.width > 0 ? config.width : session.captureWidth(); + int captureHeight = config.height > 0 ? config.height : session.captureHeight(); + + // Ensure even dimensions for H.264 + captureWidth = (captureWidth / 2) * 2; + captureHeight = (captureHeight / 2) * 2; + + // Initialize encoder + MFEncoder encoder; + std::wstring outputPathW = utf8ToWide(config.outputPath); + if (!encoder.initialize(outputPathW, captureWidth, captureHeight, config.fps, + session.device(), session.context())) { + std::cerr << "ERROR: Failed to initialize Media Foundation encoder" << std::endl; + return 1; + } + + // Set up frame callback + std::atomic frameCount{0}; + session.setFrameCallback([&](ID3D11Texture2D* texture, int64_t timestampHns) { + if (g_stopRequested) return; + if (encoder.writeFrame(texture, timestampHns)) { + frameCount++; + } + }); + + // Start stdin listener + std::thread stdinThread(stdinListenerThread); + stdinThread.detach(); + + // Start capture + if (!session.startCapture()) { + std::cerr << "ERROR: Failed to start WGC capture" << std::endl; + return 1; + } + + std::cout << "Recording started" << std::endl; + std::cout.flush(); + + // Wait for stop signal + { + std::unique_lock lock(g_stopMutex); + g_stopCv.wait(lock, [] { return g_stopRequested.load(); }); + } + + // Stop capture and finalize + session.stopCapture(); + encoder.finalize(); + + std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl; + std::cout.flush(); + + // Fast exit to avoid WinRT/COM teardown crashes during apartment cleanup + ExitProcess(0); +} diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp new file mode 100644 index 00000000..e0b74707 --- /dev/null +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -0,0 +1,200 @@ +#include "mf_encoder.h" +#include +#include +#include +#include +#include + +#pragma comment(lib, "mfplat.lib") +#pragma comment(lib, "mfreadwrite.lib") +#pragma comment(lib, "mf.lib") +#pragma comment(lib, "mfuuid.lib") + +static int clampByte(int v) { + return v < 0 ? 0 : (v > 255 ? 255 : v); +} + +MFEncoder::MFEncoder() {} + +MFEncoder::~MFEncoder() { + finalize(); +} + +bool MFEncoder::initialize(const std::wstring& outputPath, int width, int height, int fps, + ID3D11Device* device, ID3D11DeviceContext* context) { + if (initialized_) return false; + + width_ = width; + height_ = height; + fps_ = fps; + device_ = device; + context_ = context; + + HRESULT hr = MFStartup(MF_VERSION); + if (FAILED(hr)) { + std::cerr << "ERROR: MFStartup failed: 0x" << std::hex << hr << std::endl; + return false; + } + + // Output media type (H.264) + ComPtr outputType; + hr = MFCreateMediaType(&outputType); + if (FAILED(hr)) return false; + + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + outputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); + outputType->SetUINT32(MF_MT_AVG_BITRATE, 20000000); + MFSetAttributeSize(outputType.Get(), MF_MT_FRAME_SIZE, width_, height_); + MFSetAttributeRatio(outputType.Get(), MF_MT_FRAME_RATE, fps_, 1); + MFSetAttributeRatio(outputType.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + outputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + + // Input media type (NV12) + ComPtr inputType; + hr = MFCreateMediaType(&inputType); + if (FAILED(hr)) return false; + + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); + MFSetAttributeSize(inputType.Get(), MF_MT_FRAME_SIZE, width_, height_); + MFSetAttributeRatio(inputType.Get(), MF_MT_FRAME_RATE, fps_, 1); + MFSetAttributeRatio(inputType.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + + // Create SinkWriter with MPEG4 container + ComPtr writerAttrs; + hr = MFCreateAttributes(&writerAttrs, 1); + if (FAILED(hr)) return false; + + writerAttrs->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE); + + hr = MFCreateSinkWriterFromURL(outputPath.c_str(), nullptr, writerAttrs.Get(), &sinkWriter_); + if (FAILED(hr)) { + std::cerr << "ERROR: MFCreateSinkWriterFromURL failed: 0x" << std::hex << hr << std::endl; + return false; + } + + hr = sinkWriter_->AddStream(outputType.Get(), &streamIndex_); + if (FAILED(hr)) { + std::cerr << "ERROR: AddStream failed: 0x" << std::hex << hr << std::endl; + return false; + } + + hr = sinkWriter_->SetInputMediaType(streamIndex_, inputType.Get(), nullptr); + if (FAILED(hr)) { + std::cerr << "ERROR: SetInputMediaType failed: 0x" << std::hex << hr << std::endl; + return false; + } + + hr = sinkWriter_->BeginWriting(); + if (FAILED(hr)) { + std::cerr << "ERROR: BeginWriting failed: 0x" << std::hex << hr << std::endl; + return false; + } + + // Pre-allocate staging texture + D3D11_TEXTURE2D_DESC stagingDesc = {}; + stagingDesc.Width = width_; + stagingDesc.Height = height_; + stagingDesc.MipLevels = 1; + stagingDesc.ArraySize = 1; + stagingDesc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + stagingDesc.SampleDesc.Count = 1; + stagingDesc.Usage = D3D11_USAGE_STAGING; + stagingDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + + hr = device_->CreateTexture2D(&stagingDesc, nullptr, &stagingTexture_); + if (FAILED(hr)) { + std::cerr << "ERROR: Failed to create staging texture: 0x" << std::hex << hr << std::endl; + return false; + } + + // Pre-allocate NV12 buffer + const int ySize = width_ * height_; + const int uvSize = (width_ / 2) * (height_ / 2) * 2; + nv12Buffer_.resize(ySize + uvSize); + + initialized_ = true; + return true; +} + +bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns) { + if (!initialized_ || !sinkWriter_) return false; + + context_->CopyResource(stagingTexture_.Get(), texture); + + D3D11_MAPPED_SUBRESOURCE mapped; + HRESULT hr = context_->Map(stagingTexture_.Get(), 0, D3D11_MAP_READ, 0, &mapped); + if (FAILED(hr)) return false; + + // Convert BGRA → NV12 + const uint8_t* bgra = static_cast(mapped.pData); + const int bgraPitch = static_cast(mapped.RowPitch); + + // Y plane + for (int y = 0; y < height_; y++) { + for (int x = 0; x < width_; x++) { + const uint8_t* pixel = bgra + y * bgraPitch + x * 4; + uint8_t b = pixel[0], g = pixel[1], r = pixel[2]; + int yVal = ((66 * r + 129 * g + 25 * b + 128) >> 8) + 16; + nv12Buffer_[y * width_ + x] = static_cast(clampByte(yVal)); + } + } + + // UV plane (interleaved, subsampled 2x2) + const int ySize = width_ * height_; + uint8_t* uvPlane = nv12Buffer_.data() + ySize; + for (int y = 0; y < height_; y += 2) { + for (int x = 0; x < width_; x += 2) { + const uint8_t* pixel = bgra + y * bgraPitch + x * 4; + uint8_t b = pixel[0], g = pixel[1], r = pixel[2]; + int u = ((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128; + int v = ((112 * r - 94 * g - 18 * b + 128) >> 8) + 128; + int uvIdx = (y / 2) * width_ + (x / 2) * 2; + uvPlane[uvIdx] = static_cast(clampByte(u)); + uvPlane[uvIdx + 1] = static_cast(clampByte(v)); + } + } + + context_->Unmap(stagingTexture_.Get(), 0); + + // Create MF sample + DWORD bufferSize = static_cast(nv12Buffer_.size()); + ComPtr buffer; + hr = MFCreateMemoryBuffer(bufferSize, &buffer); + if (FAILED(hr)) return false; + + BYTE* bufferData = nullptr; + hr = buffer->Lock(&bufferData, nullptr, nullptr); + if (FAILED(hr)) return false; + + std::memcpy(bufferData, nv12Buffer_.data(), bufferSize); + buffer->Unlock(); + buffer->SetCurrentLength(bufferSize); + + ComPtr sample; + hr = MFCreateSample(&sample); + if (FAILED(hr)) return false; + + sample->AddBuffer(buffer.Get()); + sample->SetSampleTime(timestampHns); + sample->SetSampleDuration(10000000LL / fps_); + + hr = sinkWriter_->WriteSample(streamIndex_, sample.Get()); + return SUCCEEDED(hr); +} + +bool MFEncoder::finalize() { + if (!initialized_) return false; + initialized_ = false; + + stagingTexture_.Reset(); + nv12Buffer_.clear(); + nv12Buffer_.shrink_to_fit(); + + if (!sinkWriter_) return false; + HRESULT hr = sinkWriter_->Finalize(); + sinkWriter_.Reset(); + MFShutdown(); + return SUCCEEDED(hr); +} diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h new file mode 100644 index 00000000..7eb8e642 --- /dev/null +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +using Microsoft::WRL::ComPtr; + +class MFEncoder { +public: + MFEncoder(); + ~MFEncoder(); + + bool initialize(const std::wstring& outputPath, int width, int height, int fps, + ID3D11Device* device, ID3D11DeviceContext* context); + bool writeFrame(ID3D11Texture2D* texture, int64_t timestampHns); + bool finalize(); + +private: + ComPtr sinkWriter_; + ID3D11Device* device_ = nullptr; + ID3D11DeviceContext* context_ = nullptr; + ComPtr stagingTexture_; + std::vector nv12Buffer_; + DWORD streamIndex_ = 0; + int width_ = 0; + int height_ = 0; + int fps_ = 60; + bool initialized_ = false; +}; diff --git a/electron/native/wgc-capture/src/monitor_utils.cpp b/electron/native/wgc-capture/src/monitor_utils.cpp new file mode 100644 index 00000000..25203ebf --- /dev/null +++ b/electron/native/wgc-capture/src/monitor_utils.cpp @@ -0,0 +1,61 @@ +#include "monitor_utils.h" +#include + +static BOOL CALLBACK enumMonitorCallback(HMONITOR hMonitor, HDC, LPRECT, LPARAM lParam) { + auto* monitors = reinterpret_cast*>(lParam); + + MONITORINFOEXW mi = {}; + mi.cbSize = sizeof(mi); + if (GetMonitorInfoW(hMonitor, &mi)) { + MonitorInfo info; + info.handle = hMonitor; + info.x = mi.rcMonitor.left; + info.y = mi.rcMonitor.top; + info.width = mi.rcMonitor.right - mi.rcMonitor.left; + info.height = mi.rcMonitor.bottom - mi.rcMonitor.top; + info.deviceName = mi.szDevice; + monitors->push_back(info); + } + + return TRUE; +} + +std::vector enumerateMonitors() { + std::vector monitors; + EnumDisplayMonitors(nullptr, nullptr, enumMonitorCallback, reinterpret_cast(&monitors)); + return monitors; +} + +// Electron uses the HMONITOR handle value cast to a number as the display ID. +HMONITOR findMonitorByDisplayId(int displayId) { + auto monitors = enumerateMonitors(); + + for (const auto& m : monitors) { + if (static_cast(reinterpret_cast(m.handle)) == displayId) { + return m.handle; + } + } + + if (!monitors.empty()) { + return monitors[0].handle; + } + + return MonitorFromPoint({0, 0}, MONITOR_DEFAULTTOPRIMARY); +} + +MonitorInfo getMonitorInfo(HMONITOR monitor) { + MonitorInfo info; + info.handle = monitor; + + MONITORINFOEXW mi = {}; + mi.cbSize = sizeof(mi); + if (GetMonitorInfoW(monitor, &mi)) { + info.x = mi.rcMonitor.left; + info.y = mi.rcMonitor.top; + info.width = mi.rcMonitor.right - mi.rcMonitor.left; + info.height = mi.rcMonitor.bottom - mi.rcMonitor.top; + info.deviceName = mi.szDevice; + } + + return info; +} diff --git a/electron/native/wgc-capture/src/monitor_utils.h b/electron/native/wgc-capture/src/monitor_utils.h new file mode 100644 index 00000000..513e3105 --- /dev/null +++ b/electron/native/wgc-capture/src/monitor_utils.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include +#include + +struct MonitorInfo { + HMONITOR handle; + int x; + int y; + int width; + int height; + std::wstring deviceName; +}; + +std::vector enumerateMonitors(); +HMONITOR findMonitorByDisplayId(int displayId); +MonitorInfo getMonitorInfo(HMONITOR monitor); diff --git a/electron/native/wgc-capture/src/wgc_session.cpp b/electron/native/wgc-capture/src/wgc_session.cpp new file mode 100644 index 00000000..b00ece5f --- /dev/null +++ b/electron/native/wgc-capture/src/wgc_session.cpp @@ -0,0 +1,196 @@ +#include "wgc_session.h" + +#include +#include +#include + +#include +#include + +#include +#include + +// IDirect3DDxgiInterfaceAccess is a COM interface for getting the DXGI interface +// from a WinRT IDirect3DSurface +MIDL_INTERFACE("A9B3D012-3DF2-4EE3-B8D1-8695F457D3C1") +IDirect3DDxgiInterfaceAccess : public IUnknown { + virtual HRESULT STDMETHODCALLTYPE GetInterface(REFIID iid, void** p) = 0; +}; + +// Convert ID3D11Device → IDirect3DDevice (WinRT interop) +extern "C" { + HRESULT __stdcall CreateDirect3D11DeviceFromDXGIDevice( + IDXGIDevice* dxgiDevice, + IInspectable** graphicsDevice); +} + +WgcSession::WgcSession() {} + +WgcSession::~WgcSession() { + stopCapture(); +} + +bool WgcSession::createD3DDevice() { + D3D_FEATURE_LEVEL featureLevels[] = { + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + + HRESULT hr = D3D11CreateDevice( + nullptr, + D3D_DRIVER_TYPE_HARDWARE, + nullptr, + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + featureLevels, + ARRAYSIZE(featureLevels), + D3D11_SDK_VERSION, + &d3dDevice_, + nullptr, + &d3dContext_); + + if (FAILED(hr)) { + std::cerr << "ERROR: D3D11CreateDevice failed: 0x" << std::hex << hr << std::endl; + return false; + } + + return true; +} + +winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice WgcSession::createWinRTDevice() { + ComPtr dxgiDevice; + HRESULT hr = d3dDevice_.As(&dxgiDevice); + if (FAILED(hr)) return nullptr; + + winrt::com_ptr inspectable; + hr = CreateDirect3D11DeviceFromDXGIDevice(dxgiDevice.Get(), inspectable.put()); + if (FAILED(hr)) return nullptr; + + return inspectable.as(); +} + +winrt::Windows::Graphics::Capture::GraphicsCaptureItem WgcSession::createCaptureItemForMonitor(HMONITOR monitor) { + auto factory = winrt::get_activation_factory< + winrt::Windows::Graphics::Capture::GraphicsCaptureItem>(); + + auto interop = factory.as(); + + winrt::Windows::Graphics::Capture::GraphicsCaptureItem item{nullptr}; + HRESULT hr = interop->CreateForMonitor( + monitor, + winrt::guid_of(), + winrt::put_abi(item)); + + if (FAILED(hr)) { + std::cerr << "ERROR: CreateForMonitor failed: 0x" << std::hex << hr << std::endl; + return nullptr; + } + + return item; +} + +bool WgcSession::initialize(HMONITOR monitor, int fps) { + fps_ = fps; + frameIntervalHns_ = 10000000LL / fps_; + + if (!createD3DDevice()) return false; + + winrtDevice_ = createWinRTDevice(); + if (!winrtDevice_) { + std::cerr << "ERROR: Failed to create WinRT D3D device" << std::endl; + return false; + } + + captureItem_ = createCaptureItemForMonitor(monitor); + if (!captureItem_) return false; + + auto size = captureItem_.Size(); + captureWidth_ = size.Width; + captureHeight_ = size.Height; + + framePool_ = winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool::CreateFreeThreaded( + winrtDevice_, + winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized, + 2, + size); + + session_ = framePool_.CreateCaptureSession(captureItem_); + + session_.IsCursorCaptureEnabled(false); + session_.IsBorderRequired(false); + + return true; +} + +void WgcSession::setFrameCallback(FrameCallback callback) { + frameCallback_ = std::move(callback); +} + +bool WgcSession::startCapture() { + if (!session_ || !framePool_) return false; + + capturing_ = true; + lastFrameTimeHns_ = 0; + + frameArrivedRevoker_ = framePool_.FrameArrived( + winrt::auto_revoke, + [this](auto const& sender, auto const& args) { + onFrameArrived(sender, args); + }); + + session_.StartCapture(); + return true; +} + +void WgcSession::stopCapture() { + capturing_ = false; + + frameArrivedRevoker_.revoke(); + + if (session_) { + session_.Close(); + session_ = nullptr; + } + if (framePool_) { + framePool_.Close(); + framePool_ = nullptr; + } +} + +void WgcSession::onFrameArrived( + winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& sender, + winrt::Windows::Foundation::IInspectable const&) { + + if (!capturing_) return; + + auto frame = sender.TryGetNextFrame(); + if (!frame) return; + + auto timestamp = frame.SystemRelativeTime(); + int64_t frameTimeHns = std::chrono::duration_cast>>(timestamp).count(); + + // Frame rate limiting: skip frames that arrive too soon + if (lastFrameTimeHns_ > 0 && (frameTimeHns - lastFrameTimeHns_) < (frameIntervalHns_ * 7 / 10)) { + frame.Close(); + return; + } + lastFrameTimeHns_ = frameTimeHns; + + auto surface = frame.Surface(); + + // Get the underlying D3D texture from the WinRT surface via COM interop + winrt::com_ptr access; + try { + access = surface.as(); + } catch (...) { + frame.Close(); + return; + } + ComPtr texture; + HRESULT hr = access->GetInterface(IID_PPV_ARGS(&texture)); + + if (SUCCEEDED(hr) && texture && frameCallback_) { + frameCallback_(texture.Get(), frameTimeHns); + } + + frame.Close(); +} diff --git a/electron/native/wgc-capture/src/wgc_session.h b/electron/native/wgc-capture/src/wgc_session.h new file mode 100644 index 00000000..094d95d3 --- /dev/null +++ b/electron/native/wgc-capture/src/wgc_session.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +using Microsoft::WRL::ComPtr; + +class WgcSession { +public: + using FrameCallback = std::function; + + WgcSession(); + ~WgcSession(); + + bool initialize(HMONITOR monitor, int fps); + void setFrameCallback(FrameCallback callback); + bool startCapture(); + void stopCapture(); + + int captureWidth() const { return captureWidth_; } + int captureHeight() const { return captureHeight_; } + ID3D11Device* device() const { return d3dDevice_.Get(); } + ID3D11DeviceContext* context() const { return d3dContext_.Get(); } + +private: + ComPtr d3dDevice_; + ComPtr d3dContext_; + winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice winrtDevice_{nullptr}; + winrt::Windows::Graphics::Capture::GraphicsCaptureItem captureItem_{nullptr}; + winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool framePool_{nullptr}; + winrt::Windows::Graphics::Capture::GraphicsCaptureSession session_{nullptr}; + winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool::FrameArrived_revoker frameArrivedRevoker_; + + FrameCallback frameCallback_; + std::atomic capturing_{false}; + int fps_ = 60; + int captureWidth_ = 0; + int captureHeight_ = 0; + int64_t frameIntervalHns_ = 0; + int64_t lastFrameTimeHns_ = 0; + + bool createD3DDevice(); + winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice createWinRTDevice(); + winrt::Windows::Graphics::Capture::GraphicsCaptureItem createCaptureItemForMonitor(HMONITOR monitor); + void onFrameArrived( + winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& sender, + winrt::Windows::Foundation::IInspectable const& args); +}; diff --git a/electron/preload.ts b/electron/preload.ts index fa84e195..e8729f3f 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -177,6 +177,8 @@ contextBridge.exposeInMainWorld('electronAPI', { ipcRenderer.on('request-save-before-close', listener) return () => ipcRenderer.removeListener('request-save-before-close', listener) }, + isWgcAvailable: () => ipcRenderer.invoke('is-wgc-available'), + storeWgcAudio: (audioData: ArrayBuffer, type: 'system' | 'mic') => ipcRenderer.invoke('store-wgc-audio', audioData, type), // Cursor visibility control for cursor-free browser capture fallback hideOsCursor: () => ipcRenderer.invoke('hide-cursor'), }) diff --git a/package.json b/package.json index e38f3062..247a89e6 100644 --- a/package.json +++ b/package.json @@ -24,8 +24,9 @@ "preview": "vite preview", "rebuild:native": "electron-rebuild --force --only uiohook-napi", "build:native-helpers": "node scripts/build-native-helpers.mjs", + "build:wgc-capture": "node scripts/build-wgc-capture.mjs", "build:mac": "npm run build:native-helpers && tsc && vite build && electron-builder --mac", - "build:win": "tsc && vite build && electron-builder --win", + "build:win": "npm run build:wgc-capture && tsc && vite build && electron-builder --win", "build:linux": "tsc && vite build && electron-builder --linux", "i18n:check": "node scripts/i18n-check.mjs", "test": "vitest --run", diff --git a/scripts/build-wgc-capture.mjs b/scripts/build-wgc-capture.mjs new file mode 100644 index 00000000..1c200506 --- /dev/null +++ b/scripts/build-wgc-capture.mjs @@ -0,0 +1,90 @@ +import { execSync } from 'node:child_process'; +import { mkdirSync, existsSync } from 'node:fs'; +import path from 'node:path'; + +const projectRoot = process.cwd(); +const sourceDir = path.join(projectRoot, 'electron', 'native', 'wgc-capture'); +const buildDir = path.join(sourceDir, 'build'); + +if (process.platform !== 'win32') { + console.log('[build-wgc-capture] Skipping: host platform is not Windows.'); + process.exit(0); +} + +if (!existsSync(path.join(sourceDir, 'CMakeLists.txt'))) { + console.error('[build-wgc-capture] CMakeLists.txt not found at', sourceDir); + process.exit(1); +} + +function findCmake() { + // Check PATH first + try { + execSync('cmake --version', { stdio: 'pipe' }); + return 'cmake'; + } catch { + // not on PATH + } + + // VS 2022 bundled CMake + const vsEditions = ['Community', 'Professional', 'Enterprise', 'BuildTools']; + for (const edition of vsEditions) { + const cmakePath = path.join( + 'C:', 'Program Files', 'Microsoft Visual Studio', '2022', edition, + 'Common7', 'IDE', 'CommonExtensions', 'Microsoft', 'CMake', 'CMake', 'bin', 'cmake.exe' + ); + if (existsSync(cmakePath)) { + return `"${cmakePath}"`; + } + } + + return null; +} + +const cmake = findCmake(); +if (!cmake) { + console.error('[build-wgc-capture] CMake not found. Install Visual Studio with C++ CMake tools or standalone CMake.'); + process.exit(1); +} + +mkdirSync(buildDir, { recursive: true }); + +console.log('[build-wgc-capture] Configuring CMake...'); +try { + execSync(`${cmake} .. -G "Visual Studio 17 2022" -A x64`, { + cwd: buildDir, + stdio: 'inherit', + timeout: 120000, + }); +} catch { + console.log('[build-wgc-capture] VS 2022 generator not found, trying VS 2019...'); + try { + execSync(`${cmake} .. -G "Visual Studio 16 2019" -A x64`, { + cwd: buildDir, + stdio: 'inherit', + timeout: 120000, + }); + } catch (innerError) { + console.error('[build-wgc-capture] CMake configure failed:', innerError.message); + process.exit(1); + } +} + +console.log('[build-wgc-capture] Building...'); +try { + execSync(`${cmake} --build . --config Release`, { + cwd: buildDir, + stdio: 'inherit', + timeout: 300000, + }); +} catch (error) { + console.error('[build-wgc-capture] Build failed:', error.message); + process.exit(1); +} + +const exePath = path.join(buildDir, 'Release', 'wgc-capture.exe'); +if (existsSync(exePath)) { + console.log(`[build-wgc-capture] Built successfully: ${exePath}`); +} else { + console.error('[build-wgc-capture] Expected exe not found at', exePath); + process.exit(1); +} diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 3269297c..7abc9561 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -54,6 +54,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const chunks = useRef([]); const startTime = useRef(0); const nativeScreenRecording = useRef(false); + const wgcAudioRecorders = useRef([]); + const wgcAudioChunks = useRef>(new Map()); const startInFlight = useRef(false); const hasPromptedForReselect = useRef(false); @@ -126,6 +128,95 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return Math.round(BITRATE_BASE * highFrameRateBoost); }; + const stopWgcAudioCapture = useCallback(async () => { + for (const recorder of wgcAudioRecorders.current) { + if (recorder.state === "recording") { + recorder.stop(); + } + } + + // Wait briefly for onstop callbacks to fire + await new Promise((resolve) => setTimeout(resolve, 200)); + + for (const [type, chunks] of wgcAudioChunks.current.entries()) { + if (chunks.length > 0) { + const blob = new Blob(chunks, { type: "audio/webm" }); + const buffer = await blob.arrayBuffer(); + try { + await window.electronAPI.storeWgcAudio(buffer, type as "system" | "mic"); + } catch (error) { + console.warn(`Failed to store WGC ${type} audio:`, error); + } + } + } + + wgcAudioRecorders.current = []; + wgcAudioChunks.current = new Map(); + }, []); + + const startWgcAudioCapture = async ( + source: { id?: string; display_id?: string }, + captureSystemAudio: boolean, + captureMicrophone: boolean, + micDeviceId?: string, + ) => { + wgcAudioRecorders.current = []; + wgcAudioChunks.current = new Map(); + + if (captureSystemAudio) { + try { + const systemAudioStream = await (navigator.mediaDevices as any).getUserMedia({ + audio: { + mandatory: { + chromeMediaSource: CHROME_MEDIA_SOURCE, + chromeMediaSourceId: source.id, + }, + }, + video: false, + }); + + const systemChunks: Blob[] = []; + wgcAudioChunks.current.set("system", systemChunks); + const recorder = new MediaRecorder(systemAudioStream, { + mimeType: "audio/webm", + audioBitsPerSecond: AUDIO_BITRATE_SYSTEM, + }); + recorder.ondataavailable = (e) => { + if (e.data.size > 0) systemChunks.push(e.data); + }; + recorder.start(RECORDER_TIMESLICE_MS); + wgcAudioRecorders.current.push(recorder); + } catch (error) { + console.warn("WGC: System audio capture failed:", error); + } + } + + if (captureMicrophone) { + try { + const micStream = await navigator.mediaDevices.getUserMedia({ + audio: micDeviceId + ? { deviceId: { exact: micDeviceId }, echoCancellation: true, noiseSuppression: true } + : { echoCancellation: true, noiseSuppression: true }, + video: false, + }); + + const micChunks: Blob[] = []; + wgcAudioChunks.current.set("mic", micChunks); + const recorder = new MediaRecorder(micStream, { + mimeType: "audio/webm", + audioBitsPerSecond: AUDIO_BITRATE_VOICE, + }); + recorder.ondataavailable = (e) => { + if (e.data.size > 0) micChunks.push(e.data); + }; + recorder.start(RECORDER_TIMESLICE_MS); + wgcAudioRecorders.current.push(recorder); + } catch (error) { + console.warn("WGC: Microphone capture failed:", error); + } + } + }; + const cleanupCapturedMedia = useCallback(() => { if (stream.current) { stream.current.getTracks().forEach((track) => track.stop()); @@ -154,6 +245,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setRecording(false); void (async () => { + // Stop WGC audio recorders first so audio data is stored before stop + if (wgcAudioRecorders.current.length > 0) { + await stopWgcAudioCapture(); + } + const result = await window.electronAPI.stopNativeScreenRecording(); window.electronAPI?.setRecordingState(false); @@ -256,23 +352,49 @@ export function useScreenRecorder(): UseScreenRecorderReturn { (selectedSource.id?.startsWith("screen:") || selectedSource.id?.startsWith("window:")) && typeof window.electronAPI.startNativeScreenRecording === "function"; - if (useNativeMacScreenCapture) { + let useWgcCapture = false; + if ( + platform === "win32" && + selectedSource.id?.startsWith("screen:") && + typeof window.electronAPI.isWgcAvailable === "function" + ) { + try { + const wgcResult = await window.electronAPI.isWgcAvailable(); + useWgcCapture = wgcResult.available; + } catch { + useWgcCapture = false; + } + } + + if (useNativeMacScreenCapture || useWgcCapture) { const nativeResult = await window.electronAPI.startNativeScreenRecording(selectedSource, { capturesSystemAudio: systemAudioEnabled, capturesMicrophone: microphoneEnabled, microphoneDeviceId, }); if (!nativeResult.success) { - throw new Error( - nativeResult.error ?? nativeResult.message ?? "Failed to start native screen recording", - ); + if (useWgcCapture) { + console.warn("WGC capture failed, falling back to browser capture:", nativeResult.error ?? nativeResult.message); + } else { + throw new Error( + nativeResult.error ?? nativeResult.message ?? "Failed to start native screen recording", + ); + } } - nativeScreenRecording.current = true; - startTime.current = Date.now(); - setRecording(true); - window.electronAPI?.setRecordingState(true); - return; + if (nativeResult.success) { + nativeScreenRecording.current = true; + startTime.current = Date.now(); + setRecording(true); + window.electronAPI?.setRecordingState(true); + + // WGC: start audio-only MediaRecorders in parallel + if (useWgcCapture && (systemAudioEnabled || microphoneEnabled)) { + void startWgcAudioCapture(selectedSource, systemAudioEnabled, microphoneEnabled, microphoneDeviceId); + } + + return; + } } const wantsAudioCapture = microphoneEnabled || systemAudioEnabled; From 004fe4d1973864343bc04ca7ad78ea4a5f0ff07b Mon Sep 17 00:00:00 2001 From: tokenflow1m Date: Sun, 15 Mar 2026 03:37:27 +0900 Subject: [PATCH 2/2] fix(wgc): address PR review feedback - Use getRecordingsDir() instead of hardcoded RECORDINGS_DIR - Fix stop order: video first, then audio save, then mux via new mux-wgc-recording IPC - Explicitly stop MediaStream tracks for audio resource cleanup - Add native WASAPI audio capture (system loopback + mic) in C++ process for sample-accurate sync; browser audio APIs crash with WGC - Add window capture support (CreateForWindow) with window: source prefix - Kill WGC capture process on app before-quit - Add even-dimension guard in MFEncoder - Fix ffmpeg asar path backslash handling on Windows - Remove dead browser-side WGC audio code (storeWgcAudio IPC, ScriptProcessorNode refs, stopWgcAudioCapture) --- electron/electron-env.d.ts | 3 +- electron/ipc/handlers.ts | 106 +++++--- electron/main.ts | 6 +- electron/native/wgc-capture/CMakeLists.txt | 1 + electron/native/wgc-capture/src/main.cpp | 114 ++++++++- .../native/wgc-capture/src/mf_encoder.cpp | 5 + .../wgc-capture/src/wasapi_loopback.cpp | 242 ++++++++++++++++++ .../native/wgc-capture/src/wasapi_loopback.h | 39 +++ .../native/wgc-capture/src/wgc_session.cpp | 58 ++++- electron/native/wgc-capture/src/wgc_session.h | 3 + electron/preload.ts | 3 +- src/hooks/useScreenRecorder.ts | 130 ++-------- src/vite-env.d.ts | 1 + 13 files changed, 555 insertions(+), 156 deletions(-) create mode 100644 electron/native/wgc-capture/src/wasapi_loopback.cpp create mode 100644 electron/native/wgc-capture/src/wasapi_loopback.h diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index a4bf779f..f859957d 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -35,6 +35,7 @@ interface Window { capturesSystemAudio?: boolean capturesMicrophone?: boolean microphoneDeviceId?: string + microphoneLabel?: string } ) => Promise<{ success: boolean; path?: string; message?: string; error?: string }> stopNativeScreenRecording: () => Promise<{ success: boolean; path?: string; message?: string; error?: string }> @@ -79,7 +80,7 @@ interface Window { setHasUnsavedChanges: (hasChanges: boolean) => void onRequestSaveBeforeClose: (callback: () => Promise) => () => void isWgcAvailable: () => Promise<{ available: boolean }> - storeWgcAudio: (audioData: ArrayBuffer, type: 'system' | 'mic') => Promise<{ success: boolean; path?: string; error?: string }> + muxWgcRecording: () => Promise<{ success: boolean; path?: string; message?: string; error?: string }> /** Hide the OS cursor before browser capture starts. */ hideOsCursor: () => Promise<{ success: boolean }> } diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 690a1cf7..20b8119a 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -39,6 +39,7 @@ type NativeMacRecordingOptions = { capturesSystemAudio?: boolean capturesMicrophone?: boolean microphoneDeviceId?: string + microphoneLabel?: string } type WindowBounds = { @@ -66,6 +67,7 @@ let wgcScreenRecordingActive = false let wgcCaptureStopRequested = false let wgcSystemAudioPath: string | null = null let wgcMicAudioPath: string | null = null +let wgcPendingVideoPath: string | null = null let ffmpegScreenRecordingActive = false let ffmpegCaptureProcess: ChildProcessWithoutNullStreams | null = null let ffmpegCaptureOutputBuffer = '' @@ -92,6 +94,20 @@ export function getSelectedSourceId(): string | null { return selectedSource?.id as string | null ?? null } +export function killWgcCaptureProcess() { + if (wgcCaptureProcess) { + try { wgcCaptureProcess.kill() } catch { /* ignore */ } + wgcCaptureProcess = null + wgcCaptureTargetPath = null + wgcScreenRecordingActive = false + nativeScreenRecordingActive = false + wgcCaptureStopRequested = false + wgcSystemAudioPath = null + wgcMicAudioPath = null + wgcPendingVideoPath = null + } +} + function normalizePath(filePath: string) { return path.resolve(filePath) } @@ -500,7 +516,7 @@ function getFfmpegBinaryPath() { } if (app.isPackaged) { - return ffmpegStatic.replace('.asar/', '.asar.unpacked/') + return ffmpegStatic.replace(/\.asar([\/\\])/, '.asar.unpacked$1') } return ffmpegStatic @@ -836,7 +852,7 @@ async function muxWgcVideoWithAudio(videoPath: string, systemAudioPath: string | [ '-y', ...inputs, - '-filter_complex', '[1:a][2:a]amix=inputs=2:duration=longest:normalize=0[aout]', + '-filter_complex', '[2:a]atrim=start=0.10,asetpts=PTS-STARTPTS[m];[1:a][m]amix=inputs=2:duration=longest:normalize=0[aout]', '-map', '0:v:0', '-map', '[aout]', '-c:v', 'copy', @@ -1710,23 +1726,47 @@ export function registerIpcHandlers( try { const exePath = getWgcCaptureExePath() - const outputPath = path.join(RECORDINGS_DIR, `recording-${Date.now()}.mp4`) - const screenId = Number(source?.display_id) - const displayId = Number.isFinite(screenId) && screenId > 0 - ? screenId - : Number(getScreen().getPrimaryDisplay().id) + const recordingsDir = await getRecordingsDir() + const timestamp = Date.now() + const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`) const config: Record = { - displayId, outputPath, fps: 60, } + if (options?.capturesSystemAudio) { + const audioPath = path.join(recordingsDir, `recording-${timestamp}.system.wav`) + config.captureSystemAudio = true + config.audioOutputPath = audioPath + wgcSystemAudioPath = audioPath + } + + if (options?.capturesMicrophone) { + const micPath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`) + config.captureMic = true + config.micOutputPath = micPath + if (options.microphoneLabel) { + config.micDeviceName = options.microphoneLabel + } + wgcMicAudioPath = micPath + } + + const windowId = parseWindowId(source?.id) + if (windowId && source?.id?.startsWith('window:')) { + config.windowHandle = windowId + } else { + const screenId = Number(source?.display_id) + config.displayId = Number.isFinite(screenId) && screenId > 0 + ? screenId + : Number(getScreen().getPrimaryDisplay().id) + } + wgcCaptureOutputBuffer = '' wgcCaptureTargetPath = outputPath wgcCaptureStopRequested = false wgcCaptureProcess = spawn(exePath, [JSON.stringify(config)], { - cwd: RECORDINGS_DIR, + cwd: recordingsDir, stdio: ['pipe', 'pipe', 'pipe'], }) attachWgcCaptureLifecycle(wgcCaptureProcess) @@ -1888,17 +1928,8 @@ export function registerIpcHandlers( await moveFileWithOverwrite(tempVideoPath, finalVideoPath) } - if (wgcSystemAudioPath || wgcMicAudioPath) { - try { - await muxWgcVideoWithAudio(finalVideoPath, wgcSystemAudioPath, wgcMicAudioPath) - } catch (muxError) { - console.warn('Failed to mux WGC audio:', muxError) - } - wgcSystemAudioPath = null - wgcMicAudioPath = null - } - - return await finalizeStoredVideo(finalVideoPath) + wgcPendingVideoPath = finalVideoPath + return { success: true, path: finalVideoPath } } catch (error) { console.error('Failed to stop WGC capture:', error) const fallbackPath = wgcCaptureTargetPath @@ -1909,11 +1940,13 @@ export function registerIpcHandlers( wgcCaptureStopRequested = false wgcSystemAudioPath = null wgcMicAudioPath = null + wgcPendingVideoPath = null if (fallbackPath) { try { await fs.access(fallbackPath) - return await finalizeStoredVideo(fallbackPath) + wgcPendingVideoPath = fallbackPath + return { success: true, path: fallbackPath } } catch { // File doesn't exist } @@ -2008,18 +2041,31 @@ export function registerIpcHandlers( return { available: await isWgcCaptureAvailable() } }) - ipcMain.handle('store-wgc-audio', async (_, audioData: ArrayBuffer, type: 'system' | 'mic') => { + ipcMain.handle('mux-wgc-recording', async () => { + const videoPath = wgcPendingVideoPath + wgcPendingVideoPath = null + + if (!videoPath) { + return { success: false, message: 'No WGC video pending for mux' } + } + try { - const audioPath = path.join(RECORDINGS_DIR, `recording-${Date.now()}.${type}.webm`) - await fs.writeFile(audioPath, Buffer.from(audioData)) - if (type === 'system') { - wgcSystemAudioPath = audioPath - } else { - wgcMicAudioPath = audioPath + if (wgcSystemAudioPath || wgcMicAudioPath) { + await muxWgcVideoWithAudio(videoPath, wgcSystemAudioPath, wgcMicAudioPath) + wgcSystemAudioPath = null + wgcMicAudioPath = null } - return { success: true, path: audioPath } + + return await finalizeStoredVideo(videoPath) } catch (error) { - return { success: false, error: String(error) } + console.error('Failed to mux WGC recording:', error) + wgcSystemAudioPath = null + wgcMicAudioPath = null + try { + return await finalizeStoredVideo(videoPath) + } catch { + return { success: false, message: 'Failed to mux WGC recording', error: String(error) } + } } }) diff --git a/electron/main.ts b/electron/main.ts index c7278878..d3ef0ed9 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -3,7 +3,7 @@ import { fileURLToPath } from 'node:url' import path from 'node:path' import fs from 'node:fs/promises' import { createHudOverlayWindow, createEditorWindow, createSourceSelectorWindow } from './windows' -import { registerIpcHandlers, getSelectedSourceId } from './ipc/handlers' +import { registerIpcHandlers, getSelectedSourceId, killWgcCaptureProcess } from './ipc/handlers' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -310,6 +310,10 @@ function createSourceSelectorWindowWrapper() { // On macOS, applications and their menu bar stay active until the user quits // explicitly with Cmd + Q. +app.on('before-quit', () => { + killWgcCaptureProcess() +}) + app.on('window-all-closed', () => { // Keep app running (macOS behavior) }) diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt index 418fbb8f..6b6c07ae 100644 --- a/electron/native/wgc-capture/CMakeLists.txt +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable(wgc-capture src/wgc_session.cpp src/mf_encoder.cpp src/monitor_utils.cpp + src/wasapi_loopback.cpp ) target_compile_options(wgc-capture PRIVATE /EHsc /W3 /utf-8) diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index ac2063e4..3f5da69b 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -1,6 +1,7 @@ #include "wgc_session.h" #include "mf_encoder.h" #include "monitor_utils.h" +#include "wasapi_loopback.h" #include #include @@ -18,10 +19,16 @@ static std::condition_variable g_stopCv; struct CaptureConfig { int displayId = 0; + int64_t windowHandle = 0; std::string outputPath; + std::string audioOutputPath; + std::string micOutputPath; + std::string micDeviceName; int fps = 60; int width = 0; int height = 0; + bool captureSystemAudio = false; + bool captureMic = false; }; static bool parseSimpleJson(const std::string& json, CaptureConfig& config) { @@ -39,6 +46,20 @@ static bool parseSimpleJson(const std::string& json, CaptureConfig& config) { } }; + auto findInt64 = [&](const std::string& key) -> int64_t { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return -1; + pos = json.find(':', pos); + if (pos == std::string::npos) return -1; + pos++; + while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) pos++; + try { + return std::stoll(json.substr(pos)); + } catch (...) { + return -1; + } + }; + auto findString = [&](const std::string& key) -> std::string { auto pos = json.find("\"" + key + "\""); if (pos == std::string::npos) return ""; @@ -72,6 +93,9 @@ static bool parseSimpleJson(const std::string& json, CaptureConfig& config) { int displayId = findInt("displayId"); if (displayId >= 0) config.displayId = displayId; + int64_t windowHandle = findInt64("windowHandle"); + if (windowHandle > 0) config.windowHandle = windowHandle; + int fps = findInt("fps"); if (fps > 0) config.fps = fps; @@ -81,6 +105,22 @@ static bool parseSimpleJson(const std::string& json, CaptureConfig& config) { int height = findInt("height"); if (height > 0) config.height = height; + config.audioOutputPath = findString("audioOutputPath"); + config.micOutputPath = findString("micOutputPath"); + config.micDeviceName = findString("micDeviceName"); + + auto findBool = [&](const std::string& key) -> bool { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) return false; + auto colonPos = json.find(':', pos); + if (colonPos == std::string::npos) return false; + auto valStart = json.find_first_not_of(" \t", colonPos + 1); + return valStart != std::string::npos && json.substr(valStart, 4) == "true"; + }; + + config.captureSystemAudio = findBool("captureSystemAudio"); + config.captureMic = findBool("captureMic"); + return true; } @@ -126,18 +166,28 @@ int main(int argc, char* argv[]) { return 1; } - // Resolve monitor - HMONITOR monitor = findMonitorByDisplayId(config.displayId); - if (!monitor) { - std::cerr << "ERROR: Could not find monitor for displayId " << config.displayId << std::endl; - return 1; - } - - // Initialize WGC session WgcSession session; - if (!session.initialize(monitor, config.fps)) { - std::cerr << "ERROR: Failed to initialize WGC capture session" << std::endl; - return 1; + + if (config.windowHandle > 0) { + HWND hwnd = reinterpret_cast(static_cast(config.windowHandle)); + if (!IsWindow(hwnd)) { + std::cerr << "ERROR: Invalid window handle " << config.windowHandle << std::endl; + return 1; + } + if (!session.initialize(hwnd, config.fps)) { + std::cerr << "ERROR: Failed to initialize WGC window capture session" << std::endl; + return 1; + } + } else { + HMONITOR monitor = findMonitorByDisplayId(config.displayId); + if (!monitor) { + std::cerr << "ERROR: Could not find monitor for displayId " << config.displayId << std::endl; + return 1; + } + if (!session.initialize(monitor, config.fps)) { + std::cerr << "ERROR: Failed to initialize WGC capture session" << std::endl; + return 1; + } } int captureWidth = config.width > 0 ? config.width : session.captureWidth(); @@ -169,12 +219,41 @@ int main(int argc, char* argv[]) { std::thread stdinThread(stdinListenerThread); stdinThread.detach(); - // Start capture + // Initialize WASAPI captures (but don't start yet) + WasapiCapture loopback; + WasapiCapture micCapture; + bool audioActive = false; + bool audioInitialized = false; + bool micActive = false; + bool micInitialized = false; + + if (config.captureSystemAudio && !config.audioOutputPath.empty()) { + audioInitialized = loopback.initializeLoopback(config.audioOutputPath); + if (!audioInitialized) { + std::cerr << "WARNING: Failed to initialize WASAPI loopback" << std::endl; + } + } + + if (config.captureMic && !config.micOutputPath.empty()) { + micInitialized = micCapture.initializeMic(config.micOutputPath, config.micDeviceName); + if (!micInitialized) { + std::cerr << "WARNING: Failed to initialize WASAPI mic capture" << std::endl; + } + } + + // Start video capture, then audio immediately after for sync if (!session.startCapture()) { std::cerr << "ERROR: Failed to start WGC capture" << std::endl; return 1; } + if (audioInitialized) { + audioActive = loopback.start(); + } + if (micInitialized) { + micActive = micCapture.start(); + } + std::cout << "Recording started" << std::endl; std::cout.flush(); @@ -186,11 +265,22 @@ int main(int argc, char* argv[]) { // Stop capture and finalize session.stopCapture(); + if (audioActive) loopback.stop(); + if (micActive) micCapture.stop(); encoder.finalize(); std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl; + if (audioActive) { + std::cout << "Audio path: " << config.audioOutputPath << std::endl; + } + if (micActive) { + std::cout << "Mic path: " << config.micOutputPath << std::endl; + } std::cout.flush(); + // Allow pipe buffers to drain before forceful exit + Sleep(100); + // Fast exit to avoid WinRT/COM teardown crashes during apartment cleanup ExitProcess(0); } diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp index e0b74707..a1474c20 100644 --- a/electron/native/wgc-capture/src/mf_encoder.cpp +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -24,6 +24,11 @@ bool MFEncoder::initialize(const std::wstring& outputPath, int width, int height ID3D11Device* device, ID3D11DeviceContext* context) { if (initialized_) return false; + if (width % 2 != 0 || height % 2 != 0) { + std::cerr << "ERROR: Encoder dimensions must be even, got " << width << "x" << height << std::endl; + return false; + } + width_ = width; height_ = height; fps_ = fps; diff --git a/electron/native/wgc-capture/src/wasapi_loopback.cpp b/electron/native/wgc-capture/src/wasapi_loopback.cpp new file mode 100644 index 00000000..af9b5bc3 --- /dev/null +++ b/electron/native/wgc-capture/src/wasapi_loopback.cpp @@ -0,0 +1,242 @@ +#include "wasapi_loopback.h" +#include +#include +#include + +#pragma comment(lib, "ole32.lib") + +static const CLSID CLSID_MMDeviceEnumerator_ = __uuidof(MMDeviceEnumerator); +static const IID IID_IMMDeviceEnumerator_ = __uuidof(IMMDeviceEnumerator); +static const IID IID_IAudioClient_ = __uuidof(IAudioClient); +static const IID IID_IAudioCaptureClient_ = __uuidof(IAudioCaptureClient); + +WasapiCapture::WasapiCapture() {} + +WasapiCapture::~WasapiCapture() { + stop(); + if (mixFormat_) CoTaskMemFree(mixFormat_); + if (captureClient_) captureClient_->Release(); + if (audioClient_) audioClient_->Release(); + if (device_) device_->Release(); + if (enumerator_) enumerator_->Release(); +} + +static std::wstring utf8ToWide(const std::string& str) { + if (str.empty()) return L""; + int len = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), static_cast(str.size()), nullptr, 0); + std::wstring wstr(len, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, str.c_str(), static_cast(str.size()), &wstr[0], len); + return wstr; +} + +IMMDevice* WasapiCapture::findCaptureDeviceByName(const std::wstring& targetName) { + IMMDeviceCollection* collection = nullptr; + HRESULT hr = enumerator_->EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE, &collection); + if (FAILED(hr)) return nullptr; + + UINT count = 0; + collection->GetCount(&count); + + for (UINT i = 0; i < count; i++) { + IMMDevice* dev = nullptr; + collection->Item(i, &dev); + + IPropertyStore* store = nullptr; + dev->OpenPropertyStore(STGM_READ, &store); + PROPVARIANT pv; + PropVariantInit(&pv); + store->GetValue(PKEY_Device_FriendlyName, &pv); + std::wstring name = pv.pwszVal ? pv.pwszVal : L""; + PropVariantClear(&pv); + store->Release(); + + if (name.find(targetName) != std::wstring::npos || targetName.find(name) != std::wstring::npos) { + collection->Release(); + return dev; + } + dev->Release(); + } + + collection->Release(); + return nullptr; +} + +bool WasapiCapture::initializeLoopback(const std::string& outputPath) { + outputPath_ = outputPath; + streamFlags_ = AUDCLNT_STREAMFLAGS_LOOPBACK; + + HRESULT hr = CoCreateInstance( + CLSID_MMDeviceEnumerator_, nullptr, CLSCTX_ALL, + IID_IMMDeviceEnumerator_, reinterpret_cast(&enumerator_)); + if (FAILED(hr)) return false; + + hr = enumerator_->GetDefaultAudioEndpoint(eRender, eConsole, &device_); + if (FAILED(hr)) return false; + + return initializeCommon(); +} + +bool WasapiCapture::initializeMic(const std::string& outputPath, const std::string& deviceName) { + outputPath_ = outputPath; + streamFlags_ = 0; + + HRESULT hr = CoCreateInstance( + CLSID_MMDeviceEnumerator_, nullptr, CLSCTX_ALL, + IID_IMMDeviceEnumerator_, reinterpret_cast(&enumerator_)); + if (FAILED(hr)) return false; + + if (!deviceName.empty()) { + device_ = findCaptureDeviceByName(utf8ToWide(deviceName)); + } + if (!device_) { + hr = enumerator_->GetDefaultAudioEndpoint(eCapture, eCommunications, &device_); + if (FAILED(hr)) { + hr = enumerator_->GetDefaultAudioEndpoint(eCapture, eConsole, &device_); + if (FAILED(hr)) return false; + } + } + + return initializeCommon(); +} + +bool WasapiCapture::initializeCommon() { + HRESULT hr = device_->Activate(IID_IAudioClient_, CLSCTX_ALL, nullptr, + reinterpret_cast(&audioClient_)); + if (FAILED(hr)) return false; + + hr = audioClient_->GetMixFormat(&mixFormat_); + if (FAILED(hr)) return false; + + REFERENCE_TIME bufferDuration = 200000; // 20ms + hr = audioClient_->Initialize( + AUDCLNT_SHAREMODE_SHARED, + streamFlags_, + bufferDuration, 0, mixFormat_, nullptr); + if (FAILED(hr)) return false; + + hr = audioClient_->GetBufferSize(&bufferFrameCount_); + if (FAILED(hr)) return false; + + hr = audioClient_->GetService(IID_IAudioCaptureClient_, + reinterpret_cast(&captureClient_)); + if (FAILED(hr)) return false; + + return true; +} + +bool WasapiCapture::start() { + HRESULT hr = audioClient_->Start(); + if (FAILED(hr)) return false; + + capturing_ = true; + thread_ = std::thread(&WasapiCapture::captureThread, this); + return true; +} + +void WasapiCapture::stop() { + if (!capturing_) return; + capturing_ = false; + if (thread_.joinable()) thread_.join(); + if (audioClient_) audioClient_->Stop(); +} + +static int16_t floatToInt16(float v) { + v = v < -1.0f ? -1.0f : (v > 1.0f ? 1.0f : v); + return static_cast(v * 32767.0f); +} + +bool WasapiCapture::writeWavHeader(HANDLE file, DWORD dataSize) { + WORD channels = static_cast(mixFormat_->nChannels); + DWORD sampleRate = mixFormat_->nSamplesPerSec; + WORD bitsPerSample = 16; + WORD blockAlign = channels * (bitsPerSample / 8); + DWORD byteRate = sampleRate * blockAlign; + + DWORD written; + WriteFile(file, "RIFF", 4, &written, nullptr); + DWORD chunkSize = 36 + dataSize; + WriteFile(file, &chunkSize, 4, &written, nullptr); + WriteFile(file, "WAVE", 4, &written, nullptr); + WriteFile(file, "fmt ", 4, &written, nullptr); + DWORD fmtSize = 16; + WriteFile(file, &fmtSize, 4, &written, nullptr); + WORD audioFormat = 1; + WriteFile(file, &audioFormat, 2, &written, nullptr); + WriteFile(file, &channels, 2, &written, nullptr); + WriteFile(file, &sampleRate, 4, &written, nullptr); + WriteFile(file, &byteRate, 4, &written, nullptr); + WriteFile(file, &blockAlign, 2, &written, nullptr); + WriteFile(file, &bitsPerSample, 2, &written, nullptr); + WriteFile(file, "data", 4, &written, nullptr); + WriteFile(file, &dataSize, 4, &written, nullptr); + return true; +} + +void WasapiCapture::captureThread() { + WORD channels = static_cast(mixFormat_->nChannels); + bool isFloat = (mixFormat_->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) || + (mixFormat_->wFormatTag == WAVE_FORMAT_EXTENSIBLE && + reinterpret_cast(mixFormat_)->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT); + + std::vector pcmBuffer; + HANDLE tmpFile = CreateFileA( + outputPath_.c_str(), GENERIC_WRITE, 0, nullptr, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + + if (tmpFile == INVALID_HANDLE_VALUE) { + std::cerr << "ERROR: Cannot create audio output file" << std::endl; + return; + } + + writeWavHeader(tmpFile, 0); + DWORD totalDataBytes = 0; + + DWORD sleepMs = static_cast((static_cast(bufferFrameCount_) / mixFormat_->nSamplesPerSec) * 500.0); + if (sleepMs < 5) sleepMs = 5; + + while (capturing_) { + Sleep(sleepMs); + + UINT32 packetLength = 0; + HRESULT hr = captureClient_->GetNextPacketSize(&packetLength); + if (FAILED(hr)) break; + + while (packetLength > 0) { + BYTE* data = nullptr; + UINT32 numFrames = 0; + DWORD flags = 0; + + hr = captureClient_->GetBuffer(&data, &numFrames, &flags, nullptr, nullptr); + if (FAILED(hr)) break; + + UINT32 totalSamples = numFrames * channels; + + if (flags & AUDCLNT_BUFFERFLAGS_SILENT) { + pcmBuffer.assign(totalSamples, 0); + } else if (isFloat) { + pcmBuffer.resize(totalSamples); + const float* src = reinterpret_cast(data); + for (UINT32 i = 0; i < totalSamples; i++) { + pcmBuffer[i] = floatToInt16(src[i]); + } + } else { + pcmBuffer.resize(totalSamples); + std::memcpy(pcmBuffer.data(), data, totalSamples * sizeof(int16_t)); + } + + captureClient_->ReleaseBuffer(numFrames); + + DWORD bytesToWrite = totalSamples * sizeof(int16_t); + DWORD written; + WriteFile(tmpFile, pcmBuffer.data(), bytesToWrite, &written, nullptr); + totalDataBytes += written; + + hr = captureClient_->GetNextPacketSize(&packetLength); + if (FAILED(hr)) break; + } + } + + SetFilePointer(tmpFile, 0, nullptr, FILE_BEGIN); + writeWavHeader(tmpFile, totalDataBytes); + CloseHandle(tmpFile); +} diff --git a/electron/native/wgc-capture/src/wasapi_loopback.h b/electron/native/wgc-capture/src/wasapi_loopback.h new file mode 100644 index 00000000..3b2b76df --- /dev/null +++ b/electron/native/wgc-capture/src/wasapi_loopback.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +class WasapiCapture { +public: + WasapiCapture(); + ~WasapiCapture(); + + bool initializeLoopback(const std::string& outputPath); + bool initializeMic(const std::string& outputPath, const std::string& deviceName = ""); + bool start(); + void stop(); + +private: + bool initializeCommon(); + void captureThread(); + bool writeWavHeader(HANDLE file, DWORD dataSize); + IMMDevice* findCaptureDeviceByName(const std::wstring& name); + + std::string outputPath_; + std::thread thread_; + std::atomic capturing_{false}; + + IMMDeviceEnumerator* enumerator_ = nullptr; + IMMDevice* device_ = nullptr; + IAudioClient* audioClient_ = nullptr; + IAudioCaptureClient* captureClient_ = nullptr; + WAVEFORMATEX* mixFormat_ = nullptr; + DWORD streamFlags_ = 0; + + UINT32 bufferFrameCount_ = 0; +}; diff --git a/electron/native/wgc-capture/src/wgc_session.cpp b/electron/native/wgc-capture/src/wgc_session.cpp index b00ece5f..7e59d140 100644 --- a/electron/native/wgc-capture/src/wgc_session.cpp +++ b/electron/native/wgc-capture/src/wgc_session.cpp @@ -88,19 +88,27 @@ winrt::Windows::Graphics::Capture::GraphicsCaptureItem WgcSession::createCapture return item; } -bool WgcSession::initialize(HMONITOR monitor, int fps) { - fps_ = fps; - frameIntervalHns_ = 10000000LL / fps_; +winrt::Windows::Graphics::Capture::GraphicsCaptureItem WgcSession::createCaptureItemForWindow(HWND hwnd) { + auto factory = winrt::get_activation_factory< + winrt::Windows::Graphics::Capture::GraphicsCaptureItem>(); - if (!createD3DDevice()) return false; + auto interop = factory.as(); - winrtDevice_ = createWinRTDevice(); - if (!winrtDevice_) { - std::cerr << "ERROR: Failed to create WinRT D3D device" << std::endl; - return false; + winrt::Windows::Graphics::Capture::GraphicsCaptureItem item{nullptr}; + HRESULT hr = interop->CreateForWindow( + hwnd, + winrt::guid_of(), + winrt::put_abi(item)); + + if (FAILED(hr)) { + std::cerr << "ERROR: CreateForWindow failed: 0x" << std::hex << hr << std::endl; + return nullptr; } - captureItem_ = createCaptureItemForMonitor(monitor); + return item; +} + +bool WgcSession::initializeWithItem(int fps) { if (!captureItem_) return false; auto size = captureItem_.Size(); @@ -121,6 +129,38 @@ bool WgcSession::initialize(HMONITOR monitor, int fps) { return true; } +bool WgcSession::initialize(HMONITOR monitor, int fps) { + fps_ = fps; + frameIntervalHns_ = 10000000LL / fps_; + + if (!createD3DDevice()) return false; + + winrtDevice_ = createWinRTDevice(); + if (!winrtDevice_) { + std::cerr << "ERROR: Failed to create WinRT D3D device" << std::endl; + return false; + } + + captureItem_ = createCaptureItemForMonitor(monitor); + return initializeWithItem(fps); +} + +bool WgcSession::initialize(HWND hwnd, int fps) { + fps_ = fps; + frameIntervalHns_ = 10000000LL / fps_; + + if (!createD3DDevice()) return false; + + winrtDevice_ = createWinRTDevice(); + if (!winrtDevice_) { + std::cerr << "ERROR: Failed to create WinRT D3D device" << std::endl; + return false; + } + + captureItem_ = createCaptureItemForWindow(hwnd); + return initializeWithItem(fps); +} + void WgcSession::setFrameCallback(FrameCallback callback) { frameCallback_ = std::move(callback); } diff --git a/electron/native/wgc-capture/src/wgc_session.h b/electron/native/wgc-capture/src/wgc_session.h index 094d95d3..5ca6be34 100644 --- a/electron/native/wgc-capture/src/wgc_session.h +++ b/electron/native/wgc-capture/src/wgc_session.h @@ -24,6 +24,7 @@ public: ~WgcSession(); bool initialize(HMONITOR monitor, int fps); + bool initialize(HWND hwnd, int fps); void setFrameCallback(FrameCallback callback); bool startCapture(); void stopCapture(); @@ -53,6 +54,8 @@ private: bool createD3DDevice(); winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice createWinRTDevice(); winrt::Windows::Graphics::Capture::GraphicsCaptureItem createCaptureItemForMonitor(HMONITOR monitor); + winrt::Windows::Graphics::Capture::GraphicsCaptureItem createCaptureItemForWindow(HWND hwnd); + bool initializeWithItem(int fps); void onFrameArrived( winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& sender, winrt::Windows::Foundation::IInspectable const& args); diff --git a/electron/preload.ts b/electron/preload.ts index e8729f3f..ebc4fed1 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -35,6 +35,7 @@ contextBridge.exposeInMainWorld('electronAPI', { capturesSystemAudio?: boolean capturesMicrophone?: boolean microphoneDeviceId?: string + microphoneLabel?: string }, ) => { return ipcRenderer.invoke('start-native-screen-recording', source, options) @@ -178,7 +179,7 @@ contextBridge.exposeInMainWorld('electronAPI', { return () => ipcRenderer.removeListener('request-save-before-close', listener) }, isWgcAvailable: () => ipcRenderer.invoke('is-wgc-available'), - storeWgcAudio: (audioData: ArrayBuffer, type: 'system' | 'mic') => ipcRenderer.invoke('store-wgc-audio', audioData, type), + muxWgcRecording: () => ipcRenderer.invoke('mux-wgc-recording'), // Cursor visibility control for cursor-free browser capture fallback hideOsCursor: () => ipcRenderer.invoke('hide-cursor'), }) diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 7abc9561..53ae6720 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -54,8 +54,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const chunks = useRef([]); const startTime = useRef(0); const nativeScreenRecording = useRef(false); - const wgcAudioRecorders = useRef([]); - const wgcAudioChunks = useRef>(new Map()); + const wgcRecording = useRef(false); const startInFlight = useRef(false); const hasPromptedForReselect = useRef(false); @@ -128,95 +127,6 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return Math.round(BITRATE_BASE * highFrameRateBoost); }; - const stopWgcAudioCapture = useCallback(async () => { - for (const recorder of wgcAudioRecorders.current) { - if (recorder.state === "recording") { - recorder.stop(); - } - } - - // Wait briefly for onstop callbacks to fire - await new Promise((resolve) => setTimeout(resolve, 200)); - - for (const [type, chunks] of wgcAudioChunks.current.entries()) { - if (chunks.length > 0) { - const blob = new Blob(chunks, { type: "audio/webm" }); - const buffer = await blob.arrayBuffer(); - try { - await window.electronAPI.storeWgcAudio(buffer, type as "system" | "mic"); - } catch (error) { - console.warn(`Failed to store WGC ${type} audio:`, error); - } - } - } - - wgcAudioRecorders.current = []; - wgcAudioChunks.current = new Map(); - }, []); - - const startWgcAudioCapture = async ( - source: { id?: string; display_id?: string }, - captureSystemAudio: boolean, - captureMicrophone: boolean, - micDeviceId?: string, - ) => { - wgcAudioRecorders.current = []; - wgcAudioChunks.current = new Map(); - - if (captureSystemAudio) { - try { - const systemAudioStream = await (navigator.mediaDevices as any).getUserMedia({ - audio: { - mandatory: { - chromeMediaSource: CHROME_MEDIA_SOURCE, - chromeMediaSourceId: source.id, - }, - }, - video: false, - }); - - const systemChunks: Blob[] = []; - wgcAudioChunks.current.set("system", systemChunks); - const recorder = new MediaRecorder(systemAudioStream, { - mimeType: "audio/webm", - audioBitsPerSecond: AUDIO_BITRATE_SYSTEM, - }); - recorder.ondataavailable = (e) => { - if (e.data.size > 0) systemChunks.push(e.data); - }; - recorder.start(RECORDER_TIMESLICE_MS); - wgcAudioRecorders.current.push(recorder); - } catch (error) { - console.warn("WGC: System audio capture failed:", error); - } - } - - if (captureMicrophone) { - try { - const micStream = await navigator.mediaDevices.getUserMedia({ - audio: micDeviceId - ? { deviceId: { exact: micDeviceId }, echoCancellation: true, noiseSuppression: true } - : { echoCancellation: true, noiseSuppression: true }, - video: false, - }); - - const micChunks: Blob[] = []; - wgcAudioChunks.current.set("mic", micChunks); - const recorder = new MediaRecorder(micStream, { - mimeType: "audio/webm", - audioBitsPerSecond: AUDIO_BITRATE_VOICE, - }); - recorder.ondataavailable = (e) => { - if (e.data.size > 0) micChunks.push(e.data); - }; - recorder.start(RECORDER_TIMESLICE_MS); - wgcAudioRecorders.current.push(recorder); - } catch (error) { - console.warn("WGC: Microphone capture failed:", error); - } - } - }; - const cleanupCapturedMedia = useCallback(() => { if (stream.current) { stream.current.getTracks().forEach((track) => track.stop()); @@ -245,10 +155,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setRecording(false); void (async () => { - // Stop WGC audio recorders first so audio data is stored before stop - if (wgcAudioRecorders.current.length > 0) { - await stopWgcAudioCapture(); - } + const isWgc = wgcRecording.current; + wgcRecording.current = false; const result = await window.electronAPI.stopNativeScreenRecording(); window.electronAPI?.setRecordingState(false); @@ -258,7 +166,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } - await window.electronAPI.setCurrentVideoPath(result.path); + let finalPath = result.path; + + if (isWgc) { + const muxResult = await window.electronAPI.muxWgcRecording(); + finalPath = muxResult?.path ?? result.path; + } + + await window.electronAPI.setCurrentVideoPath(finalPath); await window.electronAPI.switchToEditor(); })(); return; @@ -355,7 +270,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { let useWgcCapture = false; if ( platform === "win32" && - selectedSource.id?.startsWith("screen:") && + (selectedSource.id?.startsWith("screen:") || selectedSource.id?.startsWith("window:")) && typeof window.electronAPI.isWgcAvailable === "function" ) { try { @@ -367,10 +282,25 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } if (useNativeMacScreenCapture || useWgcCapture) { + // WGC: resolve mic device label for native WASAPI capture + let micLabel: string | undefined; + if (useWgcCapture && microphoneEnabled) { + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const mic = devices.find( + (d) => d.deviceId === microphoneDeviceId && d.kind === "audioinput", + ); + micLabel = mic?.label || undefined; + } catch { + // Fall through — native process will use default mic + } + } + const nativeResult = await window.electronAPI.startNativeScreenRecording(selectedSource, { capturesSystemAudio: systemAudioEnabled, capturesMicrophone: microphoneEnabled, microphoneDeviceId, + microphoneLabel: micLabel, }); if (!nativeResult.success) { if (useWgcCapture) { @@ -384,15 +314,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (nativeResult.success) { nativeScreenRecording.current = true; + wgcRecording.current = useWgcCapture; startTime.current = Date.now(); setRecording(true); window.electronAPI?.setRecordingState(true); - // WGC: start audio-only MediaRecorders in parallel - if (useWgcCapture && (systemAudioEnabled || microphoneEnabled)) { - void startWgcAudioCapture(selectedSource, systemAudioEnabled, microphoneEnabled, microphoneDeviceId); - } - return; } } diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts index f70eecbf..845f5960 100644 --- a/src/vite-env.d.ts +++ b/src/vite-env.d.ts @@ -40,6 +40,7 @@ interface Window { capturesSystemAudio?: boolean capturesMicrophone?: boolean microphoneDeviceId?: string + microphoneLabel?: string }) => Promise<{ success: boolean path?: string