diff --git a/electron/ipc/captions/audioCandidates.test.ts b/electron/ipc/captions/audioCandidates.test.ts new file mode 100644 index 00000000..c08b49ff --- /dev/null +++ b/electron/ipc/captions/audioCandidates.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { getCaptionCompanionAudioCandidates } from "./audioCandidates"; + +describe("getCaptionCompanionAudioCandidates", () => { + it("prefers microphone audio and includes system audio when both sidecars exist", () => { + expect( + getCaptionCompanionAudioCandidates([ + { + platform: "win", + micPath: "recording.mic.wav", + systemPath: "recording.system.wav", + usablePaths: ["recording.system.wav", "recording.mic.wav"], + }, + ]), + ).toEqual([ + { path: "recording.mic.wav", label: "microphone audio sidecar" }, + { path: "recording.system.wav", label: "system audio sidecar" }, + ]); + }); + + it("omits companion paths that were not found on disk", () => { + expect( + getCaptionCompanionAudioCandidates([ + { + platform: "win", + micPath: "recording.mic.wav", + systemPath: "recording.system.wav", + usablePaths: ["recording.mic.wav"], + }, + ]), + ).toEqual([{ path: "recording.mic.wav", label: "microphone audio sidecar" }]); + }); +}); diff --git a/electron/ipc/captions/audioCandidates.ts b/electron/ipc/captions/audioCandidates.ts new file mode 100644 index 00000000..33e62d2e --- /dev/null +++ b/electron/ipc/captions/audioCandidates.ts @@ -0,0 +1,21 @@ +import type { CompanionAudioCandidate } from "../types"; + +export type CaptionAudioCandidate = { + path: string; + label: string; +}; + +export function getCaptionCompanionAudioCandidates( + companions: CompanionAudioCandidate[], +): CaptionAudioCandidate[] { + return companions.flatMap((companion) => { + const candidates: CaptionAudioCandidate[] = []; + if (companion.usablePaths.includes(companion.micPath)) { + candidates.push({ path: companion.micPath, label: "microphone audio sidecar" }); + } + if (companion.usablePaths.includes(companion.systemPath)) { + candidates.push({ path: companion.systemPath, label: "system audio sidecar" }); + } + return candidates; + }); +} diff --git a/electron/ipc/captions/generate.test.ts b/electron/ipc/captions/generate.test.ts new file mode 100644 index 00000000..9844fecc --- /dev/null +++ b/electron/ipc/captions/generate.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import { isMissingWindowsWhisperRuntimeDependency } from "./runtimeErrors"; + +describe("isMissingWindowsWhisperRuntimeDependency", () => { + it("recognizes the signed and unsigned STATUS_DLL_NOT_FOUND exit codes on Windows", () => { + if (process.platform !== "win32") return; + + expect(isMissingWindowsWhisperRuntimeDependency({ code: -1073741515 })).toBe(true); + expect(isMissingWindowsWhisperRuntimeDependency({ code: 3221225781 })).toBe(true); + }); + + it("does not classify ordinary Whisper failures as missing runtimes", () => { + expect(isMissingWindowsWhisperRuntimeDependency({ code: 1 })).toBe(false); + expect(isMissingWindowsWhisperRuntimeDependency(new Error("bad model"))).toBe(false); + }); +}); diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.ts index 2a3f49cd..246c4f1a 100644 --- a/electron/ipc/captions/generate.ts +++ b/electron/ipc/captions/generate.ts @@ -7,8 +7,11 @@ import { app } from "electron"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import { getBundledWhisperExecutableCandidates } from "../paths/binaries"; import { resolveRecordingSession } from "../project/session"; +import { getUsableCompanionAudioCandidates } from "../recording/diagnostics"; import { normalizeVideoSourcePath } from "../utils"; +import { getCaptionCompanionAudioCandidates } from "./audioCandidates"; import { parseSrtCues, parseWhisperJsonCues, shouldRetryWhisperWithoutJson } from "./parser"; +import { isMissingWindowsWhisperRuntimeDependency } from "./runtimeErrors"; import { segmentCuesIntoPhrases } from "./segment"; import { parseSilenceIntervals, @@ -19,6 +22,22 @@ import { const execFileAsync = promisify(execFile); +async function executeWhisper(whisperExecutablePath: string, args: string[]) { + try { + await execFileAsync(whisperExecutablePath, args, { + timeout: 30 * 60 * 1000, + maxBuffer: 20 * 1024 * 1024, + }); + } catch (error) { + if (isMissingWindowsWhisperRuntimeDependency(error)) { + throw new Error( + "Whisper could not start because the Microsoft Visual C++ x64 Redistributable is missing. Install it from https://aka.ms/vc14/vc_redist.x64.exe, then restart Recordly.", + ); + } + throw error; + } +} + export async function ensureReadableFile(filePath: string, options?: { executable?: boolean }) { await fs.access(filePath, fsConstants.R_OK); if (options?.executable) { @@ -78,7 +97,8 @@ export async function resolveWhisperExecutablePath(preferredPath?: string | null } throw new Error( - "No Whisper runtime was found. Recordly looked for a bundled binary first, then checked common system install locations.", + `No Whisper runtime was found for ${process.platform}/${process.arch}. ` + + "This Recordly build is missing its bundled caption runtime. Reinstall or update Recordly, or select a whisper-cli executable in Caption settings.", ); } @@ -97,6 +117,10 @@ export async function resolveCaptionAudioCandidates(videoPath: string) { }; pushCandidate(videoPath, "recording"); + const companionAudio = await getUsableCompanionAudioCandidates(videoPath); + for (const candidate of getCaptionCompanionAudioCandidates(companionAudio)) { + pushCandidate(candidate.path, candidate.label); + } const requestedRecordingSession = await resolveRecordingSession(videoPath); pushCandidate(requestedRecordingSession?.webcamPath, "linked webcam recording"); @@ -236,10 +260,7 @@ export async function generateAutoCaptionsFromVideo(options: { let jsonEnabled = true; try { - await execFileAsync(whisperExecutablePath, [...whisperBaseArgs, "-ojf"], { - timeout: 30 * 60 * 1000, - maxBuffer: 20 * 1024 * 1024, - }); + await executeWhisper(whisperExecutablePath, [...whisperBaseArgs, "-ojf"]); } catch (error) { if (!shouldRetryWhisperWithoutJson(error)) { throw error; @@ -250,10 +271,7 @@ export async function generateAutoCaptionsFromVideo(options: { "[auto-captions] Whisper runtime does not support JSON full output, retrying with SRT only:", error, ); - await execFileAsync(whisperExecutablePath, whisperBaseArgs, { - timeout: 30 * 60 * 1000, - maxBuffer: 20 * 1024 * 1024, - }); + await executeWhisper(whisperExecutablePath, whisperBaseArgs); } const timedCues = jsonEnabled diff --git a/electron/ipc/captions/runtimeErrors.ts b/electron/ipc/captions/runtimeErrors.ts new file mode 100644 index 00000000..707848dd --- /dev/null +++ b/electron/ipc/captions/runtimeErrors.ts @@ -0,0 +1,10 @@ +const WINDOWS_MISSING_RUNTIME_EXIT_CODES = new Set([-1073741515, 3221225781]); + +export function isMissingWindowsWhisperRuntimeDependency(error: unknown) { + if (process.platform !== "win32" || !error || typeof error !== "object") { + return false; + } + + const code = (error as { code?: unknown }).code; + return typeof code === "number" && WINDOWS_MISSING_RUNTIME_EXIT_CODES.has(code); +} diff --git a/electron/ipc/register/captions.ts b/electron/ipc/register/captions.ts index fe93afd7..7dfe5d67 100644 --- a/electron/ipc/register/captions.ts +++ b/electron/ipc/register/captions.ts @@ -247,7 +247,8 @@ export function registerCaptionHandlers() { return { success: false, error: String(error), - message: "Failed to generate auto captions", + message: + error instanceof Error ? error.message : "Failed to generate auto captions", }; } }, diff --git a/package.json b/package.json index b23ab48f..f1ef508f 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "rebuild:native": "node ./node_modules/@electron/rebuild/lib/cli.js --force --only uiohook-napi", "build:native-helpers": "node scripts/build-native-helpers.mjs", "build:whisper-runtime": "node scripts/build-whisper-runtime.mjs", - "build:platform-native-helpers": "npm run build:native-helpers && npm run build:windows-capture && npm run build:windows-gpu-export && npm run build:nvidia-cuda-compositor && npm run build:cursor-monitor && npm run build:whisper-runtime", + "build:platform-native-helpers": "npm run build:whisper-runtime && npm run build:native-helpers && npm run build:windows-capture && npm run build:windows-gpu-export && npm run build:nvidia-cuda-compositor && npm run build:cursor-monitor", "build:windows-gpu-export": "node scripts/build-windows-gpu-export.mjs", "build:nvidia-cuda-compositor": "node scripts/build-nvidia-cuda-compositor.mjs", "build:windows-capture": "node scripts/build-windows-capture.mjs", diff --git a/scripts/build-whisper-runtime.mjs b/scripts/build-whisper-runtime.mjs index e5e46c99..f1f76f7a 100644 --- a/scripts/build-whisper-runtime.mjs +++ b/scripts/build-whisper-runtime.mjs @@ -1,4 +1,5 @@ import { execFileSync, execSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { createWriteStream, existsSync, rmSync } from "node:fs"; import { chmod, cp, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; import { get as httpsGet } from "node:https"; @@ -15,6 +16,8 @@ const nativeRoot = path.join(projectRoot, "electron", "native"); const cacheRoot = path.join(projectRoot, ".tmp", "whisper-runtime"); const archivePath = path.join(cacheRoot, `${whisperVersion}.tar.gz`); const extractRoot = path.join(cacheRoot, `src-${whisperVersion}`); +const windowsX64ArchivePath = path.join(cacheRoot, `${whisperVersion}-windows-x64.zip`); +const windowsX64ArchiveSha256 = "74f973345cb52ef5ba3ec9e7e7af8e48cc8c71722d1528603b80588a11f82e3e"; function getHostArch() { return process.arch === "arm64" ? "arm64" : "x64"; @@ -225,6 +228,75 @@ async function downloadFile(url, destinationPath) { }); } +async function getFileSha256(filePath) { + return createHash("sha256") + .update(await readFile(filePath)) + .digest("hex"); +} + +async function findDirectoryContaining(rootPath, fileName) { + const pending = [rootPath]; + while (pending.length > 0) { + const currentPath = pending.shift(); + const entries = await readdir(currentPath, { withFileTypes: true }); + if (entries.some((entry) => entry.isFile() && entry.name === fileName)) { + return currentPath; + } + for (const entry of entries) { + if (entry.isDirectory()) { + pending.push(path.join(currentPath, entry.name)); + } + } + } + return null; +} + +async function stageWindowsX64PrebuiltRuntime(target) { + if (target.platform !== "win32" || target.arch !== "x64") { + return false; + } + + await mkdir(cacheRoot, { recursive: true }); + const archiveUrl = `https://github.com/ggml-org/whisper.cpp/releases/download/${whisperVersion}/whisper-bin-x64.zip`; + let archiveIsValid = + existsSync(windowsX64ArchivePath) && + (await getFileSha256(windowsX64ArchivePath)) === windowsX64ArchiveSha256; + if (!archiveIsValid) { + await rm(windowsX64ArchivePath, { force: true }); + console.log( + `[build-whisper-runtime] Downloading official whisper.cpp ${whisperVersion} Windows x64 runtime...`, + ); + await downloadFile(archiveUrl, windowsX64ArchivePath); + archiveIsValid = (await getFileSha256(windowsX64ArchivePath)) === windowsX64ArchiveSha256; + } + + if (!archiveIsValid) { + throw new Error( + "[build-whisper-runtime] Windows x64 runtime archive failed its SHA-256 integrity check.", + ); + } + + const prebuiltExtractRoot = path.join(cacheRoot, `prebuilt-${target.archTag}`); + await rm(prebuiltExtractRoot, { recursive: true, force: true }); + await mkdir(prebuiltExtractRoot, { recursive: true }); + execFileSync("tar", ["-xf", windowsX64ArchivePath, "-C", prebuiltExtractRoot], { + stdio: "inherit", + }); + + const runtimeDir = await findDirectoryContaining(prebuiltExtractRoot, "whisper-cli.exe"); + if (!runtimeDir) { + throw new Error( + "[build-whisper-runtime] Official Windows archive did not contain whisper-cli.exe.", + ); + } + + const runtimeEntries = (await readdir(runtimeDir)).filter( + (entry) => /^(whisper|ggml)/i.test(entry) || entry.toLowerCase().endsWith(".dll"), + ); + await stageRuntimeArtifacts(target, runtimeDir, runtimeEntries); + return true; +} + async function ensureSourceTree() { const extractedSourceDir = path.join( extractRoot, @@ -366,6 +438,31 @@ async function stageRuntimeArtifacts(target, candidateDir, runtimeEntries) { async function main() { const targets = getTargetConfigs(); + + // Official whisper.cpp releases include a signed, portable Windows x64 + // runtime. Prefer it so developers and packaged builds do not require a full + // Visual Studio C++ installation just to enable captions. + for (const target of targets) { + if (!(await shouldSkipBuild(target))) { + try { + await stageWindowsX64PrebuiltRuntime(target); + } catch (error) { + console.warn( + "[build-whisper-runtime] Failed to stage the official Windows runtime; falling back to a source build:", + error, + ); + } + } + } + + const stagedChecks = await Promise.all(targets.map((target) => shouldSkipBuild(target))); + if (stagedChecks.every(Boolean)) { + console.log( + `[build-whisper-runtime] Whisper runtime ${whisperVersion} is staged for ${targets.map((target) => target.archTag).join(", ")}.`, + ); + return; + } + const cmake = findCmake(); if (!cmake) { diff --git a/src/components/video-editor/captions/useAutoCaptionController.ts b/src/components/video-editor/captions/useAutoCaptionController.ts index 3ec81ee7..a008f135 100644 --- a/src/components/video-editor/captions/useAutoCaptionController.ts +++ b/src/components/video-editor/captions/useAutoCaptionController.ts @@ -195,8 +195,8 @@ export function useAutoCaptionController({ }); if (!result.success || !result.cues) { toast.error( - result.message || - getErrorMessage(result.error) || + getErrorMessage(result.error) || + result.message || "Failed to generate captions", ); return;