feat: add backend Whisper and native audio support

Add Electron IPC handlers and preload wiring for Whisper caption generation, small-model download management, and safer project media source resolution. Update native macOS capture helpers to mux separate system and microphone audio outputs before finalizing recordings.

electron/electron-env.d.ts

@@ -134,6 +134,50 @@ interface Window {
 		) => Promise<{ success: boolean; path?: string; message?: string; canceled?: boolean }>;
 		openVideoFilePicker: () => Promise<{ success: boolean; path?: string; canceled?: boolean }>;
 		openAudioFilePicker: () => Promise<{ success: boolean; path?: string; canceled?: boolean }>;
+		openWhisperExecutablePicker: () => Promise<{
+			success: boolean;
+			path?: string;
+			canceled?: boolean;
+			error?: string;
+		}>;
+		openWhisperModelPicker: () => Promise<{
+			success: boolean;
+			path?: string;
+			canceled?: boolean;
+			error?: string;
+		}>;
+		getWhisperSmallModelStatus: () => Promise<{
+			success: boolean;
+			exists: boolean;
+			path?: string | null;
+			error?: string;
+		}>;
+		downloadWhisperSmallModel: () => Promise<{
+			success: boolean;
+			path?: string;
+			alreadyDownloaded?: boolean;
+			error?: string;
+		}>;
+		deleteWhisperSmallModel: () => Promise<{ success: boolean; error?: string }>;
+		onWhisperSmallModelDownloadProgress: (
+			callback: (state: {
+				status: "idle" | "downloading" | "downloaded" | "error";
+				progress: number;
+				path?: string | null;
+				error?: string;
+			}) => void,
+		) => () => void;
+		generateAutoCaptions: (options: {
+			videoPath: string;
+			whisperExecutablePath?: string;
+			whisperModelPath: string;
+			language?: string;
+		}) => Promise<{
+			success: boolean;
+			cues?: AutoCaptionCue[];
+			message?: string;
+			error?: string;
+		}>;
 		setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>;
 		setCurrentRecordingSession: (session: {
 			videoPath: string;
@@ -256,3 +300,16 @@ interface SystemCursorAsset {
 	width: number;
 	height: number;
 }
+
+interface AutoCaptionCue {
+	id: string;
+	startMs: number;
+	endMs: number;
+	text: string;
+	words?: Array<{
+		text: string;
+		startMs: number;
+		endMs: number;
+		leadingSpace?: boolean;
+	}>;
+}

electron/native/ScreenCaptureKitRecorder.swift

@@ -10,6 +10,7 @@ struct CaptureConfig: Codable {
 	let outputPath: String?
 	let capturesSystemAudio: Bool?
 	let capturesMicrophone: Bool?
+	let systemAudioOutputPath: String?
 	let microphoneDeviceId: String?
 	let microphoneLabel: String?
 	let microphoneOutputPath: String?
@@ -21,12 +22,13 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 	private let queue = DispatchQueue(label: "openscreen.screencapturekit.video")
 	private var assetWriter: AVAssetWriter?
 	private var videoInput: AVAssetWriterInput?
-	private var audioInput: AVAssetWriterInput?
+	private var systemAudioWriter: AVAssetWriter?
+	private var systemAudioInput: AVAssetWriterInput?
 	private var microphoneOnlyWriter: AVAssetWriter?
 	private var microphoneOnlyInput: AVAssetWriterInput?
 	private var stream: SCStream?
 	private var firstSampleTime: CMTime = .zero
-	private var firstPrimaryAudioSampleTime: CMTime?
+	private var firstSystemAudioSampleTime: CMTime?
 	private var firstMicrophoneSampleTime: CMTime?
 	private var lastSampleBuffer: CMSampleBuffer?
 	private var lastVideoPresentationTime: CMTime = .zero
@@ -44,17 +46,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 	private var windowValidationTask: Task<Void, Never>?
 	private var capturesSystemAudio = false
 	private var capturesMicrophone = false
+	private var writesSystemAudioToSeparateTrack = false
 	private var writesMicrophoneToSeparateTrack = false
 
-	private enum PrimaryAudioSource {
-		case system
-		case microphone
-	}
-
 	private let microphoneOutputTypeRawValue = 2
 
-	private var primaryAudioSource: PrimaryAudioSource?
-
 	func startCapture(configJSON: String) async throws {
 		guard !isRecording else {
 			throw NSError(domain: "RecordlyCapture", code: 1, userInfo: [NSLocalizedDescriptionKey: "Recording is already in progress"])
@@ -76,8 +72,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 				userInfo: [NSLocalizedDescriptionKey: "Native microphone capture is unavailable on this macOS/Xcode runtime"]
 			)
 		}
+		writesSystemAudioToSeparateTrack = capturesSystemAudio
 		writesMicrophoneToSeparateTrack = capturesSystemAudio && capturesMicrophone
-		primaryAudioSource = capturesSystemAudio ? .system : (capturesMicrophone ? .microphone : nil)
+		if capturesMicrophone && !capturesSystemAudio {
+			writesMicrophoneToSeparateTrack = true
+		}
 		let requestedFPS = max(targetCaptureFPS, config.fps ?? targetCaptureFPS)
 		streamConfig.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(requestedFPS))
 		streamConfig.queueDepth = 6
@@ -144,7 +143,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		let outputFileType: AVFileType = destinationURL.pathExtension.lowercased() == "mp4" ? .mp4 : .mov
 		assetWriter = try AVAssetWriter(url: destinationURL, fileType: outputFileType)
 		microphoneOutputURL = nil
-		firstPrimaryAudioSampleTime = nil
+		firstSystemAudioSampleTime = nil
 		firstMicrophoneSampleTime = nil
 
 		guard let assistant = AVOutputSettingsAssistant(preset: .preset3840x2160) else {
@@ -174,21 +173,34 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		assetWriter.add(videoInput)
 		self.videoInput = videoInput
 
-		if primaryAudioSource != nil {
-			let audioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: Self.audioOutputSettings(bitRate: 160_000))
-			audioInput.expectsMediaDataInRealTime = true
-
-			guard assetWriter.canAdd(audioInput) else {
-				throw NSError(domain: "RecordlyCapture", code: 11, userInfo: [NSLocalizedDescriptionKey: "Unable to add audio writer input"])
+		if writesSystemAudioToSeparateTrack {
+			guard let systemAudioOutputPath = config.systemAudioOutputPath, !systemAudioOutputPath.isEmpty else {
+				throw NSError(domain: "RecordlyCapture", code: 11, userInfo: [NSLocalizedDescriptionKey: "Missing system audio output path for audio capture"])
 			}
 
-			assetWriter.add(audioInput)
-			self.audioInput = audioInput
+			let systemAudioURL = URL(fileURLWithPath: systemAudioOutputPath)
+			let systemAudioWriter = try AVAssetWriter(url: systemAudioURL, fileType: .m4a)
+			let systemAudioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: Self.audioOutputSettings(bitRate: 160_000))
+			systemAudioInput.expectsMediaDataInRealTime = true
+
+			guard systemAudioWriter.canAdd(systemAudioInput) else {
+				throw NSError(domain: "RecordlyCapture", code: 12, userInfo: [NSLocalizedDescriptionKey: "Unable to add system audio writer input"])
+			}
+
+			systemAudioWriter.add(systemAudioInput)
+			self.systemAudioWriter = systemAudioWriter
+			self.systemAudioInput = systemAudioInput
+
+			guard systemAudioWriter.startWriting() else {
+				throw NSError(domain: "RecordlyCapture", code: 13, userInfo: [NSLocalizedDescriptionKey: systemAudioWriter.error?.localizedDescription ?? "Unable to start system audio writing"])
+			}
+
+			systemAudioWriter.startSession(atSourceTime: .zero)
 		}
 
 		if writesMicrophoneToSeparateTrack {
 			guard let microphoneOutputPath = config.microphoneOutputPath, !microphoneOutputPath.isEmpty else {
-				throw NSError(domain: "RecordlyCapture", code: 12, userInfo: [NSLocalizedDescriptionKey: "Missing microphone output path for dual-audio capture"])
+				throw NSError(domain: "RecordlyCapture", code: 14, userInfo: [NSLocalizedDescriptionKey: "Missing microphone output path for microphone capture"])
 			}
 
 			let microphoneURL = URL(fileURLWithPath: microphoneOutputPath)
@@ -198,7 +210,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 			microphoneInput.expectsMediaDataInRealTime = true
 
 			guard microphoneWriter.canAdd(microphoneInput) else {
-				throw NSError(domain: "RecordlyCapture", code: 13, userInfo: [NSLocalizedDescriptionKey: "Unable to add microphone writer input"])
+				throw NSError(domain: "RecordlyCapture", code: 15, userInfo: [NSLocalizedDescriptionKey: "Unable to add microphone writer input"])
 			}
 
 			microphoneWriter.add(microphoneInput)
@@ -206,7 +218,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 			self.microphoneOnlyInput = microphoneInput
 
 			guard microphoneWriter.startWriting() else {
-				throw NSError(domain: "RecordlyCapture", code: 14, userInfo: [NSLocalizedDescriptionKey: microphoneWriter.error?.localizedDescription ?? "Unable to start microphone audio writing"])
+				throw NSError(domain: "RecordlyCapture", code: 16, userInfo: [NSLocalizedDescriptionKey: microphoneWriter.error?.localizedDescription ?? "Unable to start microphone audio writing"])
 			}
 
 			microphoneWriter.startSession(atSourceTime: .zero)
@@ -222,7 +234,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 			guard let microphoneOutputType = SCStreamOutputType(rawValue: microphoneOutputTypeRawValue) else {
 				throw NSError(
 					domain: "RecordlyCapture",
-					code: 15,
+					code: 17,
 					userInfo: [NSLocalizedDescriptionKey: "Microphone stream output type is unavailable"]
 				)
 			}
@@ -302,16 +314,14 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		}
 
 		if outputType == .audio {
-			guard primaryAudioSource == .system, let audioInput else { return }
-			appendAudioSampleBuffer(sampleBuffer, to: audioInput, firstSampleTime: &firstPrimaryAudioSampleTime, presentationTime: presentationTime)
+			guard let systemAudioInput else { return }
+			appendAudioSampleBuffer(sampleBuffer, to: systemAudioInput, firstSampleTime: &firstSystemAudioSampleTime, presentationTime: presentationTime)
 			return
 		}
 
 		if outputType.rawValue == microphoneOutputTypeRawValue {
-			if writesMicrophoneToSeparateTrack, let microphoneOnlyInput {
+			if let microphoneOnlyInput {
 				appendAudioSampleBuffer(sampleBuffer, to: microphoneOnlyInput, firstSampleTime: &firstMicrophoneSampleTime, presentationTime: presentationTime)
-			} else if primaryAudioSource == .microphone, let audioInput {
-				appendAudioSampleBuffer(sampleBuffer, to: audioInput, firstSampleTime: &firstPrimaryAudioSampleTime, presentationTime: presentationTime)
 			}
 			return
 		}
@@ -349,23 +359,26 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 
 		assetWriter?.endSession(atSourceTime: lastSampleBuffer?.presentationTimeStamp ?? .zero)
 		videoInput?.markAsFinished()
-		audioInput?.markAsFinished()
 		await assetWriter?.finishWriting()
 
+		systemAudioInput?.markAsFinished()
+		await systemAudioWriter?.finishWriting()
+
 		microphoneOnlyInput?.markAsFinished()
 		await microphoneOnlyWriter?.finishWriting()
 
 		let path = outputURL?.path ?? ""
 		assetWriter = nil
 		videoInput = nil
-		audioInput = nil
+		systemAudioWriter = nil
+		systemAudioInput = nil
 		microphoneOnlyWriter = nil
 		microphoneOnlyInput = nil
 		outputURL = nil
 		microphoneOutputURL = nil
 		sessionStarted = false
 		firstSampleTime = .zero
-		firstPrimaryAudioSampleTime = nil
+		firstSystemAudioSampleTime = nil
 		firstMicrophoneSampleTime = nil
 		lastSampleBuffer = nil
 		lastVideoPresentationTime = .zero
@@ -377,8 +390,8 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		accumulatedPausedDuration = .zero
 		capturesSystemAudio = false
 		capturesMicrophone = false
+		writesSystemAudioToSeparateTrack = false
 		writesMicrophoneToSeparateTrack = false
-		primaryAudioSource = nil
 		return path
 	}
 

electron/preload.ts

@@ -153,6 +153,39 @@ contextBridge.exposeInMainWorld("electronAPI", {
 	openAudioFilePicker: () => {
 		return ipcRenderer.invoke("open-audio-file-picker");
 	},
+	openWhisperExecutablePicker: () => {
+		return ipcRenderer.invoke("open-whisper-executable-picker");
+	},
+	openWhisperModelPicker: () => {
+		return ipcRenderer.invoke("open-whisper-model-picker");
+	},
+	getWhisperSmallModelStatus: () => {
+		return ipcRenderer.invoke("get-whisper-small-model-status");
+	},
+	downloadWhisperSmallModel: () => {
+		return ipcRenderer.invoke("download-whisper-small-model");
+	},
+	deleteWhisperSmallModel: () => {
+		return ipcRenderer.invoke("delete-whisper-small-model");
+	},
+	onWhisperSmallModelDownloadProgress: (
+		callback: (state: { status: "idle" | "downloading" | "downloaded" | "error"; progress: number; path?: string | null; error?: string }) => void,
+	) => {
+		const listener = (
+			_event: Electron.IpcRendererEvent,
+			payload: { status: "idle" | "downloading" | "downloaded" | "error"; progress: number; path?: string | null; error?: string },
+		) => callback(payload);
+		ipcRenderer.on("whisper-small-model-download-progress", listener);
+		return () => ipcRenderer.removeListener("whisper-small-model-download-progress", listener);
+	},
+	generateAutoCaptions: (options: {
+		videoPath: string;
+		whisperExecutablePath?: string;
+		whisperModelPath: string;
+		language?: string;
+	}) => {
+		return ipcRenderer.invoke("generate-auto-captions", options);
+	},
 	setCurrentVideoPath: (path: string) => {
 		return ipcRenderer.invoke("set-current-video-path", path);
 	},