Merge pull request #122 from mahdyarief/main

feat: Advanced Video Editor Implementation, Native WGC Integration, and AI Auto-Captions
This commit is contained in:
webadderall
2026-03-28 14:07:09 +11:00
committed by GitHub
41 changed files with 3228 additions and 875 deletions
+32 -26
View File
@@ -3,32 +3,32 @@ logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
dist
dist-electron
dist-ssr
*.local
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?
release/**
*.kiro/
# npx electron-builder --mac --win
.tmp/
.history/
*.tsbuildinfo
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
dist
dist-electron
dist-ssr
*.local
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?
release/**
*.kiro/
# npx electron-builder --mac --win
.tmp/
.history/
*.tsbuildinfo
vite.config.js
vite.config.d.ts
@@ -36,6 +36,12 @@ vite.config.d.ts
electron/native/wgc-capture/build/
electron/native/cursor-monitor/build/
# Local build tools and caches
.cache/
.cmake_ext/
ebcache/
docs/
# Local debug helpers
tmp-*.ps1
.tmp-*.ps1
+5 -5
View File
@@ -9,6 +9,7 @@
"electron/native/**"
],
"productName": "Recordly",
"copyright": "Copyright (c) 2026 webadderall",
"npmRebuild": true,
"buildDependenciesFromSource": true,
"compression": "normal",
@@ -62,9 +63,8 @@
"target": [
"nsis"
],
"icon": "icons/icons/win/icon.ico"
,
"artifactName": "Recordly.exe"
"icon": "icons/icons/win/icon.ico",
"executableName": "Recordly",
"artifactName": "Recordly.${ext}"
}
}
}
+15 -6
View File
@@ -147,23 +147,28 @@ interface Window {
canceled?: boolean;
error?: string;
}>;
getWhisperSmallModelStatus: () => Promise<{
getWhisperModelStatus: (
modelName: string,
) => Promise<{
success: boolean;
exists: boolean;
path?: string | null;
error?: string;
}>;
downloadWhisperSmallModel: () => Promise<{
downloadWhisperModel: (
modelName: string,
) => Promise<{
success: boolean;
path?: string;
alreadyDownloaded?: boolean;
error?: string;
}>;
deleteWhisperSmallModel: () => Promise<{ success: boolean; error?: string }>;
onWhisperSmallModelDownloadProgress: (
deleteWhisperModel: (modelName: string) => Promise<{ success: boolean; error?: string }>;
onWhisperModelDownloadProgress: (
callback: (state: {
status: "idle" | "downloading" | "downloaded" | "error";
progress: number;
model: string;
path?: string | null;
error?: string;
}) => void,
@@ -173,9 +178,11 @@ interface Window {
whisperExecutablePath?: string;
whisperModelPath: string;
language?: string;
durationMs?: number;
startTimeMs?: number;
}) => Promise<{
success: boolean;
cues?: AutoCaptionCue[];
cues?: CaptionCue[];
message?: string;
error?: string;
}>;
@@ -297,6 +304,8 @@ interface Window {
startCountdown: (seconds: number) => Promise<{ success: boolean; cancelled?: boolean }>;
cancelCountdown: () => Promise<{ success: boolean }>;
getActiveCountdown: () => Promise<{ success: boolean; seconds: number | null }>;
onAutoCaptionProgress: (callback: (payload: { progress: number }) => void) => () => void;
onAutoCaptionChunk: (callback: (payload: { cues: CaptionCue[] }) => void) => () => void;
onCountdownTick: (callback: (seconds: number) => void) => () => void;
};
}
@@ -338,7 +347,7 @@ interface SystemCursorAsset {
height: number;
}
interface AutoCaptionCue {
interface CaptionCue {
id: string;
startMs: number;
endMs: number;
+333 -121
View File
@@ -30,9 +30,48 @@ const AUTO_RECORDING_RETENTION_COUNT = 20
const AUTO_RECORDING_MAX_AGE_MS = 14 * 24 * 60 * 60 * 1000
const ALLOW_RECORDLY_WINDOW_CAPTURE = Boolean(process.env['VITE_DEV_SERVER_URL'])
const RECORDING_SESSION_MANIFEST_SUFFIX = '.recordly-session.json'
const WHISPER_MODEL_DOWNLOAD_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin'
const WHISPER_MODEL_DIR = path.join(app.getPath('userData'), 'whisper')
const WHISPER_SMALL_MODEL_PATH = path.join(WHISPER_MODEL_DIR, 'ggml-small.bin')
const WHISPER_MODEL_DIR = path.join(app.getPath("userData"), "whisper");
const WHISPER_MODELS = {
tiny: {
label: "Tiny",
size: "75 MB",
filename: "ggml-tiny.bin",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin",
},
base: {
label: "Base",
size: "142 MB",
filename: "ggml-base.bin",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin",
},
small: {
label: "Small",
size: "466 MB",
filename: "ggml-small.bin",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin",
},
medium: {
label: "Medium",
size: "1.5 GB",
filename: "ggml-medium.bin",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.bin",
},
large: {
label: "Large (v3)",
size: "2.9 GB",
filename: "ggml-large-v3.bin",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin",
},
} as const;
function getWhisperModelPath(modelName: string) {
const model = WHISPER_MODELS[modelName as keyof typeof WHISPER_MODELS];
if (!model) {
throw new Error(`Unsupported Whisper model: ${modelName}`);
}
return path.join(WHISPER_MODEL_DIR, model.filename);
}
function getAssetRootPath() {
if (app.isPackaged) {
@@ -54,9 +93,7 @@ function normalizeRecordingTimeOffsetMs(value: unknown): number {
function broadcastSelectedSourceChange() {
for (const window of BrowserWindow.getAllWindows()) {
if (!window.isDestroyed()) {
window.webContents.send('selected-source-changed', selectedSource)
}
safeSend(window.webContents, 'selected-source-changed', selectedSource)
}
}
@@ -952,30 +989,82 @@ function getFfmpegBinaryPath() {
return ffmpegStatic
}
function sendWhisperModelDownloadProgress(
webContents: Electron.WebContents,
payload: { status: 'idle' | 'downloading' | 'downloaded' | 'error'; progress: number; path?: string | null; error?: string },
) {
webContents.send('whisper-small-model-download-progress', payload)
function runWhisperWithProgress(
executablePath: string,
args: string[],
onProgress: (progress: number) => void,
): Promise<void> {
return new Promise((resolve, reject) => {
const proc = spawn(executablePath, args);
let output = "";
proc.stdout?.on("data", (data) => {
output += data.toString();
});
proc.stderr?.on("data", (data) => {
const text = data.toString();
output += text;
// whisper.cpp variants use this pattern on stderr
const match = text.match(/progress\s*=\s*(\d+)%/i);
if (match) {
onProgress(parseInt(match[1], 10));
}
});
proc.on("close", (code) => {
if (code === 0) {
resolve();
} else {
reject(new Error(output.trim() || `Whisper exited with code ${code}`));
}
});
proc.on("error", (err) => {
reject(err);
});
});
}
async function getWhisperSmallModelStatus() {
try {
await fs.access(WHISPER_SMALL_MODEL_PATH, fsConstants.R_OK)
return {
success: true,
exists: true,
path: WHISPER_SMALL_MODEL_PATH,
}
} catch {
return {
success: true,
exists: false,
path: null,
}
function safeSend(webContents: Electron.WebContents | undefined, channel: string, ...args: any[]) {
if (webContents && !webContents.isDestroyed()) {
webContents.send(channel, ...args)
}
}
type WhisperModelDownloadStatus = {
status: "idle" | "downloading" | "downloaded" | "error";
progress: number;
model: string;
path?: string | null;
error?: string;
}
function sendWhisperModelDownloadProgress(
webContents: Electron.WebContents | undefined,
progress: WhisperModelDownloadStatus,
) {
safeSend(webContents, 'whisper-model-download-progress', progress)
}
async function getWhisperModelStatus(_event: any, modelName: string) {
try {
const modelPath = getWhisperModelPath(modelName);
await fs.access(modelPath, fsConstants.R_OK);
return {
success: true,
exists: true,
path: modelPath,
};
} catch {
return {
success: true,
exists: false,
path: null,
};
}
}
function downloadFileWithProgress(
url: string,
destinationPath: string,
@@ -1025,7 +1114,9 @@ function downloadFileWithProgress(
reject(error)
})
fileStream.on('finish', () => {
// On Windows, the 'finish' event might fire before the OS has fully released the file handle.
// We listen for 'close' to be absolutely sure the file descriptor is closed.
fileStream.on('close', () => {
onProgress(100)
resolve()
})
@@ -1040,37 +1131,65 @@ function downloadFileWithProgress(
return request(url)
}
async function downloadWhisperSmallModel(webContents: Electron.WebContents) {
await fs.mkdir(WHISPER_MODEL_DIR, { recursive: true })
const tempPath = `${WHISPER_SMALL_MODEL_PATH}.download`
async function downloadWhisperModel(
webContents: Electron.WebContents,
modelName: string,
) {
const model = WHISPER_MODELS[modelName as keyof typeof WHISPER_MODELS];
if (!model) {
throw new Error(`Unsupported Whisper model: ${modelName}`);
}
sendWhisperModelDownloadProgress(webContents, {
status: 'downloading',
progress: 0,
path: null,
})
await fs.mkdir(WHISPER_MODEL_DIR, { recursive: true });
const modelPath = getWhisperModelPath(modelName);
const tempPath = `${modelPath}.download`;
sendWhisperModelDownloadProgress(webContents, {
status: "downloading",
progress: 0,
model: modelName,
path: null,
});
try {
await fs.rm(tempPath, { force: true })
await downloadFileWithProgress(WHISPER_MODEL_DOWNLOAD_URL, tempPath, (progress) => {
await fs.rm(tempPath, { force: true }).catch(() => undefined)
await downloadFileWithProgress(model.url, tempPath, (progress) => {
sendWhisperModelDownloadProgress(webContents, {
status: 'downloading',
progress,
model: modelName,
path: null,
})
})
await fs.rename(tempPath, WHISPER_SMALL_MODEL_PATH)
// Robust rename logic for Windows to avoid EPERM/EBUSY
let renameRetries = 0
const maxRetries = 5
while (renameRetries < maxRetries) {
try {
await fs.rename(tempPath, modelPath)
break
} catch (err) {
renameRetries++
if (renameRetries >= maxRetries) throw err
// Wait briefly between retries to allow OS to release file handles
await new Promise((resolve) => setTimeout(resolve, 100 * renameRetries))
}
}
sendWhisperModelDownloadProgress(webContents, {
status: 'downloaded',
progress: 100,
path: WHISPER_SMALL_MODEL_PATH,
model: modelName,
path: modelPath,
})
return WHISPER_SMALL_MODEL_PATH
return modelPath
} catch (error) {
await fs.rm(tempPath, { force: true }).catch(() => undefined)
sendWhisperModelDownloadProgress(webContents, {
status: 'error',
progress: 0,
model: modelName,
path: null,
error: String(error),
})
@@ -1078,8 +1197,9 @@ async function downloadWhisperSmallModel(webContents: Electron.WebContents) {
}
}
async function deleteWhisperSmallModel() {
await fs.rm(WHISPER_SMALL_MODEL_PATH, { force: true })
async function deleteWhisperModel(_event: any, modelName: string) {
const modelPath = getWhisperModelPath(modelName);
await fs.rm(modelPath, { force: true });
}
function parseSrtTimestamp(value: string) {
@@ -1368,6 +1488,8 @@ async function extractCaptionAudioSource(options: {
videoPath: string
ffmpegPath: string
wavPath: string
startTime?: number // in seconds
duration?: number // in seconds
}) {
const candidates = await resolveCaptionAudioCandidates(options.videoPath)
const attemptedCandidates: Array<{
@@ -1381,11 +1503,23 @@ async function extractCaptionAudioSource(options: {
for (const candidate of candidates) {
try {
await ensureReadableFile(candidate.path, 'video file')
console.log('[auto-captions] Extracting audio from:', path.basename(candidate.path), options.startTime ? `at ${options.startTime}s` : '')
const ffmpegArgs = ['-y'];
if (options.startTime !== undefined) {
ffmpegArgs.push('-ss', options.startTime.toString());
}
if (options.duration !== undefined) {
ffmpegArgs.push('-t', options.duration.toString());
}
ffmpegArgs.push('-i', candidate.path, '-map', '0:a:0', '-vn', '-ac', '1', '-ar', '16000', '-c:a', 'pcm_s16le', options.wavPath);
await execFileAsync(
options.ffmpegPath,
['-y', '-i', candidate.path, '-map', '0:a:0', '-vn', '-ac', '1', '-ar', '16000', '-c:a', 'pcm_s16le', options.wavPath],
ffmpegArgs,
{ timeout: 5 * 60 * 1000, maxBuffer: 20 * 1024 * 1024 },
)
console.log('[auto-captions] Audio extracted successfully to temporary workspace')
attemptedCandidates.push({ ...candidate, readable: true, extractedAudio: true })
return candidate
} catch (error) {
@@ -1399,17 +1533,22 @@ async function extractCaptionAudioSource(options: {
}
}
console.warn('[auto-captions] No audio source candidate could be extracted:', attemptedCandidates)
console.warn('[auto-captions] No audio source candidate could be extracted')
throw new Error('No audio was found to transcribe in the saved recording file. Captions need an audio track. If this recording should have contained sound, the recording was saved without an audio stream.')
}
async function generateAutoCaptionsFromVideo(options: {
videoPath: string
whisperExecutablePath?: string
whisperModelPath: string
language?: string
}) {
async function generateAutoCaptionsFromVideo(
webContents: Electron.WebContents,
options: {
videoPath: string;
whisperExecutablePath?: string;
whisperModelPath: string;
language?: string;
durationMs?: number;
startTimeMs?: number;
},
) {
const ffmpegPath = getFfmpegBinaryPath()
const normalizedVideoPath = normalizeVideoSourcePath(options.videoPath)
if (!normalizedVideoPath) {
@@ -1421,68 +1560,125 @@ async function generateAutoCaptionsFromVideo(options: {
await ensureReadableFile(whisperExecutablePath, 'whisper executable')
await ensureReadableFile(whisperModelPath, 'whisper model')
const tempBase = path.join(app.getPath('temp'), `recordly-captions-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`)
const wavPath = `${tempBase}.wav`
const outputBase = `${tempBase}-whisper`
const srtPath = `${outputBase}.srt`
const jsonPath = `${outputBase}.json`
// Constants for segmentation
const CHUNK_SIZE_MS = 5 * 60 * 1000; // 5 minutes
const OVERLAP_MS = 10 * 1000; // 10 seconds overlap for word boundaries
const startTimeMs = options.startTimeMs || 0;
const totalDurationMs = options.durationMs || 0;
const endTimeMs = totalDurationMs > 0 ? startTimeMs + totalDurationMs : Infinity;
try {
const audioSource = await extractCaptionAudioSource({
videoPath: normalizedVideoPath,
ffmpegPath,
wavPath,
})
console.log('[auto-captions] Starting segmented caption generation sequence')
console.log('[auto-captions] Video:', path.basename(normalizedVideoPath))
console.log('[auto-captions] Range:', `${(startTimeMs/1000).toFixed(2)}s - ${totalDurationMs ? `${((startTimeMs + totalDurationMs)/1000).toFixed(2)}s` : 'End'}`)
const language = options.language && options.language.trim() ? options.language.trim() : 'auto'
const whisperBaseArgs = [
'-m', whisperModelPath,
'-f', wavPath,
'-osrt',
'-of', outputBase,
'-l', language,
'-np',
]
const allCues: any[] = [];
let audioSourceLabel = 'Unknown';
for (let offsetMs = startTimeMs; offsetMs < endTimeMs; offsetMs += CHUNK_SIZE_MS) {
const chunkIndex = Math.floor((offsetMs - startTimeMs) / CHUNK_SIZE_MS);
const tempBase = path.join(app.getPath('temp'), `recordly-captions-chunk-${chunkIndex}-${Date.now()}`)
const wavPath = `${tempBase}.wav`
const outputBase = `${tempBase}-whisper`
const srtPath = `${outputBase}.srt`
const jsonPath = `${outputBase}.json`
let jsonEnabled = true
try {
await execFileAsync(whisperExecutablePath, [...whisperBaseArgs, '-ojf'], {
timeout: 30 * 60 * 1000,
maxBuffer: 20 * 1024 * 1024,
const audioSource = await extractCaptionAudioSource({
videoPath: normalizedVideoPath,
ffmpegPath,
wavPath,
startTime: offsetMs / 1000,
duration: (CHUNK_SIZE_MS + OVERLAP_MS) / 1000
})
} catch (error) {
if (!shouldRetryWhisperWithoutJson(error)) {
throw error
audioSourceLabel = audioSource.label;
const language = options.language && options.language.trim() ? options.language.trim() : 'auto'
const whisperBaseArgs = [
'-m', whisperModelPath,
'-f', wavPath,
'-osrt',
'-of', outputBase,
'-l', language,
'-np',
]
let jsonEnabled = true
const updateChunkProgress = (progress: number) => {
if (totalDurationMs > 0) {
const totalRangeMs = totalDurationMs > 0 ? totalDurationMs : 1;
const rangeOffsetMs = offsetMs - startTimeMs;
const totalProgress = (rangeOffsetMs / totalRangeMs * 100) + (progress / (totalRangeMs / CHUNK_SIZE_MS));
safeSend(webContents, 'auto-caption-progress', { progress: Math.min(99, totalProgress) })
} else {
safeSend(webContents, 'auto-caption-progress', { progress })
}
};
try {
await runWhisperWithProgress(whisperExecutablePath, [...whisperBaseArgs, '-ojf'], updateChunkProgress)
} catch (error) {
if (!shouldRetryWhisperWithoutJson(error)) throw error
jsonEnabled = false
console.warn(`[auto-captions] Whisper runtime error, retrying with SRT: ${error}`)
await runWhisperWithProgress(whisperExecutablePath, whisperBaseArgs, updateChunkProgress)
}
jsonEnabled = false
console.warn('[auto-captions] Whisper runtime does not support JSON full output, retrying with SRT only:', error)
await execFileAsync(whisperExecutablePath, whisperBaseArgs, {
timeout: 30 * 60 * 1000,
maxBuffer: 20 * 1024 * 1024,
})
}
let cues = jsonEnabled
? parseWhisperJsonCues(await fs.readFile(jsonPath, 'utf-8'))
: parseSrtCues(await fs.readFile(srtPath, 'utf-8'))
if (cues.length === 0 && !jsonEnabled) {
// If JSON failed, SRT might be empty or not yet read?
try { cues = parseSrtCues(await fs.readFile(srtPath, 'utf-8')); } catch { /* ignore */ }
}
const timedCues = jsonEnabled
? parseWhisperJsonCues(await fs.readFile(jsonPath, 'utf-8'))
: []
const cues = timedCues.length > 0
? timedCues
: parseSrtCues(await fs.readFile(srtPath, 'utf-8'))
if (cues.length === 0) {
throw new Error('Whisper completed, but no caption cues were produced.')
}
// Adjust timings and deduplicate
const adjustedCues = cues
.map((cue, idx) => ({
...cue,
id: `caption-${offsetMs}-${idx}`,
startMs: cue.startMs + offsetMs,
endMs: cue.endMs + offsetMs
}))
// Only keep cues that START within this chunk's main window (prevent overlap duplicates)
// Except for the very last chunk where we take everything
.filter(cue => {
const isLastChunk = offsetMs + CHUNK_SIZE_MS >= endTimeMs;
if (isLastChunk) return true;
return cue.startMs < offsetMs + CHUNK_SIZE_MS;
});
return {
cues,
audioSourceLabel: audioSource.label,
if (adjustedCues.length > 0) {
allCues.push(...adjustedCues);
safeSend(webContents, 'auto-caption-chunk', { cues: adjustedCues });
}
// If we don't know duration and this was a short chunk, we might be at the end
// Actually, FFmpeg will just produce a short file if duration is past EOS.
const stats = await fs.stat(wavPath).catch(() => null);
if (stats && stats.size < 1000) { // Tiny audio file means we hit the end
break;
}
if (offsetMs + CHUNK_SIZE_MS >= endTimeMs) {
break;
}
} finally {
await Promise.allSettled([
fs.rm(wavPath, { force: true }),
fs.rm(srtPath, { force: true }),
fs.rm(jsonPath, { force: true }),
])
}
} finally {
await Promise.allSettled([
fs.rm(wavPath, { force: true }),
fs.rm(srtPath, { force: true }),
fs.rm(jsonPath, { force: true }),
])
}
safeSend(webContents, 'auto-caption-progress', { progress: 100 })
return {
cues: allCues,
audioSourceLabel,
}
}
@@ -1772,7 +1968,7 @@ function attachWindowsCaptureLifecycle(proc: ChildProcessWithoutNullStreams) {
const sourceName = selectedSource?.name ?? 'Screen'
BrowserWindow.getAllWindows().forEach((window) => {
if (!window.isDestroyed()) {
window.webContents.send('recording-state-changed', {
safeSend(window.webContents, 'recording-state-changed', {
recording: false,
sourceName,
})
@@ -2009,7 +2205,7 @@ async function muxNativeMacRecordingWithAudio(
function emitRecordingInterrupted(reason: string, message: string) {
BrowserWindow.getAllWindows().forEach((window) => {
if (!window.isDestroyed()) {
window.webContents.send('recording-interrupted', { reason, message })
safeSend(window.webContents, 'recording-interrupted', { reason, message })
}
})
}
@@ -2017,7 +2213,7 @@ function emitRecordingInterrupted(reason: string, message: string) {
function emitCursorStateChanged(cursorType: CursorVisualType) {
BrowserWindow.getAllWindows().forEach((window) => {
if (!window.isDestroyed()) {
window.webContents.send('cursor-state-changed', { cursorType })
safeSend(window.webContents, 'cursor-state-changed', { cursorType })
}
})
}
@@ -2053,7 +2249,7 @@ function attachNativeCaptureLifecycle(process: ChildProcessWithoutNullStreams) {
const sourceName = selectedSource?.name ?? 'Screen'
BrowserWindow.getAllWindows().forEach((window) => {
if (!window.isDestroyed()) {
window.webContents.send('recording-state-changed', {
safeSend(window.webContents, 'recording-state-changed', {
recording: false,
sourceName,
})
@@ -3033,6 +3229,18 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
config.displayId = Number.isFinite(screenId) && screenId > 0
? screenId
: Number(getScreen().getPrimaryDisplay().id)
// Include display bounds for more robust matching in the native helper
const allDisplays = getScreen().getAllDisplays()
const display = allDisplays.find((d) => String(d.id) === String(config.displayId))
|| getScreen().getPrimaryDisplay()
if (display) {
config.displayX = Math.round(display.bounds.x)
config.displayY = Math.round(display.bounds.y)
config.displayW = Math.round(display.bounds.width)
config.displayH = Math.round(display.bounds.height)
}
}
windowsCaptureOutputBuffer = ''
@@ -3589,7 +3797,7 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
const source = selectedSource || { name: 'Screen' }
BrowserWindow.getAllWindows().forEach((window) => {
if (!window.isDestroyed()) {
window.webContents.send('recording-state-changed', {
safeSend(window.webContents, 'recording-state-changed', {
recording,
sourceName: source.name,
})
@@ -3931,57 +4139,59 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
}
})
ipcMain.handle('get-whisper-small-model-status', async () => {
ipcMain.handle('get-whisper-model-status', async (event, modelName: string) => {
try {
return await getWhisperSmallModelStatus()
return await getWhisperModelStatus(event, modelName)
} catch (error) {
return { success: false, exists: false, path: null, error: String(error) }
}
})
ipcMain.handle('download-whisper-small-model', async (event) => {
ipcMain.handle('download-whisper-model', async (event, modelName: string) => {
try {
const existing = await getWhisperSmallModelStatus()
const existing = await getWhisperModelStatus(event, modelName)
if (existing.exists) {
sendWhisperModelDownloadProgress(event.sender, {
status: 'downloaded',
progress: 100,
model: modelName,
path: existing.path,
})
return { success: true, path: existing.path, alreadyDownloaded: true }
}
const modelPath = await downloadWhisperSmallModel(event.sender)
const modelPath = await downloadWhisperModel(event.sender, modelName)
return { success: true, path: modelPath }
} catch (error) {
console.error('Failed to download Whisper small model:', error)
console.error(`Failed to download Whisper model ${modelName}:`, error)
return { success: false, error: String(error) }
}
})
ipcMain.handle('delete-whisper-small-model', async (event) => {
ipcMain.handle('delete-whisper-model', async (event, modelName: string) => {
try {
await deleteWhisperSmallModel()
await deleteWhisperModel(event, modelName)
sendWhisperModelDownloadProgress(event.sender, {
status: 'idle',
progress: 0,
model: modelName,
path: null,
})
return { success: true }
} catch (error) {
console.error('Failed to delete Whisper small model:', error)
console.error(`Failed to delete Whisper model ${modelName}:`, error)
return { success: false, error: String(error) }
}
})
ipcMain.handle('generate-auto-captions', async (_, options: {
ipcMain.handle('generate-auto-captions', async (event, options: {
videoPath: string
whisperExecutablePath: string
whisperModelPath: string
language?: string
}) => {
try {
const result = await generateAutoCaptionsFromVideo(options)
const result = await generateAutoCaptionsFromVideo(event.sender, options)
return {
success: true,
cues: result.cues,
@@ -4404,7 +4614,7 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
let remaining = seconds
countdownRemaining = remaining
countdownWin.webContents.send('countdown-tick', remaining)
safeSend(countdownWin.webContents, 'countdown-tick', remaining)
countdownTimer = setInterval(() => {
if (countdownCancelled) {
@@ -4433,9 +4643,11 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
resolve({ success: true })
} else {
const win = getCountdownWindow()
if (win && !win.isDestroyed()) {
win.webContents.send('countdown-tick', remaining)
}
try {
if (win && !win.isDestroyed()) {
safeSend(win.webContents, 'countdown-tick', remaining)
}
} catch {}
}
}, 1000)
})
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,6 @@
{
"version": "v1.8.4",
"platform": "win32",
"arch": "x64",
"binary": "whisper-cli.exe"
}
Binary file not shown.
+23
View File
@@ -33,6 +33,11 @@ struct CaptureConfig {
int fps = 60;
int width = 0;
int height = 0;
int displayX = 0;
int displayY = 0;
int displayW = 0;
int displayH = 0;
bool hasDisplayBounds = false;
bool captureSystemAudio = false;
bool captureMic = false;
};
@@ -127,6 +132,18 @@ static bool parseSimpleJson(const std::string& json, CaptureConfig& config) {
config.captureSystemAudio = findBool("captureSystemAudio");
config.captureMic = findBool("captureMic");
int dx = findInt("displayX");
int dy = findInt("displayY");
int dw = findInt("displayW");
int dh = findInt("displayH");
if (dw > 0 && dh > 0) {
config.displayX = dx;
config.displayY = dy;
config.displayW = dw;
config.displayH = dh;
config.hasDisplayBounds = true;
}
return true;
}
@@ -198,6 +215,12 @@ int main(int argc, char* argv[]) {
}
} else {
HMONITOR monitor = findMonitorByDisplayId(config.displayId);
if (!monitor && config.hasDisplayBounds) {
std::cerr << "Monitor ID match failed, attempting coordinate-based match: "
<< config.displayX << "," << config.displayY << " " << config.displayW << "x" << config.displayH << std::endl;
monitor = findMonitorByBounds(config.displayX, config.displayY, config.displayW, config.displayH);
}
if (!monitor) {
std::cerr << "ERROR: Could not find monitor for displayId " << config.displayId << std::endl;
return 1;
@@ -1,5 +1,6 @@
#include "monitor_utils.h"
#include <ShellScalingApi.h>
#include <iostream>
static BOOL CALLBACK enumMonitorCallback(HMONITOR hMonitor, HDC, LPRECT, LPARAM lParam) {
auto* monitors = reinterpret_cast<std::vector<MonitorInfo>*>(lParam);
@@ -26,16 +27,56 @@ std::vector<MonitorInfo> enumerateMonitors() {
return monitors;
}
static std::string wstringToString(const std::wstring& wstr) {
if (wstr.empty()) return "";
int size_needed = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), (int)wstr.size(), NULL, 0, NULL, NULL);
std::string str(size_needed, 0);
WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), (int)wstr.size(), &str[0], size_needed, NULL, NULL);
return str;
}
// Electron uses the HMONITOR handle value cast to a number as the display ID.
HMONITOR findMonitorByDisplayId(int64_t displayId) {
auto monitors = enumerateMonitors();
// Try exact match first
for (const auto& m : monitors) {
if (static_cast<int64_t>(reinterpret_cast<intptr_t>(m.handle)) == displayId) {
return m.handle;
}
}
// No exact ID match found. Return nullptr to allow main.cpp to try coordinate-based matching.
return nullptr;
}
HMONITOR findMonitorByBounds(int x, int y, int width, int height) {
auto monitors = enumerateMonitors();
// 1. Try exact bounds match
for (const auto& m : monitors) {
if (m.x == x && m.y == y && m.width == width && m.height == height) {
std::cerr << "Found monitor by exact bounds: " << x << "," << y << " " << width << "x" << height << std::endl;
return m.handle;
}
}
// 2. Try matching top-left point (sometimes size differs due to DPI scaling, but top-left is usually more stable in screen-space)
for (const auto& m : monitors) {
if (m.x == x && m.y == y) {
std::cerr << "Found monitor by top-left point match: " << x << "," << y << std::endl;
return m.handle;
}
}
// 3. Last resort: MonitorFromRect/Point (OS choice for the best matching monitor for these coordinates)
RECT rect = { x, y, x + width, y + height };
HMONITOR hMonitor = MonitorFromRect(&rect, MONITOR_DEFAULTTONULL);
if (hMonitor) {
std::cerr << "Found monitor via Windows OS MonitorFromRect fallback" << std::endl;
return hMonitor;
}
return nullptr;
}
@@ -16,4 +16,5 @@ struct MonitorInfo {
std::vector<MonitorInfo> enumerateMonitors();
HMONITOR findMonitorByDisplayId(int64_t displayId);
HMONITOR findMonitorByBounds(int x, int y, int width, int height);
MonitorInfo getMonitorInfo(HMONITOR monitor);
+35 -11
View File
@@ -164,24 +164,36 @@ contextBridge.exposeInMainWorld("electronAPI", {
openWhisperModelPicker: () => {
return ipcRenderer.invoke("open-whisper-model-picker");
},
getWhisperSmallModelStatus: () => {
return ipcRenderer.invoke("get-whisper-small-model-status");
getWhisperModelStatus: (modelName: string) => {
return ipcRenderer.invoke("get-whisper-model-status", modelName);
},
downloadWhisperSmallModel: () => {
return ipcRenderer.invoke("download-whisper-small-model");
downloadWhisperModel: (modelName: string) => {
return ipcRenderer.invoke("download-whisper-model", modelName);
},
deleteWhisperSmallModel: () => {
return ipcRenderer.invoke("delete-whisper-small-model");
deleteWhisperModel: (modelName: string) => {
return ipcRenderer.invoke("delete-whisper-model", modelName);
},
onWhisperSmallModelDownloadProgress: (
callback: (state: { status: "idle" | "downloading" | "downloaded" | "error"; progress: number; path?: string | null; error?: string }) => void,
onWhisperModelDownloadProgress: (
callback: (state: {
status: "idle" | "downloading" | "downloaded" | "error";
progress: number;
model: string;
path?: string | null;
error?: string;
}) => void,
) => {
const listener = (
_event: Electron.IpcRendererEvent,
payload: { status: "idle" | "downloading" | "downloaded" | "error"; progress: number; path?: string | null; error?: string },
payload: {
status: "idle" | "downloading" | "downloaded" | "error";
progress: number;
model: string;
path?: string | null;
error?: string;
},
) => callback(payload);
ipcRenderer.on("whisper-small-model-download-progress", listener);
return () => ipcRenderer.removeListener("whisper-small-model-download-progress", listener);
ipcRenderer.on("whisper-model-download-progress", listener);
return () => ipcRenderer.removeListener("whisper-model-download-progress", listener);
},
generateAutoCaptions: (options: {
videoPath: string;
@@ -191,6 +203,18 @@ contextBridge.exposeInMainWorld("electronAPI", {
}) => {
return ipcRenderer.invoke("generate-auto-captions", options);
},
onAutoCaptionProgress: (callback: (payload: { progress: number }) => void) => {
const listener = (_event: Electron.IpcRendererEvent, payload: { progress: number }) =>
callback(payload);
ipcRenderer.on("auto-caption-progress", listener);
return () => ipcRenderer.removeListener("auto-caption-progress", listener);
},
onAutoCaptionChunk: (callback: (payload: { cues: CaptionCue[] }) => void) => {
const listener = (_event: Electron.IpcRendererEvent, payload: { cues: CaptionCue[] }) =>
callback(payload);
ipcRenderer.on("auto-caption-chunk", listener);
return () => ipcRenderer.removeListener("auto-caption-chunk", listener);
},
setCurrentVideoPath: (path: string) => {
return ipcRenderer.invoke("set-current-video-path", path);
},
+6 -2
View File
@@ -246,7 +246,9 @@ export function createHudOverlayWindow(): BrowserWindow {
}
win.webContents.on("did-finish-load", () => {
win?.webContents.send("main-process-message", new Date().toLocaleString());
if (!win.isDestroyed()) {
win.webContents.send("main-process-message", new Date().toLocaleString());
}
setTimeout(() => {
if (!win.isDestroyed()) {
win.show();
@@ -310,7 +312,9 @@ export function createEditorWindow(): BrowserWindow {
});
win.webContents.on("did-finish-load", () => {
win?.webContents.send("main-process-message", new Date().toLocaleString());
if (!win.isDestroyed()) {
win.webContents.send("main-process-message", new Date().toLocaleString());
}
});
if (VITE_DEV_SERVER_URL) {
+2 -1
View File
@@ -20,6 +20,7 @@
"build": "npm run build:platform-native-helpers && tsc && vite build && electron-builder",
"lint": "biome check .",
"lint:fix": "biome check --write .",
"typecheck": "tsc --noEmit",
"format": "biome format --write .",
"preview": "vite preview",
"rebuild:native": "node ./node_modules/electron-rebuild/lib/src/cli.js --force --only uiohook-napi",
@@ -104,4 +105,4 @@
"web-demuxer": "^4.0.0"
},
"main": "dist-electron/main.js"
}
}
+8 -1
View File
@@ -25,6 +25,13 @@ function findCmake() {
// not on PATH
}
// Local .cmake_ext path
const localCmake = path.join(projectRoot, ".cmake_ext", "cmake-4.3.0-windows-x86_64", "bin", "cmake.exe");
if (existsSync(localCmake)) {
return `"${localCmake}"`;
}
// Standalone CMake paths
const standaloneCmakePaths = [
path.join("C:", "Program Files", "CMake", "bin", "cmake.exe"),
path.join("C:", "Program Files (x86)", "CMake", "bin", "cmake.exe"),
@@ -35,7 +42,7 @@ function findCmake() {
}
}
// VS 2022 bundled CMake
// VS 2022/2019 bundled CMake
const vsRoots = [
path.join("C:", "Program Files", "Microsoft Visual Studio"),
path.join("C:", "Program Files (x86)", "Microsoft Visual Studio"),
+7 -1
View File
@@ -137,6 +137,12 @@ function findCmake() {
}
if (process.platform === "win32") {
// Local .cmake_ext path
const localCmake = path.join(projectRoot, ".cmake_ext", "cmake-4.3.0-windows-x86_64", "bin", "cmake.exe");
if (existsSync(localCmake)) {
return localCmake;
}
const vsEditions = ["Community", "Professional", "Enterprise", "BuildTools"];
for (const edition of vsEditions) {
const cmakePath = path.join(
@@ -236,7 +242,7 @@ async function ensureSourceTree() {
}
ensureTarAvailable();
execFileSync("tar", ["-xzf", archivePath, "-C", extractRoot], { stdio: "inherit" });
execFileSync("tar", ["-xzf", path.relative(projectRoot, archivePath), "-C", path.relative(projectRoot, extractRoot)], { stdio: "inherit" });
if (!existsSync(path.join(extractedSourceDir, "CMakeLists.txt"))) {
throw new Error(
+103 -94
View File
@@ -1,127 +1,136 @@
import { execSync } from 'node:child_process';
import { mkdirSync, existsSync, rmSync } from 'node:fs';
import path from 'node:path';
import { execSync } from "node:child_process";
import { mkdirSync, existsSync, rmSync } from "node:fs";
import path from "node:path";
const projectRoot = process.cwd();
const sourceDir = path.join(projectRoot, 'electron', 'native', 'wgc-capture');
const buildDir = path.join(sourceDir, 'build');
const sourceDir = path.join(projectRoot, "electron", "native", "wgc-capture");
const buildDir = path.join(sourceDir, "build");
if (process.platform !== 'win32') {
console.log('[build-windows-capture] Skipping native Windows capture build: host platform is not Windows.');
process.exit(0);
if (process.platform !== "win32") {
console.log("[build-windows-capture] Skipping native Windows capture build: host platform is not Windows.");
process.exit(0);
}
if (!existsSync(path.join(sourceDir, 'CMakeLists.txt'))) {
console.error('[build-windows-capture] CMakeLists.txt not found at', sourceDir);
process.exit(1);
if (!existsSync(path.join(sourceDir, "CMakeLists.txt"))) {
console.error("[build-windows-capture] CMakeLists.txt not found at", sourceDir);
process.exit(1);
}
function findCmake() {
// Check PATH first
try {
execSync('cmake --version', { stdio: 'pipe' });
return 'cmake';
} catch {
// not on PATH
}
// Check PATH first
try {
execSync("cmake --version", { stdio: "pipe" });
return "cmake";
} catch {
// not on PATH
}
const standaloneCmakePaths = [
path.join('C:', 'Program Files', 'CMake', 'bin', 'cmake.exe'),
path.join('C:', 'Program Files (x86)', 'CMake', 'bin', 'cmake.exe'),
];
for (const cmakePath of standaloneCmakePaths) {
if (existsSync(cmakePath)) {
return `"${cmakePath}"`;
}
}
// Local .cmake_ext path
const localCmake = path.join(projectRoot, ".cmake_ext", "cmake-4.3.0-windows-x86_64", "bin", "cmake.exe");
if (existsSync(localCmake)) {
return `"${localCmake}"`;
}
// VS 2022 bundled CMake
const vsRoots = [
path.join('C:', 'Program Files', 'Microsoft Visual Studio'),
path.join('C:', 'Program Files (x86)', 'Microsoft Visual Studio'),
];
const vsEditions = ['Community', 'Professional', 'Enterprise', 'BuildTools'];
const vsVersions = ['2022', '2019'];
for (const root of vsRoots) {
for (const version of vsVersions) {
for (const edition of vsEditions) {
const cmakePath = path.join(
root,
version,
edition,
'Common7',
'IDE',
'CommonExtensions',
'Microsoft',
'CMake',
'CMake',
'bin',
'cmake.exe'
);
if (existsSync(cmakePath)) {
return `"${cmakePath}"`;
}
}
}
}
// Standalone CMake paths
const standaloneCmakePaths = [
path.join("C:", "Program Files", "CMake", "bin", "cmake.exe"),
path.join("C:", "Program Files (x86)", "CMake", "bin", "cmake.exe"),
];
for (const cmakePath of standaloneCmakePaths) {
if (existsSync(cmakePath)) {
return `"${cmakePath}"`;
}
}
return null;
// VS 2022/2019 bundled CMake
const vsRoots = [
path.join("C:", "Program Files", "Microsoft Visual Studio"),
path.join("C:", "Program Files (x86)", "Microsoft Visual Studio"),
];
const vsEditions = ["Community", "Professional", "Enterprise", "BuildTools"];
const vsVersions = ["2022", "2019"];
for (const root of vsRoots) {
for (const version of vsVersions) {
for (const edition of vsEditions) {
const cmakePath = path.join(
root,
version,
edition,
"Common7",
"IDE",
"CommonExtensions",
"Microsoft",
"CMake",
"CMake",
"bin",
"cmake.exe",
);
if (existsSync(cmakePath)) {
return `"${cmakePath}"`;
}
}
}
}
return null;
}
const cmake = findCmake();
if (!cmake) {
console.error('[build-windows-capture] CMake not found. Install Visual Studio with C++ CMake tools or standalone CMake.');
process.exit(1);
console.error(
"[build-windows-capture] CMake not found. Install Visual Studio with C++ CMake tools or standalone CMake.",
);
process.exit(1);
}
mkdirSync(buildDir, { recursive: true });
const cacheFile = path.join(buildDir, 'CMakeCache.txt');
const cacheDir = path.join(buildDir, 'CMakeFiles');
const cacheFile = path.join(buildDir, "CMakeCache.txt");
const cacheDir = path.join(buildDir, "CMakeFiles");
function clearCmakeCache() {
rmSync(cacheFile, { force: true });
rmSync(cacheDir, { recursive: true, force: true });
rmSync(cacheFile, { force: true });
rmSync(cacheDir, { recursive: true, force: true });
}
console.log('[build-windows-capture] Configuring CMake...');
console.log("[build-windows-capture] Configuring CMake...");
try {
clearCmakeCache();
execSync(`${cmake} .. -G "Visual Studio 17 2022" -A x64`, {
cwd: buildDir,
stdio: 'inherit',
timeout: 120000,
});
clearCmakeCache();
execSync(`${cmake} .. -G "Visual Studio 17 2022" -A x64`, {
cwd: buildDir,
stdio: "inherit",
timeout: 120000,
});
} catch {
console.log('[build-windows-capture] VS 2022 generator not found, trying VS 2019...');
try {
clearCmakeCache();
execSync(`${cmake} .. -G "Visual Studio 16 2019" -A x64`, {
cwd: buildDir,
stdio: 'inherit',
timeout: 120000,
});
} catch (innerError) {
console.error('[build-windows-capture] CMake configure failed:', innerError.message);
process.exit(1);
}
console.log("[build-windows-capture] VS 2022 generator not found, trying VS 2019...");
try {
clearCmakeCache();
execSync(`${cmake} .. -G "Visual Studio 16 2019" -A x64`, {
cwd: buildDir,
stdio: "inherit",
timeout: 120000,
});
} catch (innerError) {
console.error("[build-windows-capture] CMake configure failed:", innerError.message);
process.exit(1);
}
}
console.log('[build-windows-capture] Building native Windows capture helper...');
console.log("[build-windows-capture] Building native Windows capture helper...");
try {
execSync(`${cmake} --build . --config Release`, {
cwd: buildDir,
stdio: 'inherit',
timeout: 300000,
});
execSync(`${cmake} --build . --config Release`, {
cwd: buildDir,
stdio: "inherit",
timeout: 300000,
});
} catch (error) {
console.error('[build-windows-capture] Build failed:', error.message);
process.exit(1);
console.error("[build-windows-capture] Build failed:", error.message);
process.exit(1);
}
const exePath = path.join(buildDir, 'Release', 'wgc-capture.exe');
const exePath = path.join(buildDir, "Release", "wgc-capture.exe");
if (existsSync(exePath)) {
console.log(`[build-windows-capture] Built successfully: ${exePath}`);
console.log(`[build-windows-capture] Built successfully: ${exePath}`);
} else {
console.error('[build-windows-capture] Expected exe not found at', exePath);
process.exit(1);
console.error("[build-windows-capture] Expected exe not found at", exePath);
process.exit(1);
}
@@ -111,6 +111,17 @@ export function AnnotationOverlay({
</div>
);
case 'blur':
return (
<div
className="w-full h-full rounded-lg backdrop-blur-md"
style={{
backdropFilter: `blur(${annotation.blurIntensity ?? 12}px)`,
WebkitBackdropFilter: `blur(${annotation.blurIntensity ?? 12}px)`
}}
/>
);
default:
return null;
}
@@ -208,6 +219,7 @@ export function AnnotationOverlay({
annotation.type === 'text' && "bg-transparent",
annotation.type === 'image' && "bg-transparent",
annotation.type === 'figure' && "bg-transparent",
annotation.type === 'blur' && "bg-transparent",
isSelected && "shadow-lg"
)}
>
@@ -1,10 +1,10 @@
import { useRef, useState, useEffect, useMemo } from "react";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Button } from "@/components/ui/button";
import { Trash2, Type, Image as ImageIcon, Upload, Bold, Italic, Underline, AlignLeft, AlignCenter, AlignRight, ChevronDown, Info } from "lucide-react";
import { Trash2, Type, Image as ImageIcon, Upload, Bold, Italic, Underline, AlignLeft, AlignCenter, AlignRight, ChevronDown, Info, Droplets } from "lucide-react";
import { toast } from "sonner";
import Block from '@uiw/react-color-block';
import type { AnnotationRegion, AnnotationType, ArrowDirection, FigureData } from "./types";
import { DEFAULT_BLUR_INTENSITY, type AnnotationRegion, type AnnotationType, type ArrowDirection, type FigureData } from "./types";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
@@ -21,6 +21,7 @@ interface AnnotationSettingsPanelProps {
onTypeChange: (type: AnnotationType) => void;
onStyleChange: (style: Partial<AnnotationRegion['style']>) => void;
onFigureDataChange?: (figureData: FigureData) => void;
onBlurIntensityChange?: (intensity: number) => void;
onDelete: () => void;
}
@@ -43,6 +44,7 @@ export function AnnotationSettingsPanel({
onTypeChange,
onStyleChange,
onFigureDataChange,
onBlurIntensityChange,
onDelete,
}: AnnotationSettingsPanelProps) {
const t = useScopedT('editor');
@@ -128,7 +130,7 @@ export function AnnotationSettingsPanel({
{/* Type Selector */}
<Tabs value={annotation.type} onValueChange={(value) => onTypeChange(value as AnnotationType)} className="mb-6">
<TabsList className="mb-4 bg-white/5 border border-white/5 p-1 w-full grid grid-cols-3 h-auto rounded-xl">
<TabsList className="mb-4 bg-white/5 border border-white/5 p-1 w-full grid grid-cols-4 h-auto rounded-xl">
<TabsTrigger value="text" className="data-[state=active]:bg-[#2563EB] data-[state=active]:text-white text-slate-400 py-2 rounded-lg transition-all gap-2">
<Type className="w-4 h-4" />
{t('annotations.text')}
@@ -143,8 +145,32 @@ export function AnnotationSettingsPanel({
</svg>
{t('annotations.arrow')}
</TabsTrigger>
<TabsTrigger value="blur" className="data-[state=active]:bg-[#2563EB] data-[state=active]:text-white text-slate-400 py-2 rounded-lg transition-all gap-2">
<Droplets className="w-4 h-4" />
Blur
</TabsTrigger>
</TabsList>
<TabsContent value="blur" className="mt-0 space-y-4">
<div>
<label className="text-xs font-medium text-slate-200 mb-2 block">
Blur Intensity: {annotation.blurIntensity ?? DEFAULT_BLUR_INTENSITY}px
</label>
<Slider
value={[annotation.blurIntensity ?? DEFAULT_BLUR_INTENSITY]}
onValueChange={([value]) => {
if (onBlurIntensityChange) {
onBlurIntensityChange(value);
}
}}
min={2}
max={50}
step={1}
className="w-full"
/>
</div>
</TabsContent>
{/* Text Content */}
<TabsContent value="text" className="mt-0 space-y-4">
<div>
@@ -0,0 +1,221 @@
import { Volume2, VolumeX, Trash2, Music } from "lucide-react";
import { Button } from "@/components/ui/button";
import { Slider } from "@/components/ui/slider";
import { cn } from "@/lib/utils";
import { generateWaveform } from "@/utils/audioWaveform";
import { useEffect, useState } from "react";
import type { AudioRegion } from "./types";
interface AudioSettingsPanelProps {
audio: AudioRegion;
onVolumeChange: (volume: number) => void;
onMutedChange: (muted: boolean) => void;
onSoloedChange: (soloed: boolean) => void;
onFadeInMsChange: (ms: number) => void;
onFadeOutMsChange: (ms: number) => void;
onDelete: () => void;
}
function formatFadeTime(ms: number): string {
if (ms === 0) return "Off";
if (ms < 1000) return `${ms}ms`;
return `${(ms / 1000).toFixed(1)}s`;
}
export function AudioSettingsPanel({
audio,
onVolumeChange,
onMutedChange,
onSoloedChange,
onFadeInMsChange,
onFadeOutMsChange,
onDelete,
}: AudioSettingsPanelProps) {
const [waveform, setWaveform] = useState<number[] | null>(null);
useEffect(() => {
let active = true;
if (audio.audioPath) {
generateWaveform(audio.audioPath, 120).then(result => {
if (active) setWaveform(result);
});
}
return () => { active = false; };
}, [audio.audioPath]);
const clipDurationMs = audio.endMs - audio.startMs;
const maxFadeMs = Math.max(0, Math.floor(clipDurationMs / 2));
const volumePct = Math.round(audio.volume * 100);
const isMaster = audio.id === "master";
// Mute and Solo are mutually exclusive
const handleMuteToggle = () => {
const nextMuted = !audio.muted;
onMutedChange(nextMuted);
if (nextMuted && audio.soloed) onSoloedChange(false);
};
const handleSoloToggle = () => {
const nextSoloed = !audio.soloed;
onSoloedChange(nextSoloed);
if (nextSoloed && audio.muted) onMutedChange(false);
};
return (
<section className="flex flex-col gap-3 pb-4">
{/* Header */}
<div className="flex items-center justify-between gap-2 pb-1">
<div className="flex items-center gap-2">
<div className="p-1.5 rounded-lg bg-purple-500/10 text-purple-400 shrink-0">
<Music className="w-3.5 h-3.5" />
</div>
<div>
<p className="text-sm font-semibold text-slate-100 leading-none">
{isMaster ? "Original Audio" : "Audio Region"}
</p>
{!isMaster && (
<p className="text-[10px] text-slate-500 mt-0.5 truncate max-w-[160px]">
{audio.audioPath.split(/[\\/]/).pop()}
</p>
)}
{isMaster && (
<p className="text-[10px] text-slate-500 mt-0.5">
Adjust the volume of the video's audio
</p>
)}
</div>
</div>
<span className="text-[9px] uppercase tracking-widest font-semibold text-[#2563EB] bg-[#2563EB]/10 px-2 py-1 rounded-full shrink-0">
Active
</span>
</div>
{/* Waveform — only for audio regions with a dedicated audio path */}
{waveform && !isMaster && (
<div className="h-10 bg-white/[0.03] rounded-xl border border-white/5 flex items-center overflow-hidden relative">
<div className="absolute inset-0 flex items-center pointer-events-none px-2">
<svg
width="100%"
height="100%"
viewBox={`0 0 ${waveform.length} 100`}
preserveAspectRatio="none"
className="text-purple-400 opacity-50"
>
{waveform.map((peak, i) => (
<rect
key={i}
x={i}
y={50 - peak * 50}
width={0.8}
height={peak * 100}
fill="currentColor"
rx={0.2}
/>
))}
</svg>
</div>
</div>
)}
{/* Mute / Solo — only for audio regions, not master */}
{!isMaster && (
<div className="grid grid-cols-2 gap-2">
<button
type="button"
onClick={handleMuteToggle}
className={cn(
"flex items-center gap-2 px-3 py-2 rounded-xl border text-xs font-medium transition-all",
audio.muted
? "bg-red-500/15 border-red-500/30 text-red-400"
: "bg-white/[0.03] border-white/[0.08] text-slate-400 hover:bg-white/[0.06] hover:text-slate-200"
)}
>
<VolumeX className="w-3.5 h-3.5 shrink-0" />
<span>Mute</span>
</button>
<button
type="button"
onClick={handleSoloToggle}
className={cn(
"flex items-center gap-2 px-3 py-2 rounded-xl border text-xs font-medium transition-all",
audio.soloed
? "bg-amber-500/15 border-amber-500/30 text-amber-400"
: "bg-white/[0.03] border-white/[0.08] text-slate-400 hover:bg-white/[0.06] hover:text-slate-200"
)}
>
<span className="text-[11px] font-bold w-3.5 text-center shrink-0">S</span>
<span>Solo</span>
</button>
</div>
)}
{/* Volume */}
<div className="rounded-xl bg-white/[0.03] border border-white/5 px-3 py-2.5 space-y-3">
<div className="flex items-center justify-between">
<div className="flex items-center gap-1.5">
<Volume2 className="w-3.5 h-3.5 text-slate-500" />
<span className="text-xs font-medium text-slate-300">Volume</span>
</div>
<span
className={cn(
"text-[11px] tabular-nums font-semibold px-1.5 py-0.5 rounded-md",
volumePct > 100
? "text-amber-400 bg-amber-500/10"
: "text-[#2563EB] bg-[#2563EB]/10"
)}
>
{volumePct}%
</span>
</div>
<Slider
value={[audio.volume * 100]}
onValueChange={([value]) => onVolumeChange(value / 100)}
min={0}
max={200}
step={1}
/>
{volumePct > 100 && (
<p className="text-[10px] text-amber-500/70 leading-snug">
Amplifying above 100% may clip the audio.
</p>
)}
</div>
{/* Fades — only for audio regions */}
{!isMaster && (
<div className="rounded-xl bg-white/[0.03] border border-white/5 px-3 py-2.5 space-y-3">
<span className="text-xs font-medium text-slate-300">Fades</span>
<div className="grid grid-cols-2 gap-x-4 gap-y-3">
<div className="space-y-2">
<div className="flex items-center justify-between">
<span className="text-[10px] text-slate-500 uppercase tracking-wider font-semibold">Fade In</span>
<span className="text-[10px] tabular-nums text-slate-400 font-medium">{formatFadeTime(audio.fadeInMs || 0)}</span>
</div>
<Slider value={[audio.fadeInMs || 0]} onValueChange={([v]) => onFadeInMsChange(v)} min={0} max={maxFadeMs} step={50} />
</div>
<div className="space-y-2">
<div className="flex items-center justify-between">
<span className="text-[10px] text-slate-500 uppercase tracking-wider font-semibold">Fade Out</span>
<span className="text-[10px] tabular-nums text-slate-400 font-medium">{formatFadeTime(audio.fadeOutMs || 0)}</span>
</div>
<Slider value={[audio.fadeOutMs || 0]} onValueChange={([v]) => onFadeOutMsChange(v)} min={0} max={maxFadeMs} step={50} />
</div>
</div>
</div>
)}
{/* Delete — only for audio regions */}
{!isMaster && (
<Button
onClick={onDelete}
variant="ghost"
size="sm"
className="w-full gap-2 text-red-400/70 hover:text-red-400 hover:bg-red-500/10 border border-transparent hover:border-red-500/20 transition-all mt-1"
>
<Trash2 className="w-3.5 h-3.5" />
Remove Audio Region
</Button>
)}
</section>
);
}
+283 -26
View File
@@ -1,4 +1,4 @@
import { Palette, Trash2, Upload, X } from "lucide-react";
import { MessageSquare, Music, Palette, Trash2, Upload, X } from "lucide-react";
import { AnimatePresence, LayoutGroup, motion } from "motion/react";
import { useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
@@ -26,11 +26,13 @@ import parchedCursorUrl from "../../assets/cursors/parched/default.png";
import turtleCursorUrl from "../../assets/cursors/turtle/default.png";
import { useI18n, useScopedT } from "../../contexts/I18nContext";
import { AnnotationSettingsPanel } from "./AnnotationSettingsPanel";
import { AudioSettingsPanel } from "./AudioSettingsPanel";
import { loadEditorPreferences, saveEditorPreferences } from "./editorPreferences";
import { SliderControl } from "./SliderControl";
import type {
AnnotationRegion,
AnnotationType,
AudioRegion,
AutoCaptionAnimation,
AutoCaptionSettings,
CaptionCue,
@@ -111,6 +113,7 @@ export type EditorEffectSection =
| "cursor"
| "captions"
| "webcam"
| "audio"
| "zoom"
| "frame"
| "crop";
@@ -208,8 +211,13 @@ interface SettingsPanelProps {
onAnnotationTypeChange?: (id: string, type: AnnotationType) => void;
onAnnotationStyleChange?: (id: string, style: Partial<AnnotationRegion["style"]>) => void;
onAnnotationFigureDataChange?: (id: string, figureData: FigureData) => void;
onAnnotationBlurIntensityChange?: (id: string, blurIntensity: number) => void;
onAnnotationDelete?: (id: string) => void;
onSeek?: (time: number) => void;
autoCaptions?: CaptionCue[];
onAutoCaptionsChange?: (captions: CaptionCue[]) => void;
selectedCaptionId?: string | null;
onSelectCaption?: (id: string | null) => void;
autoCaptionSettings?: AutoCaptionSettings;
whisperExecutablePath?: string | null;
whisperModelPath?: string | null;
@@ -221,12 +229,31 @@ interface SettingsPanelProps {
onPickWhisperModel?: () => void;
onGenerateAutoCaptions?: () => void;
onClearAutoCaptions?: () => void;
onDownloadWhisperSmallModel?: () => void;
onDeleteWhisperSmallModel?: () => void;
autoCaptionProgress?: number;
onDownloadWhisperModel?: () => void;
onDeleteWhisperModel?: () => void;
selectedSpeedId?: string | null;
selectedSpeedValue?: PlaybackSpeed | null;
onSpeedChange?: (speed: PlaybackSpeed) => void;
onSpeedDelete?: (id: string) => void;
audioRegions?: AudioRegion[];
selectedAudioId?: string | null;
onAudioVolumeChange?: (id: string, volume: number) => void;
onAudioMutedChange?: (id: string, muted: boolean) => void;
onAudioSoloedChange?: (id: string, soloed: boolean) => void;
onAudioFadeInMsChange?: (id: string, ms: number) => void;
onAudioFadeOutMsChange?: (id: string, ms: number) => void;
onAudioDelete?: (id: string) => void;
timeSelection?: { startMs: number; endMs: number } | null;
isMasterSelected?: boolean;
masterAudioVolume?: number;
masterAudioMuted?: boolean;
masterAudioSoloed?: boolean;
videoDuration?: number;
videoPath?: string;
onMasterAudioVolumeChange?: (volume: number) => void;
onMasterAudioMutedChange?: (muted: boolean) => void;
onMasterAudioSoloedChange?: (soloed: boolean) => void;
}
export default SettingsPanel;
@@ -278,8 +305,24 @@ const CAPTION_LANGUAGE_OPTIONS = [
{ value: "zh", label: "Chinese" },
{ value: "ja", label: "Japanese" },
{ value: "ko", label: "Korean" },
{ value: "id", label: "Indonesian" },
] as const;
export type WhisperModelInfo = {
value: "tiny" | "base" | "small" | "medium" | "large" | "custom";
label: string;
size: string;
};
const WHISPER_MODEL_OPTIONS: WhisperModelInfo[] = [
{ value: "tiny", label: "Tiny", size: "75 MB" },
{ value: "base", label: "Base", size: "142 MB" },
{ value: "small", label: "Small", size: "466 MB" },
{ value: "medium", label: "Medium", size: "1.5 GB" },
{ value: "large", label: "Large (v3)", size: "2.9 GB" },
{ value: "custom", label: "Custom", size: "Local File" },
];
function loadPreviewImage(url: string) {
return new Promise<HTMLImageElement>((resolve, reject) => {
const image = new Image();
@@ -545,8 +588,12 @@ export function SettingsPanel({
onAnnotationTypeChange,
onAnnotationStyleChange,
onAnnotationFigureDataChange,
onAnnotationBlurIntensityChange,
onAnnotationDelete,
onSeek,
autoCaptions = [],
onAutoCaptionsChange,
autoCaptionProgress = 0,
autoCaptionSettings = DEFAULT_AUTO_CAPTION_SETTINGS,
whisperModelPath,
whisperModelDownloadStatus = "idle",
@@ -556,15 +603,36 @@ export function SettingsPanel({
onPickWhisperModel,
onGenerateAutoCaptions,
onClearAutoCaptions,
onDownloadWhisperSmallModel,
onDeleteWhisperSmallModel,
onDownloadWhisperModel,
onDeleteWhisperModel,
selectedCaptionId,
onSelectCaption,
selectedSpeedId,
selectedSpeedValue,
onSpeedChange,
onSpeedDelete,
audioRegions = [],
selectedAudioId,
onAudioVolumeChange,
onAudioMutedChange,
onAudioSoloedChange,
onAudioFadeInMsChange,
onAudioFadeOutMsChange,
onAudioDelete,
timeSelection,
isMasterSelected,
masterAudioVolume = 1,
masterAudioMuted = false,
masterAudioSoloed = false,
videoDuration,
videoPath,
onMasterAudioVolumeChange,
onMasterAudioMutedChange,
onMasterAudioSoloedChange,
}: SettingsPanelProps) {
const tSettings = useScopedT("settings");
const { t } = useI18n();
const isBackgroundPanel = panelMode === "background";
const initialEditorPreferences = useMemo(() => loadEditorPreferences(), []);
const [builtInWallpapers, setBuiltInWallpapers] =
@@ -1179,6 +1247,11 @@ export function SettingsPanel({
? (figureData) => onAnnotationFigureDataChange(selectedAnnotation.id, figureData)
: undefined
}
onBlurIntensityChange={
onAnnotationBlurIntensityChange
? (intensity) => onAnnotationBlurIntensityChange(selectedAnnotation.id, intensity)
: undefined
}
onDelete={() => onAnnotationDelete(selectedAnnotation.id)}
/>
);
@@ -1375,16 +1448,6 @@ export function SettingsPanel({
</div>
<div className="rounded-lg bg-white/[0.03] px-2.5 py-2 space-y-3">
<div>
<Button
type="button"
variant="outline"
onClick={onPickWhisperModel}
className="h-10 w-full rounded-xl border-white/10 bg-white/5 px-4 text-sm text-slate-200 hover:bg-white/10 hover:text-white"
>
{tSettings("captions.selectModel", "Select Model")}
</Button>
</div>
<div className="flex items-center justify-between gap-3">
<div className="text-sm font-medium text-slate-200">
{tSettings("captions.language", "Language")}
@@ -1405,6 +1468,44 @@ export function SettingsPanel({
</SelectContent>
</Select>
</div>
<div className="flex items-center justify-between gap-3">
<div className="text-sm font-medium text-slate-200">Model</div>
<Select
value={autoCaptionSettings.selectedModel || "small"}
onValueChange={(value) => updateAutoCaptionSettings({ selectedModel: value as any })}
>
<SelectTrigger className="h-10 w-[180px] rounded-xl border-white/10 bg-white/5 text-sm text-slate-200 hover:bg-white/10">
<SelectValue />
</SelectTrigger>
<SelectContent className="border-white/10 bg-[#1a1a1f] text-slate-200">
{WHISPER_MODEL_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
<div className="flex items-center justify-between w-full gap-4">
<span>{option.label}</span>
<span className="text-[10px] text-slate-500">{option.size}</span>
</div>
</SelectItem>
))}
</SelectContent>
</Select>
</div>
{autoCaptionSettings.selectedModel === "custom" && (
<div>
<Button
type="button"
variant="outline"
onClick={onPickWhisperModel}
className="h-10 w-full rounded-xl border-white/10 bg-white/5 px-4 text-sm text-slate-200 hover:bg-white/10 hover:text-white"
>
{tSettings("captions.selectModel", "Select Model")}
</Button>
{whisperModelPath && (
<p className="mt-1 truncate px-1 text-[10px] text-slate-500">
{whisperModelPath.split(/[\\/]/).pop()}
</p>
)}
</div>
)}
<div className="flex flex-wrap items-center gap-2">
<div className="grid w-full grid-cols-2 gap-2">
{whisperModelDownloadStatus === "downloading" ? (
@@ -1420,23 +1521,26 @@ export function SettingsPanel({
<Button
type="button"
variant="outline"
onClick={onDeleteWhisperSmallModel}
onClick={onDeleteWhisperModel}
className="h-10 w-full rounded-xl border-white/10 bg-white/5 px-4 text-sm text-slate-200 hover:bg-white/10 hover:text-white"
>
{tSettings("captions.deleteModel", "Delete Model")}
</Button>
) : (
) : autoCaptionSettings.selectedModel !== "custom" ? (
<Button
type="button"
onClick={onDownloadWhisperSmallModel}
onClick={onDownloadWhisperModel}
className="h-10 w-full rounded-xl bg-[#2563EB] px-4 text-sm font-medium text-white hover:bg-[#2563EB]/90"
>
{tSettings("captions.downloadModel", "Download Model")}
</Button>
) : (
<div className="flex h-10 w-full items-center justify-center rounded-xl bg-white/5 px-4 text-[10px] text-slate-500 italic">
No local model selected
</div>
)}
<Button
type="button"
variant="outline"
onClick={onClearAutoCaptions}
disabled={captionCueCount === 0}
className="h-10 w-full rounded-xl border-white/10 bg-white/5 px-4 text-sm text-slate-200 hover:bg-white/10 hover:text-white disabled:opacity-50"
@@ -1445,19 +1549,124 @@ export function SettingsPanel({
</Button>
</div>
</div>
<div className="flex flex-col gap-3 pt-1">
<div className="flex flex-col gap-1.5 px-1">
<SectionLabel>Generation Range</SectionLabel>
<ToggleGroup
type="single"
value={autoCaptionSettings?.generationRange || "full"}
onValueChange={(val) =>
val &&
onAutoCaptionSettingsChange?.({
...autoCaptionSettings!,
generationRange: val as any,
})
}
className="justify-start gap-1"
>
<ToggleGroupItem
value="full"
className="h-7 cursor-pointer rounded-lg border border-white/5 bg-white/5 px-2.5 text-[10px] data-[state=on]:border-blue-500/50 data-[state=on]:bg-blue-500/20 data-[state=on]:text-blue-400"
>
Full Video
</ToggleGroupItem>
<ToggleGroupItem
value="selected"
className="h-7 cursor-pointer rounded-lg border border-white/5 bg-white/5 px-2.5 text-[10px] data-[state=on]:border-blue-500/50 data-[state=on]:bg-blue-500/20 data-[state=on]:text-blue-400"
>
Selected Timeline {timeSelection ? `(${(timeSelection.startMs / 1000).toFixed(1)}s - ${(timeSelection.endMs / 1000).toFixed(1)}s)` : ""}
</ToggleGroupItem>
</ToggleGroup>
</div>
</div>
<div className="flex flex-col gap-2">
<Button
type="button"
onClick={onGenerateAutoCaptions}
disabled={isGeneratingCaptions || !whisperModelPath}
className="h-10 w-full rounded-xl bg-[#2563EB] px-4 text-sm font-medium text-white hover:bg-[#2563EB]/90 disabled:opacity-60"
disabled={isGeneratingCaptions}
className="relative h-10 w-full overflow-hidden rounded-xl bg-[#2563EB] px-4 text-sm font-medium text-white hover:bg-[#2563EB]/90 disabled:bg-[#2563EB]/50"
>
{isGeneratingCaptions
? tSettings("captions.generating", "Generating...")
: captionCueCount > 0
? tSettings("captions.regenerateFull", "Regenerate Captions")
: tSettings("captions.generateFull", "Generate Captions")}
{isGeneratingCaptions && (
<motion.div
className="absolute inset-y-0 left-0 bg-white/20"
initial={{ width: 0 }}
animate={{ width: `${autoCaptionProgress}%` }}
transition={{ duration: 0.3 }}
/>
)}
<span className="relative z-10">
{isGeneratingCaptions
? `${tSettings("captions.generating", "Generating...")} (${autoCaptionProgress}%)`
: tSettings("captions.generateAutoCaptions", "Generate Captions")}
</span>
</Button>
{autoCaptions.length > 0 && (
<div className="mt-4 flex flex-col gap-2">
<div className="flex items-center justify-between px-1">
<span className="text-[10px] font-semibold uppercase tracking-wider text-slate-500">
{selectedCaptionId ? "Edit Selected Cue" : "Select Cue on Timeline"}
</span>
</div>
<div className="rounded-xl border border-white/5 bg-black/20 p-2">
{selectedCaptionId ? (
(() => {
const index = autoCaptions.findIndex((c) => c.id === selectedCaptionId);
const cue = autoCaptions[index];
if (!cue) return null;
return (
<div className="flex flex-col gap-2">
<div className="flex items-center justify-between">
<button
type="button"
onClick={() => onSeek?.(cue.startMs / 1000)}
className="text-[10px] font-medium text-slate-500 hover:text-[#2563EB]"
>
{(cue.startMs / 1000).toFixed(2)}s – {(cue.endMs / 1000).toFixed(2)}s
</button>
<button
type="button"
onClick={() => {
const newCaptions = [...autoCaptions];
newCaptions.splice(index, 1);
onAutoCaptionsChange?.(newCaptions);
onSelectCaption?.(null);
}}
className="text-slate-500 hover:text-red-400 p-1"
>
<Trash2 className="h-4 w-4" />
</button>
</div>
<textarea
value={cue.text}
onChange={(e) => {
const newCaptions = [...autoCaptions];
newCaptions[index] = { ...cue, text: e.target.value };
onAutoCaptionsChange?.(newCaptions);
}}
rows={2}
autoFocus
className="w-full resize-none rounded-lg border-none bg-white/5 p-2 text-sm text-slate-300 placeholder:text-slate-600 focus:outline-none focus:ring-1 focus:ring-[#2563EB]"
/>
</div>
);
})()
) : (
<div className="flex flex-col items-center justify-center py-6 px-4 text-center">
<div className="mb-2 rounded-full bg-white/5 p-2.5">
<MessageSquare className="h-4 w-4 text-slate-600" />
</div>
<p className="text-[11px] text-slate-500 leading-relaxed max-w-[160px]">
Select a caption block on the timeline to edit its text and timing
</p>
</div>
)}
</div>
</div>
)}
{isGeneratingCaptions ? (
<div className="space-y-1">
<div className="text-xs text-slate-400">
@@ -1610,6 +1819,54 @@ export function SettingsPanel({
return sceneSectionContent;
case "captions":
return captionsSectionContent;
case "audio": {
const selectedAudio = audioRegions?.find((a) => a.id === selectedAudioId);
if (selectedAudio) {
return (
<AudioSettingsPanel
audio={selectedAudio}
onVolumeChange={(volume) => onAudioVolumeChange?.(selectedAudio.id, volume)}
onMutedChange={(muted) => onAudioMutedChange?.(selectedAudio.id, muted)}
onSoloedChange={(soloed) => onAudioSoloedChange?.(selectedAudio.id, soloed)}
onFadeInMsChange={(ms) => onAudioFadeInMsChange?.(selectedAudio.id, ms)}
onFadeOutMsChange={(ms) => onAudioFadeOutMsChange?.(selectedAudio.id, ms)}
onDelete={() => onAudioDelete?.(selectedAudio.id)}
/>
);
}
if (isMasterSelected) {
const masterAudioMock: AudioRegion = {
id: "master",
startMs: 0,
endMs: (videoDuration || 0) * 1000,
volume: masterAudioVolume,
muted: masterAudioMuted,
soloed: masterAudioSoloed,
audioPath: videoPath || "",
fadeInMs: 0,
fadeOutMs: 0,
};
return (
<AudioSettingsPanel
audio={masterAudioMock}
onVolumeChange={onMasterAudioVolumeChange || (() => {})}
onMutedChange={onMasterAudioMutedChange || (() => {})}
onSoloedChange={onMasterAudioSoloedChange || (() => {})}
onFadeInMsChange={() => {}}
onFadeOutMsChange={() => {}}
onDelete={() => {}}
/>
);
}
return (
<div className="flex flex-col items-center justify-center h-full text-slate-500 gap-2 py-12">
<Music className="w-8 h-8 opacity-20" />
<p className="text-xs">Select an audio region to edit its settings</p>
</div>
);
}
case "cursor":
return (
<section className="flex flex-col gap-2">
File diff suppressed because it is too large Load Diff
@@ -202,8 +202,10 @@ interface VideoPlaybackProps {
cursorClickBounceDuration?: number;
cursorSway?: number;
volume?: number;
timeSelection?: import("./types").TimeSelection | null;
}
export interface VideoPlaybackRef {
video: HTMLVideoElement | null;
app: Application | null;
@@ -213,6 +215,7 @@ export interface VideoPlaybackRef {
play: () => Promise<void>;
pause: () => void;
refreshFrame: () => Promise<void>;
seek: (time: number) => void;
}
const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
@@ -268,7 +271,9 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
cursorClickBounceDuration = DEFAULT_CURSOR_CLICK_BOUNCE_DURATION,
cursorSway = DEFAULT_CURSOR_SWAY,
volume = 1,
timeSelection = null,
},
ref,
) => {
const videoRef = useRef<HTMLVideoElement | null>(null);
@@ -346,6 +351,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const cursorClickBounceRef = useRef(cursorClickBounce);
const cursorClickBounceDurationRef = useRef(cursorClickBounceDuration);
const cursorSwayRef = useRef(cursorSway);
const timeSelectionRef = useRef(timeSelection);
const activeCaptionLayout = useMemo(() => {
if (!autoCaptionSettings?.enabled || autoCaptions.length === 0 || typeof document === "undefined") {
@@ -660,6 +667,11 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
video.currentTime = nudgeTarget;
});
},
seek: (time: number) => {
const video = videoRef.current;
if (!video) return;
video.currentTime = time;
},
}));
const updateFocusFromClientPoint = (clientX: number, clientY: number) => {
@@ -835,6 +847,11 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
cursorSwayRef.current = cursorSway;
}, [cursorSway]);
useEffect(() => {
timeSelectionRef.current = timeSelection;
}, [timeSelection]);
useEffect(() => {
currentTimeRef.current = currentTime * 1000;
}, [currentTime]);
@@ -1177,8 +1194,10 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
onTimeUpdate,
trimRegionsRef,
speedRegionsRef,
timeSelectionRef,
});
video.addEventListener("play", handlePlay);
video.addEventListener("pause", handlePause);
video.addEventListener("ended", handlePause);
@@ -33,6 +33,9 @@ type PersistedEditorControls = Pick<
| "gifFrameRate"
| "gifLoop"
| "gifSizePreset"
| "masterAudioMuted"
| "masterAudioSoloed"
| "masterAudioVolume"
>;
type PartialEditorControls = Partial<PersistedEditorControls>;
@@ -43,6 +46,7 @@ export interface EditorPreferences extends PersistedEditorControls {
customWallpapers: string[];
whisperExecutablePath: string | null;
whisperModelPath: string | null;
whisperSelectedModel: string;
}
export const EDITOR_PREFERENCES_STORAGE_KEY = "recordly.editor.preferences";
@@ -81,11 +85,15 @@ export const DEFAULT_EDITOR_PREFERENCES: EditorPreferences = {
gifFrameRate: DEFAULT_EDITOR_CONTROLS.gifFrameRate,
gifLoop: DEFAULT_EDITOR_CONTROLS.gifLoop,
gifSizePreset: DEFAULT_EDITOR_CONTROLS.gifSizePreset,
masterAudioMuted: DEFAULT_EDITOR_CONTROLS.masterAudioMuted,
masterAudioSoloed: DEFAULT_EDITOR_CONTROLS.masterAudioSoloed,
masterAudioVolume: DEFAULT_EDITOR_CONTROLS.masterAudioVolume,
customAspectWidth: "16",
customAspectHeight: "9",
customWallpapers: [],
whisperExecutablePath: null,
whisperModelPath: null,
whisperSelectedModel: "small",
};
function normalizePositiveIntegerString(value: unknown, fallback: string): string {
@@ -159,6 +167,9 @@ function normalizeEditorControls(
gifFrameRate: raw.gifFrameRate ?? fallback.gifFrameRate,
gifLoop: raw.gifLoop ?? fallback.gifLoop,
gifSizePreset: raw.gifSizePreset ?? fallback.gifSizePreset,
masterAudioMuted: raw.masterAudioMuted ?? fallback.masterAudioMuted,
masterAudioSoloed: raw.masterAudioSoloed ?? fallback.masterAudioSoloed,
masterAudioVolume: raw.masterAudioVolume ?? fallback.masterAudioVolume,
};
const normalized = normalizeProjectEditor(candidate);
@@ -195,6 +206,9 @@ function normalizeEditorControls(
gifFrameRate: normalized.gifFrameRate,
gifLoop: normalized.gifLoop,
gifSizePreset: normalized.gifSizePreset,
masterAudioMuted: normalized.masterAudioMuted,
masterAudioSoloed: normalized.masterAudioSoloed,
masterAudioVolume: normalized.masterAudioVolume,
};
}
@@ -220,6 +234,10 @@ export function normalizeEditorPreferences(
normalizeNullablePath(raw.whisperExecutablePath) ?? fallback.whisperExecutablePath,
whisperModelPath:
normalizeNullablePath(raw.whisperModelPath) ?? fallback.whisperModelPath,
whisperSelectedModel:
typeof raw.whisperSelectedModel === "string"
? raw.whisperSelectedModel
: fallback.whisperSelectedModel,
};
}
@@ -94,6 +94,11 @@ export interface ProjectEditorState {
gifFrameRate: GifFrameRate;
gifLoop: boolean;
gifSizePreset: GifSizePreset;
masterAudioMuted: boolean;
masterAudioSoloed: boolean;
masterAudioVolume: number;
audioTrackVolume: number;
isMasterSelected?: boolean;
}
export interface EditorProjectData {
@@ -336,7 +341,10 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
id: region.id,
startMs,
endMs,
type: region.type === "image" || region.type === "figure" ? region.type : "text",
type:
region.type === "image" || region.type === "figure" || region.type === "blur"
? region.type
: "text",
content: typeof region.content === "string" ? region.content : "",
textContent: typeof region.textContent === "string" ? region.textContent : undefined,
imageContent: typeof region.imageContent === "string" ? region.imageContent : undefined,
@@ -379,10 +387,11 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
zIndex: isFiniteNumber(region.zIndex) ? region.zIndex : index + 1,
figureData: region.figureData
? {
...DEFAULT_FIGURE_DATA,
...region.figureData,
}
...DEFAULT_FIGURE_DATA,
...region.figureData,
}
: undefined,
blurIntensity: isFiniteNumber(region.blurIntensity) ? region.blurIntensity : undefined,
};
})
: [];
@@ -403,7 +412,11 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
startMs,
endMs,
audioPath: typeof region.audioPath === "string" ? region.audioPath : "",
volume: isFiniteNumber(region.volume) ? clamp(region.volume, 0, 1) : 1,
volume: isFiniteNumber(region.volume) ? clamp(region.volume, 0, 2) : 1,
muted: typeof region.muted === "boolean" ? region.muted : false,
soloed: typeof region.soloed === "boolean" ? region.soloed : false,
fadeInMs: isFiniteNumber(region.fadeInMs) ? clamp(region.fadeInMs, 0, 10000) : 0,
fadeOutMs: isFiniteNumber(region.fadeOutMs) ? clamp(region.fadeOutMs, 0, 10000) : 0,
};
})
: [];
@@ -500,6 +513,14 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
backgroundOpacity: isFiniteNumber(rawAutoCaptionSettings.backgroundOpacity)
? clamp(rawAutoCaptionSettings.backgroundOpacity, 0, 1)
: DEFAULT_AUTO_CAPTION_SETTINGS.backgroundOpacity,
selectedModel: typeof rawAutoCaptionSettings.selectedModel === "string"
? rawAutoCaptionSettings.selectedModel
: DEFAULT_AUTO_CAPTION_SETTINGS.selectedModel,
generationRange:
rawAutoCaptionSettings.generationRange === "full" ||
rawAutoCaptionSettings.generationRange === "selected"
? rawAutoCaptionSettings.generationRange
: "full",
};
const rawCropX = isFiniteNumber(editor.cropRegion?.x)
@@ -675,6 +696,11 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
editor.gifSizePreset === "original"
? editor.gifSizePreset
: "medium",
masterAudioMuted: typeof editor.masterAudioMuted === "boolean" ? editor.masterAudioMuted : false,
masterAudioSoloed: typeof editor.masterAudioSoloed === "boolean" ? editor.masterAudioSoloed : false,
masterAudioVolume: isFiniteNumber(editor.masterAudioVolume) ? clamp(editor.masterAudioVolume, 0, 2) : 1,
audioTrackVolume: isFiniteNumber(editor.audioTrackVolume) ? clamp(editor.audioTrackVolume, 0, 2) : 1,
isMasterSelected: Boolean(editor.isMasterSelected),
};
}
+152 -21
View File
@@ -1,9 +1,13 @@
import type { Span } from "dnd-timeline";
import { useItem } from "dnd-timeline";
import { Gauge, MessageSquare, Music, Scissors, ZoomIn } from "lucide-react";
import { useMemo } from "react";
import { useEffect, useMemo, useRef, useState } from "react";
import { cn } from "@/lib/utils";
import glassStyles from "./ItemGlass.module.css";
import { generateWaveform } from "@/utils/audioWaveform";
interface ItemProps {
id: string;
@@ -14,7 +18,14 @@ interface ItemProps {
onSelect?: () => void;
zoomDepth?: number;
speedValue?: number;
variant?: 'zoom' | 'trim' | 'annotation' | 'speed' | 'audio';
audioPath?: string;
variant?: 'zoom' | 'trim' | 'annotation' | 'speed' | 'audio' | 'caption' | 'caption-range';
isDraggable?: boolean;
isResizable?: boolean;
muted?: boolean;
fadeInMs?: number;
fadeOutMs?: number;
timelineMode?: 'move' | 'select';
}
// Map zoom depth to multiplier labels
@@ -45,19 +56,69 @@ export default function Item({
onSelect,
zoomDepth = 1,
speedValue,
audioPath,
variant = "zoom",
children,
isDraggable = true,
isResizable = true,
muted = false,
fadeInMs,
fadeOutMs,
timelineMode = 'move',
}: ItemProps) {
const isDraggableEffective = isDraggable && timelineMode === 'move';
const { setNodeRef, attributes, listeners, itemStyle, itemContentStyle } = useItem({
id,
span,
disabled: !isDraggableEffective,
data: { rowId },
});
const durationMs = span.end - span.start;
const isZoom = variant === 'zoom';
const isTrim = variant === 'trim';
const isSpeed = variant === 'speed';
const isAudio = variant === 'audio';
const isCaption = variant === 'caption';
const isCaptionRange = variant === 'caption-range';
const [waveform, setWaveform] = useState<number[] | null>(null);
useEffect(() => {
if (isAudio && audioPath) {
generateWaveform(audioPath).then((peaks) => {
setWaveform(peaks);
});
}
}, [isAudio, audioPath]);
const mouseDownPosRef = useRef({ x: 0, y: 0 });
const handlePointerDown = (e: React.PointerEvent) => {
mouseDownPosRef.current = { x: e.clientX, y: e.clientY };
// If dragging is enabled, let dnd-timeline handle its part
if (isDraggableEffective) {
listeners?.onPointerDown(e);
}
};
const handlePointerUp = (e: React.PointerEvent) => {
// We MUST NOT stop propagation on the pointer-up event.
// Standard dnd libraries (like dnd-timeline) need this event to bubble
// up to the window/document to successfully terminate the drag state.
// If we stop it here, the item stays 'sticky' to the mouse.
const deltaX = Math.abs(e.clientX - mouseDownPosRef.current.x);
const deltaY = Math.abs(e.clientY - mouseDownPosRef.current.y);
if (deltaX < 5 && deltaY < 5) {
onSelect?.();
}
};
const glassClass = isZoom
? glassStyles.glassGreen
@@ -67,6 +128,10 @@ export default function Item({
? glassStyles.glassAmber
: isAudio
? glassStyles.glassPurple
: isCaption
? glassStyles.glassCyan
: isCaptionRange
? glassStyles.glassCyanDashed
: glassStyles.glassYellow;
const endCapColor = isZoom
@@ -77,6 +142,10 @@ export default function Item({
? '#d97706'
: isAudio
? '#a855f7'
: isCaption
? '#0891b2'
: isCaptionRange
? '#06b6d4'
: '#B4A046';
const timeLabel = useMemo(
@@ -91,37 +160,99 @@ export default function Item({
<div
ref={setNodeRef}
style={safeItemStyle}
{...listeners}
{...attributes}
onPointerDownCapture={() => onSelect?.()}
{...(isDraggableEffective ? attributes : {})}
onPointerDown={handlePointerDown}
onPointerUp={handlePointerUp}
onMouseDown={(e) => {
// Prevent background selection logic in Move mode
if (timelineMode === 'move') {
e.stopPropagation();
}
}}
onClick={(e) => {
// Prevent the timeline background's onClick (seeking) from firing
// when we click on a track item.
e.stopPropagation();
}}
className="group h-full"
>
<div className="h-full" style={{ ...itemContentStyle, minWidth: 24, height: "100%" }}>
<div
className={cn(
glassClass,
"w-full h-full overflow-hidden flex items-center justify-center gap-1.5 cursor-grab active:cursor-grabbing relative",
isSelected && glassStyles.selected
"w-full h-full overflow-hidden flex items-center justify-center gap-1.5 relative",
isDraggableEffective ? "cursor-grab active:cursor-grabbing" : "cursor-default",
isSelected && glassStyles.selected,
muted && "opacity-40 grayscale-[0.5]"
)}
style={{ height: "100%", minHeight: 22, color: '#fff', minWidth: 24 }}
onClick={(event) => {
event.stopPropagation();
onSelect?.();
}}
>
<div
className={cn(glassStyles.zoomEndCap, glassStyles.left)}
style={{ cursor: 'col-resize', pointerEvents: 'auto', width: 8, opacity: 0.9, background: endCapColor }}
title="Resize left"
/>
<div
className={cn(glassStyles.zoomEndCap, glassStyles.right)}
style={{ cursor: 'col-resize', pointerEvents: 'auto', width: 8, opacity: 0.9, background: endCapColor }}
title="Resize right"
/>
{/* Waveform Background for Audio */}
{isAudio && waveform && (
<div className="absolute inset-0 z-0 opacity-30 flex items-center pointer-events-none px-4">
<svg
width="100%"
height="80%"
viewBox={`0 0 ${waveform.length} 100`}
preserveAspectRatio="none"
className="text-white"
>
{waveform.map((peak, i) => (
<rect
key={i}
x={i}
y={50 - (peak * 50)}
width={0.8}
height={peak * 100}
fill="currentColor"
rx={0.2}
/>
))}
</svg>
</div>
)}
{/* Fade Visualizations */}
{isAudio && (fadeInMs || fadeOutMs) && (
<div className="absolute inset-0 z-[5] pointer-events-none flex">
{fadeInMs && fadeInMs > 0 && (
<div
className="h-full bg-gradient-to-r from-black/40 to-transparent"
style={{ width: `${(fadeInMs / durationMs) * 100}%` }}
/>
)}
<div className="flex-1" />
{fadeOutMs && fadeOutMs > 0 && (
<div
className="h-full bg-gradient-to-l from-black/40 to-transparent"
style={{ width: `${(fadeOutMs / durationMs) * 100}%` }}
/>
)}
</div>
)}
{isResizable && (
<>
<div
className={cn(glassStyles.zoomEndCap, glassStyles.left)}
style={{ cursor: 'col-resize', pointerEvents: 'auto', width: 8, opacity: 0.9, background: endCapColor }}
title="Resize left"
/>
<div
className={cn(glassStyles.zoomEndCap, glassStyles.right)}
style={{ cursor: 'col-resize', pointerEvents: 'auto', width: 8, opacity: 0.9, background: endCapColor }}
title="Resize right"
/>
</>
)}
{/* Content */}
<div className="relative z-10 flex flex-col items-center justify-center text-white/90 opacity-80 group-hover:opacity-100 transition-opacity select-none overflow-hidden">
<div className="flex items-center gap-1.5">
<div className="relative z-10 flex flex-col items-center justify-center text-white/90 opacity-80 group-hover:opacity-100 transition-opacity select-none overflow-hidden max-w-full">
<div className="flex items-center gap-1.5 max-w-full">
{isZoom ? (
<>
<ZoomIn className="w-3.5 h-3.5 shrink-0" />
@@ -146,7 +277,7 @@ export default function Item({
) : isAudio ? (
<>
<Music className="w-3.5 h-3.5 shrink-0" />
<span className="text-[11px] font-semibold tracking-tight truncate max-w-full">
<span className="text-[11px] font-semibold tracking-tight truncate max-w-full px-2">
{children}
</span>
</>
@@ -128,6 +128,32 @@
z-index: 10;
}
.glassCyan {
position: relative;
border-radius: 8px;
-corner-smoothing: antialiased;
background: rgba(8, 145, 178, 0.15);
border: 1px solid rgba(8, 145, 178, 0.3);
box-shadow: 0 2px 12px 0 rgba(8, 145, 178, 0.1) inset;
margin: 1px 0;
backdrop-filter: blur(4px);
-webkit-backdrop-filter: blur(4px);
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
}
.glassCyan:hover {
background: rgba(8, 145, 178, 0.25);
border-color: rgba(8, 145, 178, 0.5);
box-shadow: 0 4px 20px 0 rgba(8, 145, 178, 0.2) inset;
}
.glassCyan.selected {
background: rgba(8, 145, 178, 0.35);
border-color: #0891b2;
box-shadow: 0 0 0 1px #0891b2, 0 4px 20px 0 rgba(8, 145, 178, 0.3) inset;
z-index: 10;
}
.zoomEndCap {
position: absolute;
top: 0;
@@ -148,10 +174,39 @@
.glassAmber:hover .zoomEndCap,
.glassAmber.selected .zoomEndCap,
.glassPurple:hover .zoomEndCap,
.glassPurple.selected .zoomEndCap {
.glassPurple.selected .zoomEndCap,
.glassCyan.selected .zoomEndCap,
.glassCyanDashed:hover .zoomEndCap,
.glassCyanDashed.selected .zoomEndCap {
opacity: 1;
}
.glassCyanDashed {
position: relative;
border-radius: 8px;
-corner-smoothing: antialiased;
background: rgba(8, 145, 178, 0.05);
border: 1px dashed rgba(8, 145, 178, 0.5);
box-shadow: 0 2px 12px 0 rgba(8, 145, 178, 0.05) inset;
margin: 1px 0;
backdrop-filter: blur(2px);
-webkit-backdrop-filter: blur(2px);
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
}
.glassCyanDashed:hover {
background: rgba(8, 145, 178, 0.1);
border-color: rgba(8, 145, 178, 0.7);
}
.glassCyanDashed.selected {
background: rgba(8, 145, 178, 0.2);
border-color: #0891b2;
border-style: solid;
box-shadow: 0 0 0 1px #0891b2, 0 4px 20px 0 rgba(8, 145, 178, 0.15) inset;
z-index: 10;
}
.zoomEndCap.left {
left: 0;
cursor: ew-resize;
@@ -23,7 +23,7 @@ const KeyframeMarkers: React.FC<KeyframeMarkersProps> = ({
videoDurationMs,
timelineRef
}) => {
const { sidebarWidth, range, valueToPixels, pixelsToValue } = useTimelineContext();
const { sidebarWidth = 0, range, valueToPixels, pixelsToValue } = useTimelineContext();
const [draggingKeyframeId, setDraggingKeyframeId] = useState<string | null>(null);
useEffect(() => {
+14 -5
View File
@@ -7,9 +7,10 @@ interface RowProps extends RowDefinition {
hint?: string;
isEmpty?: boolean;
labelColor?: string;
controls?: React.ReactNode;
}
export default function Row({ id, children, label, hint, isEmpty, labelColor = '#666' }: RowProps) {
export default function Row({ id, children, label, hint, isEmpty, labelColor = '#666', controls }: RowProps) {
const { setNodeRef, rowWrapperStyle, rowStyle } = useRow({ id });
return (
@@ -17,12 +18,20 @@ export default function Row({ id, children, label, hint, isEmpty, labelColor = '
className="border-b border-[#18181b] bg-[#18181b] relative flex-1 min-h-[26px]"
style={{ ...rowWrapperStyle, marginBottom: 2 }}
>
{label && (
{(label || controls) && (
<div
className="absolute left-1.5 top-1/2 -translate-y-1/2 text-[9px] font-semibold uppercase tracking-widest z-20 pointer-events-none select-none"
style={{ color: labelColor, writingMode: 'horizontal-tb' }}
className="absolute left-1.5 top-1/2 -translate-y-1/2 z-20 flex items-center gap-2"
style={{ writingMode: 'horizontal-tb' }}
>
{label}
{label && (
<div
className="text-[9px] font-semibold uppercase tracking-widest select-none pointer-events-none"
style={{ color: labelColor }}
>
{label}
</div>
)}
{controls}
</div>
)}
{isEmpty && hint && (
File diff suppressed because it is too large Load Diff
@@ -20,7 +20,7 @@ interface TimelineWrapperProps {
minItemDurationMs: number;
minVisibleRangeMs: number;
gridSizeMs?: number;
onItemSpanChange: (id: string, span: Span) => void;
onItemSpanChange: (id: string, span: Span, rowId: string) => void;
allRegionSpans?: { id: string; start: number; end: number }[];
}
@@ -133,6 +133,7 @@ export default function TimelineWrapper({
if (!updatedSpan) return;
const activeItemId = event.active.id as string;
const rowId = event.active.data.current.rowId as string;
let clampedSpan = clampSpanToBounds(updatedSpan);
const effectiveMinDuration = totalMs > 0
@@ -151,7 +152,7 @@ export default function TimelineWrapper({
}
}
onItemSpanChange(activeItemId, clampedSpan);
onItemSpanChange(activeItemId, clampedSpan, rowId);
},
[clampSpanToBounds, clampToNeighbours, hasOverlap, minItemDurationMs, onItemSpanChange, totalMs]
);
@@ -163,6 +164,7 @@ export default function TimelineWrapper({
if (!updatedSpan || !activeRowId) return;
const activeItemId = event.active.id as string;
const rowId = event.active.data.current.rowId as string;
let clampedSpan = clampSpanToBounds(updatedSpan);
// Clamp to neighbour boundaries instead of rejecting
@@ -173,7 +175,7 @@ export default function TimelineWrapper({
}
}
onItemSpanChange(activeItemId, clampedSpan);
onItemSpanChange(activeItemId, clampedSpan, rowId);
},
[clampSpanToBounds, clampToNeighbours, hasOverlap, onItemSpanChange]
);
+20 -2
View File
@@ -127,7 +127,7 @@ export interface TrimRegion {
endMs: number;
}
export type AnnotationType = "text" | "image" | "figure";
export type AnnotationType = "text" | "image" | "figure" | "blur";
export type ArrowDirection =
| "up"
@@ -195,8 +195,11 @@ export interface AnnotationRegion {
style: AnnotationTextStyle;
zIndex: number;
figureData?: FigureData;
blurIntensity?: number;
}
export const DEFAULT_BLUR_INTENSITY = 12;
export const DEFAULT_ANNOTATION_POSITION: AnnotationPosition = {
x: 50,
y: 50,
@@ -244,6 +247,16 @@ export interface AudioRegion {
endMs: number;
audioPath: string;
volume: number;
muted?: boolean;
soloed?: boolean;
fadeInMs?: number;
fadeOutMs?: number;
}
export interface TimeSelection {
startMs: number;
endMs: number;
}
export interface CaptionCue {
@@ -262,10 +275,12 @@ export interface CaptionCueWord {
}
export type AutoCaptionAnimation = "none" | "fade" | "rise" | "pop";
export type WhisperModel = "tiny" | "base" | "small" | "medium" | "large" | "custom";
export interface AutoCaptionSettings {
enabled: boolean;
language: string;
selectedModel: WhisperModel;
fontFamily: string;
fontSize: number;
bottomOffset: number;
@@ -276,11 +291,13 @@ export interface AutoCaptionSettings {
textColor: string;
inactiveTextColor: string;
backgroundOpacity: number;
generationRange: "full" | "selected";
}
export const DEFAULT_AUTO_CAPTION_SETTINGS: AutoCaptionSettings = {
enabled: false,
language: "auto",
selectedModel: "small",
fontFamily: getDefaultCaptionFontFamily(),
fontSize: 30,
bottomOffset: 3,
@@ -290,7 +307,8 @@ export const DEFAULT_AUTO_CAPTION_SETTINGS: AutoCaptionSettings = {
boxRadius: 17.5,
textColor: "#FFFFFF",
inactiveTextColor: "#A3A3A3",
backgroundOpacity: 0.9,
backgroundOpacity: 0.1,
generationRange: "full",
};
export type PlaybackSpeed = 0.25 | 0.5 | 0.75 | 1.25 | 1.5 | 1.75 | 2;
@@ -12,8 +12,10 @@ interface VideoEventHandlersParams {
onTimeUpdate: (time: number) => void;
trimRegionsRef: React.MutableRefObject<TrimRegion[]>;
speedRegionsRef: React.MutableRefObject<SpeedRegion[]>;
timeSelectionRef: React.MutableRefObject<import('../types').TimeSelection | null>;
}
export function createVideoEventHandlers(params: VideoEventHandlersParams) {
const {
video,
@@ -26,8 +28,10 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) {
onTimeUpdate,
trimRegionsRef,
speedRegionsRef,
timeSelectionRef,
} = params;
const emitTime = (timeValue: number) => {
currentTimeRef.current = timeValue * 1000;
onTimeUpdate(timeValue);
@@ -52,8 +56,21 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) {
if (!video) return;
const currentTimeMs = video.currentTime * 1000;
// Selection awareness: stop playback at the end of selection range
const selection = timeSelectionRef.current;
if (selection && !video.paused && !isSeekingRef.current) {
if (currentTimeMs >= selection.endMs) {
video.pause();
video.currentTime = selection.startMs / 1000;
emitTime(selection.startMs / 1000);
return; // Selection boundary reached, stop update loop
}
}
const activeTrimRegion = findActiveTrimRegion(currentTimeMs);
// If we're in a trim region during playback, skip to the end of it
if (activeTrimRegion && !video.paused && !video.ended) {
const skipToTime = activeTrimRegion.endMs / 1000;
+17
View File
@@ -121,6 +121,23 @@
.custom-scrollbar::-webkit-scrollbar {
display: none; /* Chrome, Safari, Opera */
}
.subtle-scrollbar::-webkit-scrollbar {
width: 4px;
}
.subtle-scrollbar::-webkit-scrollbar-track {
background: transparent;
}
.subtle-scrollbar::-webkit-scrollbar-thumb {
background: rgba(255, 255, 255, 0.1);
border-radius: 10px;
}
.subtle-scrollbar::-webkit-scrollbar-thumb:hover {
background: rgba(255, 255, 255, 0.2);
}
/* Smooth playback scrubber */
input[type="range"] {
+65
View File
@@ -289,6 +289,67 @@ async function renderImage(
});
}
function renderBlur(
ctx: CanvasRenderingContext2D,
annotation: AnnotationRegion,
x: number,
y: number,
width: number,
height: number,
scaleFactor: number
) {
// Get intensity scaled to canvas resolution
const intensity = (annotation.blurIntensity ?? 12) * scaleFactor;
if (intensity <= 0 || width <= 0 || height <= 0) return;
ctx.save();
try {
// Determine bounds and ensure they are within canvas to avoid ImageData errors
const srcX = Math.max(0, x);
const srcY = Math.max(0, y);
const srcWidth = Math.min(x + width, ctx.canvas.width) - srcX;
const srcHeight = Math.min(y + height, ctx.canvas.height) - srcY;
if (srcWidth <= 0 || srcHeight <= 0) {
ctx.restore();
return;
}
// Capture the current canvas region precisely for this intersection
const imageData = ctx.getImageData(srcX, srcY, srcWidth, srcHeight);
let offscreen: HTMLCanvasElement | OffscreenCanvas;
if (typeof document !== 'undefined') {
offscreen = document.createElement('canvas');
} else {
offscreen = new OffscreenCanvas(srcWidth, srcHeight);
}
offscreen.width = srcWidth;
offscreen.height = srcHeight;
const offCtx = offscreen.getContext('2d') as CanvasRenderingContext2D;
offCtx.putImageData(imageData, 0, 0);
// Create rounded rect clipping path (matches UI's rounded-lg approx 8px)
const radius = 8 * scaleFactor;
ctx.beginPath();
if (ctx.roundRect) {
ctx.roundRect(x, y, width, height, radius);
} else {
ctx.rect(x, y, width, height);
}
ctx.clip();
// Apply the blur filter and draw the captured region back at its source position
ctx.filter = `blur(${intensity}px)`;
ctx.drawImage(offscreen as any, srcX, srcY);
} catch (err) {
console.warn('[AnnotationRenderer] Blur annotation render failed:', err);
}
ctx.restore();
}
export async function renderAnnotations(
ctx: CanvasRenderingContext2D,
annotations: AnnotationRegion[],
@@ -335,6 +396,10 @@ export async function renderAnnotations(
);
}
break;
case 'blur':
renderBlur(ctx, annotation, x, y, width, height, scaleFactor);
break;
}
}
}
+63 -7
View File
@@ -25,6 +25,10 @@ export class AudioProcessor {
speedRegions?: SpeedRegion[],
readEndSec?: number,
audioRegions?: AudioRegion[],
masterAudioVolume = 1,
audioTrackVolume = 1,
masterAudioMuted = false,
masterAudioSoloed = false,
): Promise<void> {
const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : []
const sortedSpeedRegions = speedRegions
@@ -36,13 +40,24 @@ export class AudioProcessor {
? [...audioRegions].sort((a, b) => a.startMs - b.startMs)
: []
// When audio regions or speed edits are present, use AudioContext mixing path.
if (sortedSpeedRegions.length > 0 || sortedAudioRegions.length > 0) {
// When audio regions, speed edits, or global volume/mute settings are present, use AudioContext mixing path.
if (
sortedSpeedRegions.length > 0 ||
sortedAudioRegions.length > 0 ||
masterAudioVolume !== 1 ||
audioTrackVolume !== 1 ||
masterAudioMuted ||
masterAudioSoloed
) {
const renderedAudioBlob = await this.renderMixedTimelineAudio(
videoUrl,
sortedTrims,
sortedSpeedRegions,
sortedAudioRegions,
masterAudioVolume,
audioTrackVolume,
masterAudioMuted,
masterAudioSoloed,
)
if (!this.cancelled) {
await this.muxRenderedAudioBlob(renderedAudioBlob, muxer)
@@ -279,6 +294,10 @@ export class AudioProcessor {
trimRegions: TrimRegion[],
speedRegions: SpeedRegion[],
audioRegions: AudioRegion[],
masterAudioVolume: number,
audioTrackVolume: number,
masterAudioMuted: boolean,
masterAudioSoloed: boolean,
): Promise<Blob> {
const mediaSource = await resolveMediaElementSource(videoUrl)
const media = document.createElement('audio')
@@ -302,9 +321,11 @@ export class AudioProcessor {
const audioContext = new AudioContext()
const destinationNode = audioContext.createMediaStreamDestination()
// Connect original video audio
// Connect original video audio with its own gain node
const sourceNode = audioContext.createMediaElementSource(media)
sourceNode.connect(destinationNode)
const masterGainNode = audioContext.createGain()
sourceNode.connect(masterGainNode)
masterGainNode.connect(destinationNode)
// Prepare external audio region elements
const audioRegionElements: {
@@ -331,7 +352,8 @@ export class AudioProcessor {
const regionSourceNode = audioContext.createMediaElementSource(audioEl)
const gainNode = audioContext.createGain()
gainNode.gain.value = Math.max(0, Math.min(1, region.volume))
// Initial volume (will be updated in the tick loop for fades)
gainNode.gain.value = 0
regionSourceNode.connect(gainNode)
gainNode.connect(destinationNode)
@@ -402,22 +424,56 @@ export class AudioProcessor {
}
}
// Check for Solo - if any track is soloed, everything else is muted
const anyTrackSoloed = masterAudioSoloed || audioRegionElements.some(e => e.region.soloed);
// Update Master Video Audio Track Volume
let masterTargetVolume = masterAudioMuted ? 0 : audioTrackVolume;
if (anyTrackSoloed && !masterAudioSoloed) {
masterTargetVolume = 0;
}
// Factor in the overall Master output volume
masterTargetVolume *= masterAudioVolume;
masterGainNode.gain.setTargetAtTime(masterTargetVolume, audioContext.currentTime, 0.015);
// Sync external audio regions with the video timeline position
for (const entry of audioRegionElements) {
const { media: audioEl, region } = entry
const { media: audioEl, region, gainNode } = entry
const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs
if (isInRegion) {
const audioOffset = (currentTimeMs - region.startMs) / 1000
// Apply fade-in / fade-out multiplier
let fadeMultiplier = 1;
if (region.fadeInMs && currentTimeMs < region.startMs + region.fadeInMs) {
fadeMultiplier = (currentTimeMs - region.startMs) / region.fadeInMs;
} else if (region.fadeOutMs && currentTimeMs > region.endMs - region.fadeOutMs) {
fadeMultiplier = (region.endMs - currentTimeMs) / region.fadeOutMs;
}
fadeMultiplier = Math.max(0, Math.min(1, fadeMultiplier));
// Respect Solo and Mute
let regionTargetVolume = (region.muted || masterAudioMuted) ? 0 : region.volume;
if (anyTrackSoloed && !region.soloed) {
regionTargetVolume = 0;
}
// Also factor in global master volume
regionTargetVolume *= masterAudioVolume * fadeMultiplier;
gainNode.gain.setTargetAtTime(regionTargetVolume, audioContext.currentTime, 0.015);
if (audioEl.paused) {
audioEl.currentTime = audioOffset
audioEl.play().catch(() => {})
} else if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) {
} else if (Math.abs(audioEl.currentTime - audioOffset) > 0.1) {
// Tightened sync for export
audioEl.currentTime = audioOffset
}
} else {
if (!audioEl.paused) {
audioEl.pause()
gainNode.gain.value = 0
}
}
}
+3
View File
@@ -4,6 +4,9 @@ export interface ExportConfig {
frameRate: number;
bitrate: number;
codec?: string;
masterAudioVolume?: number;
audioTrackVolume?: number;
masterAudioMuted?: boolean;
}
export interface ExportProgress {
+8
View File
@@ -59,6 +59,10 @@ interface VideoExporterConfig extends ExportConfig {
previewWidth?: number;
previewHeight?: number;
onProgress?: (progress: ExportProgress) => void;
masterAudioVolume?: number;
masterAudioMuted?: boolean;
masterAudioSoloed?: boolean;
audioTrackVolume?: number;
}
export class VideoExporter {
@@ -207,6 +211,10 @@ export class VideoExporter {
this.config.speedRegions,
undefined,
this.config.audioRegions,
this.config.masterAudioVolume,
this.config.audioTrackVolume,
this.config.masterAudioMuted,
this.config.masterAudioSoloed,
),
"audio processing",
);
+50
View File
@@ -0,0 +1,50 @@
/**
* Extracts waveform peaks from an audio file.
* We decode the audio file using the Web Audio API and calculate the max peaks for each sample.
*/
import { toFileUrl } from "@/components/video-editor/projectPersistence";
const waveformCache = new Map<string, number[]>();
export async function generateWaveform(audioPath: string, samples = 200): Promise<number[]> {
const cacheKey = `${audioPath}:${samples}`;
if (waveformCache.has(cacheKey)) {
return waveformCache.get(cacheKey)!;
}
const audioContext = new (window.AudioContext || (window as any).webkitAudioContext)();
try {
const response = await fetch(toFileUrl(audioPath));
const arrayBuffer = await response.arrayBuffer();
// Use an offline audio context to decode the data
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
const channelData = audioBuffer.getChannelData(0); // Use the first channel
const blockSize = Math.floor(channelData.length / samples);
const peaks: number[] = [];
for (let i = 0; i < samples; i++) {
const start = i * blockSize;
let max = 0;
for (let j = 0; j < blockSize; j++) {
const value = Math.abs(channelData[start + j]);
if (value > max) max = value;
}
peaks.push(max);
}
waveformCache.set(cacheKey, peaks);
return peaks;
} catch (error) {
console.error('Failed to generate waveform:', error);
return new Array(samples).fill(0);
} finally {
try {
await audioContext.close();
} catch (e) {
// ignore close errors
}
}
}