mirror of
https://github.com/webadderallorg/Recordly.git
synced 2026-09-24 23:05:49 +00:00
fix: always apply aresample sync filter and tighten audio sync tolerances
- Always route audio through aresample=async=1:first_pts=0 filter during muxing on both macOS and Windows, even when duration delta is small. Previously skipped when delta ≤50ms, which left progressive clock drift (from CPU load) completely uncorrected. - Lower sync detection threshold from 50ms to 20ms so tempo correction kicks in earlier. - Use explicit 48kHz AudioContext sample rate in both browser recording and export rendering to prevent sample rate mismatch drift. - Reduce MediaRecorder timeslice from 1000ms to 250ms to reduce chunk loss under CPU pressure. - Tighten export audio sync: seek threshold 300ms→150ms, tolerance 15ms→8ms, correction window 2s→0.5s, max adjustment ±8%→±12%. - Lower companion audio start delay threshold from 50ms to 25ms.
This commit is contained in:
@@ -40,7 +40,7 @@ export function getAudioSyncAdjustment(
|
||||
|
||||
const durationDeltaMs = Math.round((videoDuration - audioDuration) * 1000);
|
||||
const absDeltaMs = Math.abs(durationDeltaMs);
|
||||
if (absDeltaMs <= 50) {
|
||||
if (absDeltaMs <= 20) {
|
||||
return { mode: "none", delayMs: 0, tempoRatio: 1, durationDeltaMs };
|
||||
}
|
||||
|
||||
|
||||
@@ -194,53 +194,35 @@ export async function muxNativeMacRecordingWithAudio(
|
||||
tempoRatio: 1,
|
||||
durationDeltaMs: 0,
|
||||
};
|
||||
const needsFilter = systemAdjustment.mode !== "none" || micAdjustment.mode !== "none";
|
||||
|
||||
// Always route through the filter graph so that aresample=async=1 is
|
||||
// applied to every audio stream. This corrects progressive clock drift
|
||||
// between the video and audio tracks that a simple duration comparison
|
||||
// cannot detect (e.g. audio gradually falling behind under CPU load).
|
||||
let args: string[];
|
||||
if (availableAudioInputs.length === 2) {
|
||||
if (needsFilter) {
|
||||
const filterParts: string[] = [];
|
||||
appendSyncedAudioFilter(filterParts, "[1:a]", "s", systemAdjustment);
|
||||
appendSyncedAudioFilter(filterParts, "[2:a]", "m", micAdjustment);
|
||||
filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]");
|
||||
args = [
|
||||
"-y",
|
||||
...inputs,
|
||||
"-filter_complex",
|
||||
filterParts.join(";"),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"[aout]",
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
"-shortest",
|
||||
mixedOutputPath,
|
||||
];
|
||||
} else {
|
||||
args = [
|
||||
"-y",
|
||||
...inputs,
|
||||
"-filter_complex",
|
||||
"[1:a][2:a]amix=inputs=2:duration=longest:normalize=0[aout]",
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"[aout]",
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
"-shortest",
|
||||
mixedOutputPath,
|
||||
];
|
||||
}
|
||||
const filterParts: string[] = [];
|
||||
appendSyncedAudioFilter(filterParts, "[1:a]", "s", systemAdjustment);
|
||||
appendSyncedAudioFilter(filterParts, "[2:a]", "m", micAdjustment);
|
||||
filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]");
|
||||
args = [
|
||||
"-y",
|
||||
...inputs,
|
||||
"-filter_complex",
|
||||
filterParts.join(";"),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"[aout]",
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
"-shortest",
|
||||
mixedOutputPath,
|
||||
];
|
||||
} else {
|
||||
const singleAdjustment = audioAdjustments.get(availableAudioInputs[0]) ?? {
|
||||
mode: "none",
|
||||
@@ -248,45 +230,26 @@ export async function muxNativeMacRecordingWithAudio(
|
||||
tempoRatio: 1,
|
||||
durationDeltaMs: 0,
|
||||
};
|
||||
if (singleAdjustment.mode !== "none") {
|
||||
const filterParts: string[] = [];
|
||||
appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment);
|
||||
args = [
|
||||
"-y",
|
||||
...inputs,
|
||||
"-filter_complex",
|
||||
filterParts.join(";"),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"[aout]",
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
"-shortest",
|
||||
mixedOutputPath,
|
||||
];
|
||||
} else {
|
||||
args = [
|
||||
"-y",
|
||||
...inputs,
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"1:a:0",
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
"-shortest",
|
||||
mixedOutputPath,
|
||||
];
|
||||
}
|
||||
const filterParts: string[] = [];
|
||||
appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment);
|
||||
args = [
|
||||
"-y",
|
||||
...inputs,
|
||||
"-filter_complex",
|
||||
filterParts.join(";"),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"[aout]",
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
"-shortest",
|
||||
mixedOutputPath,
|
||||
];
|
||||
}
|
||||
|
||||
console.log("[mux] Running ffmpeg:", ffmpegPath, args.join(" "));
|
||||
|
||||
@@ -278,58 +278,38 @@ export async function muxNativeWindowsVideoWithAudio(
|
||||
durationDeltaMs: 0,
|
||||
};
|
||||
|
||||
if (pauseFilter || singleAdjustment.mode !== "none") {
|
||||
const filterParts: string[] = [];
|
||||
if (pauseFilter) {
|
||||
filterParts.push(pauseFilter);
|
||||
}
|
||||
const srcLabel = pauseFilter ? "[trimmed_audio]" : "[1:a]";
|
||||
appendSyncedAudioFilter(filterParts, srcLabel, "aout", singleAdjustment);
|
||||
|
||||
await execFileAsync(
|
||||
ffmpegPath,
|
||||
[
|
||||
"-y",
|
||||
...inputs,
|
||||
"-filter_complex",
|
||||
filterParts.join(";"),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"[aout]",
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
"-shortest",
|
||||
mixedOutputPath,
|
||||
],
|
||||
{ timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
|
||||
);
|
||||
} else {
|
||||
await execFileAsync(
|
||||
ffmpegPath,
|
||||
[
|
||||
"-y",
|
||||
...inputs,
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"1:a:0",
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
"-shortest",
|
||||
mixedOutputPath,
|
||||
],
|
||||
{ timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
|
||||
);
|
||||
// Always route through the filter graph so that aresample=async=1 is
|
||||
// applied. This corrects progressive clock drift between video and
|
||||
// audio tracks that a simple duration comparison cannot detect.
|
||||
const filterParts: string[] = [];
|
||||
if (pauseFilter) {
|
||||
filterParts.push(pauseFilter);
|
||||
}
|
||||
const srcLabel = pauseFilter ? "[trimmed_audio]" : "[1:a]";
|
||||
appendSyncedAudioFilter(filterParts, srcLabel, "aout", singleAdjustment);
|
||||
|
||||
await execFileAsync(
|
||||
ffmpegPath,
|
||||
[
|
||||
"-y",
|
||||
...inputs,
|
||||
"-filter_complex",
|
||||
filterParts.join(";"),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"[aout]",
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
"-shortest",
|
||||
mixedOutputPath,
|
||||
],
|
||||
{ timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
|
||||
);
|
||||
}
|
||||
|
||||
await moveFileWithOverwrite(mixedOutputPath, videoPath);
|
||||
|
||||
@@ -18,7 +18,7 @@ const HIGH_FRAME_RATE_BOOST = 1.7;
|
||||
const DEFAULT_WIDTH = 1920;
|
||||
const DEFAULT_HEIGHT = 1080;
|
||||
const CODEC_ALIGNMENT = 2;
|
||||
const RECORDER_TIMESLICE_MS = 1000;
|
||||
const RECORDER_TIMESLICE_MS = 250;
|
||||
const BITS_PER_MEGABIT = 1_000_000;
|
||||
const MIN_FRAME_RATE = 30;
|
||||
const CHROME_MEDIA_SOURCE = "desktop";
|
||||
@@ -1090,7 +1090,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
|
||||
const micAudioTrack = microphoneStream.current?.getAudioTracks()[0];
|
||||
|
||||
if (systemAudioTrack && micAudioTrack) {
|
||||
const context = new AudioContext();
|
||||
const context = new AudioContext({ sampleRate: 48000 });
|
||||
mixingContext.current = context;
|
||||
const systemSource = context.createMediaStreamSource(
|
||||
new MediaStream([systemAudioTrack]),
|
||||
|
||||
@@ -555,7 +555,7 @@ export class AudioProcessor {
|
||||
throw new Error("Export cancelled");
|
||||
}
|
||||
|
||||
audioContext = new AudioContext();
|
||||
audioContext = new AudioContext({ sampleRate: 48000 });
|
||||
const currentDestinationNode = audioContext.createMediaStreamDestination();
|
||||
destinationNode = currentDestinationNode;
|
||||
|
||||
@@ -721,7 +721,7 @@ export class AudioProcessor {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
|
||||
if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.15) {
|
||||
audioEl.currentTime = targetTimeSec;
|
||||
}
|
||||
|
||||
@@ -729,6 +729,9 @@ export class AudioProcessor {
|
||||
basePlaybackRate: playbackRate,
|
||||
currentTime: audioEl.currentTime,
|
||||
targetTime: targetTimeSec,
|
||||
toleranceSeconds: 0.008,
|
||||
correctionWindowSeconds: 0.5,
|
||||
maxAdjustment: 0.12,
|
||||
});
|
||||
if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
|
||||
audioEl.playbackRate = syncedPlaybackRate;
|
||||
@@ -748,7 +751,7 @@ export class AudioProcessor {
|
||||
|
||||
if (isInRegion) {
|
||||
const audioOffset = (currentTimeMs - region.startMs) / 1000;
|
||||
if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) {
|
||||
if (Math.abs(audioEl.currentTime - audioOffset) > 0.15) {
|
||||
audioEl.currentTime = audioOffset;
|
||||
}
|
||||
|
||||
@@ -756,6 +759,9 @@ export class AudioProcessor {
|
||||
basePlaybackRate: playbackRate,
|
||||
currentTime: audioEl.currentTime,
|
||||
targetTime: audioOffset,
|
||||
toleranceSeconds: 0.008,
|
||||
correctionWindowSeconds: 0.5,
|
||||
maxAdjustment: 0.12,
|
||||
});
|
||||
if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
|
||||
audioEl.playbackRate = syncedPlaybackRate;
|
||||
|
||||
@@ -23,10 +23,11 @@ describe("clampMediaTimeToDuration", () => {
|
||||
describe("estimateCompanionAudioStartDelaySeconds", () => {
|
||||
it("returns the positive tail gap when companion audio is shorter", () => {
|
||||
expect(estimateCompanionAudioStartDelaySeconds(10, 9.6)).toBeCloseTo(0.4);
|
||||
expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBeCloseTo(0.03);
|
||||
});
|
||||
|
||||
it("ignores tiny or negative differences", () => {
|
||||
expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBe(0);
|
||||
expect(estimateCompanionAudioStartDelaySeconds(10, 9.99)).toBe(0);
|
||||
expect(estimateCompanionAudioStartDelaySeconds(10, 10.5)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -19,7 +19,7 @@ export function estimateCompanionAudioStartDelaySeconds(
|
||||
const safeAudioDuration = Math.max(0, audioDuration ?? 0);
|
||||
const estimatedDelaySeconds = safeTimelineDuration - safeAudioDuration;
|
||||
|
||||
return estimatedDelaySeconds > 0.05 ? estimatedDelaySeconds : 0;
|
||||
return estimatedDelaySeconds > 0.025 ? estimatedDelaySeconds : 0;
|
||||
}
|
||||
|
||||
export function getMediaSyncPlaybackRate({
|
||||
|
||||
Reference in New Issue
Block a user