fix: always apply aresample sync filter and tighten audio sync tolerances

- Always route audio through aresample=async=1:first_pts=0 filter during
  muxing on both macOS and Windows, even when duration delta is small.
  Previously skipped when delta ≤50ms, which left progressive clock drift
  (from CPU load) completely uncorrected.
- Lower sync detection threshold from 50ms to 20ms so tempo correction
  kicks in earlier.
- Use explicit 48kHz AudioContext sample rate in both browser recording
  and export rendering to prevent sample rate mismatch drift.
- Reduce MediaRecorder timeslice from 1000ms to 250ms to reduce chunk
  loss under CPU pressure.
- Tighten export audio sync: seek threshold 300ms→150ms, tolerance
  15ms→8ms, correction window 2s→0.5s, max adjustment ±8%→±12%.
- Lower companion audio start delay threshold from 50ms to 25ms.
This commit is contained in:
webadderall
2026-04-18 19:03:36 +10:00
parent 944d4f8031
commit f024c89890
7 changed files with 92 additions and 142 deletions
+1 -1
View File
@@ -40,7 +40,7 @@ export function getAudioSyncAdjustment(
const durationDeltaMs = Math.round((videoDuration - audioDuration) * 1000);
const absDeltaMs = Math.abs(durationDeltaMs);
if (absDeltaMs <= 50) {
if (absDeltaMs <= 20) {
return { mode: "none", delayMs: 0, tempoRatio: 1, durationDeltaMs };
}
+46 -83
View File
@@ -194,53 +194,35 @@ export async function muxNativeMacRecordingWithAudio(
tempoRatio: 1,
durationDeltaMs: 0,
};
const needsFilter = systemAdjustment.mode !== "none" || micAdjustment.mode !== "none";
// Always route through the filter graph so that aresample=async=1 is
// applied to every audio stream. This corrects progressive clock drift
// between the video and audio tracks that a simple duration comparison
// cannot detect (e.g. audio gradually falling behind under CPU load).
let args: string[];
if (availableAudioInputs.length === 2) {
if (needsFilter) {
const filterParts: string[] = [];
appendSyncedAudioFilter(filterParts, "[1:a]", "s", systemAdjustment);
appendSyncedAudioFilter(filterParts, "[2:a]", "m", micAdjustment);
filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]");
args = [
"-y",
...inputs,
"-filter_complex",
filterParts.join(";"),
"-map",
"0:v:0",
"-map",
"[aout]",
"-c:v",
"copy",
"-c:a",
"aac",
"-b:a",
"192k",
"-shortest",
mixedOutputPath,
];
} else {
args = [
"-y",
...inputs,
"-filter_complex",
"[1:a][2:a]amix=inputs=2:duration=longest:normalize=0[aout]",
"-map",
"0:v:0",
"-map",
"[aout]",
"-c:v",
"copy",
"-c:a",
"aac",
"-b:a",
"192k",
"-shortest",
mixedOutputPath,
];
}
const filterParts: string[] = [];
appendSyncedAudioFilter(filterParts, "[1:a]", "s", systemAdjustment);
appendSyncedAudioFilter(filterParts, "[2:a]", "m", micAdjustment);
filterParts.push("[s][m]amix=inputs=2:duration=longest:normalize=0[aout]");
args = [
"-y",
...inputs,
"-filter_complex",
filterParts.join(";"),
"-map",
"0:v:0",
"-map",
"[aout]",
"-c:v",
"copy",
"-c:a",
"aac",
"-b:a",
"192k",
"-shortest",
mixedOutputPath,
];
} else {
const singleAdjustment = audioAdjustments.get(availableAudioInputs[0]) ?? {
mode: "none",
@@ -248,45 +230,26 @@ export async function muxNativeMacRecordingWithAudio(
tempoRatio: 1,
durationDeltaMs: 0,
};
if (singleAdjustment.mode !== "none") {
const filterParts: string[] = [];
appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment);
args = [
"-y",
...inputs,
"-filter_complex",
filterParts.join(";"),
"-map",
"0:v:0",
"-map",
"[aout]",
"-c:v",
"copy",
"-c:a",
"aac",
"-b:a",
"192k",
"-shortest",
mixedOutputPath,
];
} else {
args = [
"-y",
...inputs,
"-map",
"0:v:0",
"-map",
"1:a:0",
"-c:v",
"copy",
"-c:a",
"aac",
"-b:a",
"192k",
"-shortest",
mixedOutputPath,
];
}
const filterParts: string[] = [];
appendSyncedAudioFilter(filterParts, "[1:a]", "aout", singleAdjustment);
args = [
"-y",
...inputs,
"-filter_complex",
filterParts.join(";"),
"-map",
"0:v:0",
"-map",
"[aout]",
"-c:v",
"copy",
"-c:a",
"aac",
"-b:a",
"192k",
"-shortest",
mixedOutputPath,
];
}
console.log("[mux] Running ffmpeg:", ffmpegPath, args.join(" "));
+31 -51
View File
@@ -278,58 +278,38 @@ export async function muxNativeWindowsVideoWithAudio(
durationDeltaMs: 0,
};
if (pauseFilter || singleAdjustment.mode !== "none") {
const filterParts: string[] = [];
if (pauseFilter) {
filterParts.push(pauseFilter);
}
const srcLabel = pauseFilter ? "[trimmed_audio]" : "[1:a]";
appendSyncedAudioFilter(filterParts, srcLabel, "aout", singleAdjustment);
await execFileAsync(
ffmpegPath,
[
"-y",
...inputs,
"-filter_complex",
filterParts.join(";"),
"-map",
"0:v:0",
"-map",
"[aout]",
"-c:v",
"copy",
"-c:a",
"aac",
"-b:a",
"192k",
"-shortest",
mixedOutputPath,
],
{ timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
);
} else {
await execFileAsync(
ffmpegPath,
[
"-y",
...inputs,
"-map",
"0:v:0",
"-map",
"1:a:0",
"-c:v",
"copy",
"-c:a",
"aac",
"-b:a",
"192k",
"-shortest",
mixedOutputPath,
],
{ timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
);
// Always route through the filter graph so that aresample=async=1 is
// applied. This corrects progressive clock drift between video and
// audio tracks that a simple duration comparison cannot detect.
const filterParts: string[] = [];
if (pauseFilter) {
filterParts.push(pauseFilter);
}
const srcLabel = pauseFilter ? "[trimmed_audio]" : "[1:a]";
appendSyncedAudioFilter(filterParts, srcLabel, "aout", singleAdjustment);
await execFileAsync(
ffmpegPath,
[
"-y",
...inputs,
"-filter_complex",
filterParts.join(";"),
"-map",
"0:v:0",
"-map",
"[aout]",
"-c:v",
"copy",
"-c:a",
"aac",
"-b:a",
"192k",
"-shortest",
mixedOutputPath,
],
{ timeout: 120000, maxBuffer: 10 * 1024 * 1024 },
);
}
await moveFileWithOverwrite(mixedOutputPath, videoPath);
+2 -2
View File
@@ -18,7 +18,7 @@ const HIGH_FRAME_RATE_BOOST = 1.7;
const DEFAULT_WIDTH = 1920;
const DEFAULT_HEIGHT = 1080;
const CODEC_ALIGNMENT = 2;
const RECORDER_TIMESLICE_MS = 1000;
const RECORDER_TIMESLICE_MS = 250;
const BITS_PER_MEGABIT = 1_000_000;
const MIN_FRAME_RATE = 30;
const CHROME_MEDIA_SOURCE = "desktop";
@@ -1090,7 +1090,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn {
const micAudioTrack = microphoneStream.current?.getAudioTracks()[0];
if (systemAudioTrack && micAudioTrack) {
const context = new AudioContext();
const context = new AudioContext({ sampleRate: 48000 });
mixingContext.current = context;
const systemSource = context.createMediaStreamSource(
new MediaStream([systemAudioTrack]),
+9 -3
View File
@@ -555,7 +555,7 @@ export class AudioProcessor {
throw new Error("Export cancelled");
}
audioContext = new AudioContext();
audioContext = new AudioContext({ sampleRate: 48000 });
const currentDestinationNode = audioContext.createMediaStreamDestination();
destinationNode = currentDestinationNode;
@@ -721,7 +721,7 @@ export class AudioProcessor {
continue;
}
if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) {
if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.15) {
audioEl.currentTime = targetTimeSec;
}
@@ -729,6 +729,9 @@ export class AudioProcessor {
basePlaybackRate: playbackRate,
currentTime: audioEl.currentTime,
targetTime: targetTimeSec,
toleranceSeconds: 0.008,
correctionWindowSeconds: 0.5,
maxAdjustment: 0.12,
});
if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
audioEl.playbackRate = syncedPlaybackRate;
@@ -748,7 +751,7 @@ export class AudioProcessor {
if (isInRegion) {
const audioOffset = (currentTimeMs - region.startMs) / 1000;
if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) {
if (Math.abs(audioEl.currentTime - audioOffset) > 0.15) {
audioEl.currentTime = audioOffset;
}
@@ -756,6 +759,9 @@ export class AudioProcessor {
basePlaybackRate: playbackRate,
currentTime: audioEl.currentTime,
targetTime: audioOffset,
toleranceSeconds: 0.008,
correctionWindowSeconds: 0.5,
maxAdjustment: 0.12,
});
if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) {
audioEl.playbackRate = syncedPlaybackRate;
+2 -1
View File
@@ -23,10 +23,11 @@ describe("clampMediaTimeToDuration", () => {
describe("estimateCompanionAudioStartDelaySeconds", () => {
it("returns the positive tail gap when companion audio is shorter", () => {
expect(estimateCompanionAudioStartDelaySeconds(10, 9.6)).toBeCloseTo(0.4);
expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBeCloseTo(0.03);
});
it("ignores tiny or negative differences", () => {
expect(estimateCompanionAudioStartDelaySeconds(10, 9.97)).toBe(0);
expect(estimateCompanionAudioStartDelaySeconds(10, 9.99)).toBe(0);
expect(estimateCompanionAudioStartDelaySeconds(10, 10.5)).toBe(0);
});
});
+1 -1
View File
@@ -19,7 +19,7 @@ export function estimateCompanionAudioStartDelaySeconds(
const safeAudioDuration = Math.max(0, audioDuration ?? 0);
const estimatedDelaySeconds = safeTimelineDuration - safeAudioDuration;
return estimatedDelaySeconds > 0.05 ? estimatedDelaySeconds : 0;
return estimatedDelaySeconds > 0.025 ? estimatedDelaySeconds : 0;
}
export function getMediaSyncPlaybackRate({