diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts index f1a6dd21..c8fad00e 100644 --- a/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts +++ b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts @@ -111,6 +111,24 @@ describe("buildInteractionZoomSuggestions (click-cluster logic)", () => { expect(s.end).toBe(lastClickMs + CLICK_CLUSTER_PAD_MS); }); + it("weights every click coordinate equally when choosing a cluster focus", () => { + const telemetry = withMoves( + [makeClick(4_000, 0.2, 0.3), makeClick(5_000, 0.8, 0.7)], + TOTAL_MS, + ); + + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: telemetry, + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toHaveLength(1); + expect(result.suggestions[0].focus.cx).toBeCloseTo(0.5); + expect(result.suggestions[0].focus.cy).toBeCloseTo(0.5); + }); + it("splits two clicks more than 2500ms apart into separate zoom tracks", () => { const click1 = 3_000; const click2 = 3_000 + CLICK_CLUSTER_MERGE_GAP_MS + 1; // just outside the merge gap @@ -170,7 +188,7 @@ describe("buildInteractionZoomSuggestions (click-cluster logic)", () => { expect(result.suggestions).toHaveLength(0); }); - it("ignores dwell-derived click-like heuristics when there are no explicit clicks", () => { + it("does not infer clicks from cursor dwell time", () => { const telemetry: CursorTelemetryPoint[] = [ makeMove(0, 0.5, 0.5), makeMove(200, 0.5005, 0.5005), diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.ts index bf38efa6..418c3dab 100644 --- a/src/components/video-editor/timeline/zoomSuggestionUtils.ts +++ b/src/components/video-editor/timeline/zoomSuggestionUtils.ts @@ -1,26 +1,10 @@ import type { CursorTelemetryPoint, ZoomFocus } from "../types"; -export const MIN_DWELL_DURATION_MS = 450; -export const MAX_DWELL_DURATION_MS = 2600; -export const DWELL_MOVE_THRESHOLD = 0.02; export const MIN_FRESH_RECORDING_AUTO_ZOOM_SOURCE_ASPECT_RATIO = 1.2; -export interface ZoomDwellCandidate { +interface ClickCandidate { centerTimeMs: number; focus: ZoomFocus; - strength: number; -} - -export interface CursorInteractionCandidate extends ZoomDwellCandidate { - kind: - | "dwell" - | "click-like" - | "double-click-like" - | "text-focus-like" - | "dropdown-open" - | "text-selection" - | "text-field-click"; - source: "explicit" | "heuristic"; } export interface SuggestedZoomRegion { @@ -89,25 +73,11 @@ function normalizeTelemetrySample( }; } -function applyCursorTypeInRange( - samples: CursorTelemetryPoint[], - startMs: number, - endMs: number, - cursorType: NonNullable, -) { - for (const sample of samples) { - if (sample.timeMs < startMs || sample.timeMs > endMs) continue; - if (!sample.cursorType) { - sample.cursorType = cursorType; - } - } -} - export function normalizeCursorTelemetry( telemetry: CursorTelemetryPoint[], totalMs: number, ): CursorTelemetryPoint[] { - const normalized = [...telemetry] + return [...telemetry] .filter( (sample) => Number.isFinite(sample.timeMs) && @@ -116,188 +86,16 @@ export function normalizeCursorTelemetry( ) .sort((a, b) => a.timeMs - b.timeMs) .map((sample) => normalizeTelemetrySample(sample, totalMs)); - - const interactions = detectInteractionCandidates(normalized); - for (const candidate of interactions) { - if (candidate.kind === "text-selection") { - applyCursorTypeInRange( - normalized, - candidate.centerTimeMs - 140, - candidate.centerTimeMs + 1200, - "text", - ); - continue; - } - - if (candidate.kind === "text-field-click" || candidate.kind === "text-focus-like") { - applyCursorTypeInRange( - normalized, - candidate.centerTimeMs - 100, - candidate.centerTimeMs + 900, - "text", - ); - continue; - } - } - - for (const sample of normalized) { - if (sample.interactionType !== "click" && sample.interactionType !== "double-click") { - continue; - } - - const mouseUp = normalized.find( - (candidate) => - candidate.timeMs > sample.timeMs && candidate.interactionType === "mouseup", - ); - if (!mouseUp) { - continue; - } - - const dragDuration = mouseUp.timeMs - sample.timeMs; - const dragDistance = Math.hypot(mouseUp.cx - sample.cx, mouseUp.cy - sample.cy); - if (dragDuration >= 160 && dragDistance > 0.015) { - const isTextDrag = - Math.abs(mouseUp.cx - sample.cx) > Math.abs(mouseUp.cy - sample.cy) * 1.8; - applyCursorTypeInRange( - normalized, - sample.timeMs, - mouseUp.timeMs, - isTextDrag ? "text" : "closed-hand", - ); - } - } - - return normalized; -} - -export function detectZoomDwellCandidates(samples: CursorTelemetryPoint[]): ZoomDwellCandidate[] { - if (samples.length < 2) { - return []; - } - - const dwellCandidates: ZoomDwellCandidate[] = []; - let runStart = 0; - - const pushRunIfDwell = (startIndex: number, endIndexExclusive: number) => { - if (endIndexExclusive - startIndex < 2) { - return; - } - - const start = samples[startIndex]; - const end = samples[endIndexExclusive - 1]; - const runDuration = end.timeMs - start.timeMs; - if (runDuration < MIN_DWELL_DURATION_MS || runDuration > MAX_DWELL_DURATION_MS) { - return; - } - - const runSamples = samples.slice(startIndex, endIndexExclusive); - const avgCx = runSamples.reduce((sum, sample) => sum + sample.cx, 0) / runSamples.length; - const avgCy = runSamples.reduce((sum, sample) => sum + sample.cy, 0) / runSamples.length; - - dwellCandidates.push({ - centerTimeMs: Math.round((start.timeMs + end.timeMs) / 2), - focus: { cx: avgCx, cy: avgCy }, - strength: runDuration, - }); - }; - - for (let index = 1; index < samples.length; index += 1) { - const prev = samples[index - 1]; - const curr = samples[index]; - const distance = Math.hypot(curr.cx - prev.cx, curr.cy - prev.cy); - - if (distance > DWELL_MOVE_THRESHOLD) { - pushRunIfDwell(runStart, index); - runStart = index; - } - } - pushRunIfDwell(runStart, samples.length); - - return dwellCandidates; -} - -export function detectInteractionCandidates( - samples: CursorTelemetryPoint[], -): CursorInteractionCandidate[] { - // --- Phase 1: Explicit interaction events (from uiohook telemetry) --- - const clickEvents = samples.filter((sample) => isExplicitClickType(sample.interactionType)); - - const explicitInteractionCandidates: CursorInteractionCandidate[] = []; - - for (const clickSample of clickEvents) { - // Classify what happened AFTER this click by analyzing cursor trajectory - const kind = classifyPostClickBehavior(samples, clickSample); - - const baseStrength = - kind === "double-click-like" - ? 1500 - : kind === "dropdown-open" - ? 1200 - : kind === "text-selection" - ? 1300 - : kind === "text-field-click" - ? 1100 - : 900; - - explicitInteractionCandidates.push({ - centerTimeMs: Math.round(clickSample.timeMs), - focus: { cx: clickSample.cx, cy: clickSample.cy }, - strength: baseStrength, - kind, - source: "explicit", - }); - } - - // --- Phase 2: Dwell-based heuristic candidates --- - const dwellCandidates = detectZoomDwellCandidates(samples).map( - (candidate) => { - if (candidate.strength >= 1100) { - return { ...candidate, kind: "text-focus-like", source: "heuristic" }; - } - if (candidate.strength <= 800) { - return { ...candidate, kind: "click-like", source: "heuristic" }; - } - return { ...candidate, kind: "dwell", source: "heuristic" }; - }, - ); - - // --- Phase 3: Synthetic double-click detection from dwell pairs --- - const doubleClickCandidates: CursorInteractionCandidate[] = []; - const sortedByTime = [...dwellCandidates].sort((a, b) => a.centerTimeMs - b.centerTimeMs); - - for (let index = 1; index < sortedByTime.length; index += 1) { - const prev = sortedByTime[index - 1]; - const curr = sortedByTime[index]; - const timeGap = curr.centerTimeMs - prev.centerTimeMs; - const spatialGap = Math.hypot(curr.focus.cx - prev.focus.cx, curr.focus.cy - prev.focus.cy); - const bothShort = prev.strength <= 900 && curr.strength <= 900; - - if (bothShort && timeGap <= 450 && spatialGap <= 0.035) { - doubleClickCandidates.push({ - centerTimeMs: Math.round((prev.centerTimeMs + curr.centerTimeMs) / 2), - focus: { - cx: (prev.focus.cx + curr.focus.cx) / 2, - cy: (prev.focus.cy + curr.focus.cy) / 2, - }, - strength: prev.strength + curr.strength + 500, - kind: "double-click-like", - source: "heuristic", - }); - } - } - - return [...explicitInteractionCandidates, ...dwellCandidates, ...doubleClickCandidates]; } /** * Groups a sorted list of click timestamps into clusters where consecutive * clicks are no more than `mergeGapMs` apart. Returns an array of - * `{ firstMs, lastMs, focus }` objects, one per cluster. The focus is taken - * from the click with the highest interaction strength, falling back to the - * centroid of all clicks in the cluster. + * `{ firstMs, lastMs, focus }` objects, one per cluster. Every click is weighted + * equally, so the focus is the centroid of all click coordinates in the cluster. */ function buildClickClusters( - clicks: CursorInteractionCandidate[], + clicks: ClickCandidate[], mergeGapMs: number, ): Array<{ firstMs: number; lastMs: number; focus: ZoomFocus }> { if (clicks.length === 0) { @@ -309,8 +107,6 @@ function buildClickClusters( let clusterStart = sorted[0].centerTimeMs; let clusterEnd = sorted[0].centerTimeMs; - let bestStrength = sorted[0].strength; - let bestFocus = sorted[0].focus; let sumCx = sorted[0].focus.cx; let sumCy = sorted[0].focus.cy; let count = 1; @@ -322,10 +118,6 @@ function buildClickClusters( if (gap <= mergeGapMs) { // Extend current cluster clusterEnd = Math.max(clusterEnd, click.centerTimeMs); - if (click.strength > bestStrength) { - bestStrength = click.strength; - bestFocus = click.focus; - } sumCx += click.focus.cx; sumCy += click.focus.cy; count += 1; @@ -334,12 +126,10 @@ function buildClickClusters( clusters.push({ firstMs: clusterStart, lastMs: clusterEnd, - focus: bestFocus ?? { cx: sumCx / count, cy: sumCy / count }, + focus: { cx: sumCx / count, cy: sumCy / count }, }); clusterStart = click.centerTimeMs; clusterEnd = click.centerTimeMs; - bestStrength = click.strength; - bestFocus = click.focus; sumCx = click.focus.cx; sumCy = click.focus.cy; count = 1; @@ -350,7 +140,7 @@ function buildClickClusters( clusters.push({ firstMs: clusterStart, lastMs: clusterEnd, - focus: bestFocus ?? { cx: sumCx / count, cy: sumCy / count }, + focus: { cx: sumCx / count, cy: sumCy / count }, }); return clusters; @@ -389,10 +179,12 @@ export function buildInteractionZoomSuggestions(params: { return { status: "no-telemetry", suggestions: [] }; } - // Only use explicit click events (uiohook telemetry) – ignore dwell heuristics - const clickCandidates = detectInteractionCandidates(normalizedSamples).filter( - (candidate) => candidate.source === "explicit", - ); + const clickCandidates: ClickCandidate[] = normalizedSamples + .filter((sample) => isExplicitClickType(sample.interactionType)) + .map((sample) => ({ + centerTimeMs: Math.round(sample.timeMs), + focus: { cx: sample.cx, cy: sample.cy }, + })); if (clickCandidates.length === 0) { return { status: "no-interactions", suggestions: [] }; @@ -437,85 +229,3 @@ export function buildInteractionZoomSuggestions(params: { return { status: "ok", suggestions }; } - -/** - * Analyzes cursor movement after a click to classify the interaction pattern. - * - * - **dropdown-open**: click followed by slow downward cursor movement (browsing items) - * - **text-selection**: click followed by primarily horizontal drag movement - * - **text-field-click**: click followed by cursor staying mostly still (dwell) - * - **double-click-like**: explicit double-click interaction type - * - **click-like**: generic click with no recognizable post-click pattern - */ -function classifyPostClickBehavior( - samples: CursorTelemetryPoint[], - clickSample: CursorTelemetryPoint, -): CursorInteractionCandidate["kind"] { - // Explicit double-click from uiohook - if (clickSample.interactionType === "double-click") { - return "double-click-like"; - } - - const clickTime = clickSample.timeMs; - - // Check for mouseup shortly after (drag detection) - const mouseUpAfter = samples.find( - (s) => - s.interactionType === "mouseup" && s.timeMs > clickTime && s.timeMs - clickTime < 3000, - ); - - if (mouseUpAfter) { - const dragDx = Math.abs(mouseUpAfter.cx - clickSample.cx); - const dragDy = Math.abs(mouseUpAfter.cy - clickSample.cy); - const dragDuration = mouseUpAfter.timeMs - clickTime; - - // Text selection: horizontal drag > 3% of screen, mostly horizontal, duration 200ms+ - if (dragDuration >= 200 && dragDx > 0.03 && dragDx > dragDy * 1.8) { - return "text-selection"; - } - } - - // Analyze trajectory in the 400ms-2000ms window after click - const moveSamples = samples.filter( - (s) => - s.timeMs > clickTime + 100 && - s.timeMs <= clickTime + 2000 && - (s.interactionType === "move" || !s.interactionType), - ); - - if (moveSamples.length < 3) { - // Very few move samples after click = cursor stayed still = text field click - return "text-field-click"; - } - - // Compute displacement from click position - let maxDist = 0; - let totalAbsDy = 0; - let totalAbsDx = 0; - for (const s of moveSamples) { - const dist = Math.hypot(s.cx - clickSample.cx, s.cy - clickSample.cy); - maxDist = Math.max(maxDist, dist); - totalAbsDx += Math.abs(s.cx - clickSample.cx); - totalAbsDy += Math.abs(s.cy - clickSample.cy); - } - - // Cursor barely moved after click: text field click (dwell) - if (maxDist < 0.02) { - return "text-field-click"; - } - - // Primarily downward movement after click: dropdown open - const lastMoveSample = moveSamples[moveSamples.length - 1]; - const netDy = lastMoveSample.cy - clickSample.cy; - - if (netDy > 0.03 && totalAbsDy > totalAbsDx * 1.5) { - return "dropdown-open"; - } - - // Primarily horizontal movement: text selection (fallback if no mouseup) - if (totalAbsDx > 0.03 && totalAbsDx > totalAbsDy * 1.8) { - return "text-selection"; - } - - return "click-like"; -}