lee/Recordly
1
0
Fork 0
mirror of https://github.com/webadderallorg/Recordly.git synced 2026-09-24 06:46:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Simplify automatic zoom click handling

Browse Source
This commit is contained in:
webadderall
2026-08-28 09:47:07 +10:00
parent 60e186234b
commit 75d49b1bd6
2 changed files with 32 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/video-editor/timeline/zoomSuggestionUtils.test.ts
+19 -1
View File
@@ -111,6 +111,24 @@ describe("buildInteractionZoomSuggestions (click-cluster logic)", () => {
expect(s.end).toBe(lastClickMs + CLICK_CLUSTER_PAD_MS);
});
it("weights every click coordinate equally when choosing a cluster focus", () => {
const telemetry = withMoves(
[makeClick(4_000, 0.2, 0.3), makeClick(5_000, 0.8, 0.7)],
TOTAL_MS,
);
const result = buildInteractionZoomSuggestions({
cursorTelemetry: telemetry,
totalMs: TOTAL_MS,
defaultDurationMs: 3_000,
});
expect(result.status).toBe("ok");
expect(result.suggestions).toHaveLength(1);
expect(result.suggestions[0].focus.cx).toBeCloseTo(0.5);
expect(result.suggestions[0].focus.cy).toBeCloseTo(0.5);
});
it("splits two clicks more than 2500ms apart into separate zoom tracks", () => {
const click1 = 3_000;
const click2 = 3_000 + CLICK_CLUSTER_MERGE_GAP_MS + 1; // just outside the merge gap
@@ -170,7 +188,7 @@ describe("buildInteractionZoomSuggestions (click-cluster logic)", () => {
expect(result.suggestions).toHaveLength(0);
});
it("ignores dwell-derived click-like heuristics when there are no explicit clicks", () => {
it("does not infer clicks from cursor dwell time", () => {
const telemetry: CursorTelemetryPoint[] = [
makeMove(0, 0.5, 0.5),
makeMove(200, 0.5005, 0.5005),
src/components/video-editor/timeline/zoomSuggestionUtils.ts
+13 -303
View File
@@ -1,26 +1,10 @@
import type { CursorTelemetryPoint, ZoomFocus } from "../types";
export const MIN_DWELL_DURATION_MS = 450;
export const MAX_DWELL_DURATION_MS = 2600;
export const DWELL_MOVE_THRESHOLD = 0.02;
export const MIN_FRESH_RECORDING_AUTO_ZOOM_SOURCE_ASPECT_RATIO = 1.2;
export interface ZoomDwellCandidate {
interface ClickCandidate {
centerTimeMs: number;
focus: ZoomFocus;
strength: number;
}
export interface CursorInteractionCandidate extends ZoomDwellCandidate {
kind:
| "dwell"
| "click-like"
| "double-click-like"
| "text-focus-like"
| "dropdown-open"
| "text-selection"
| "text-field-click";
source: "explicit" | "heuristic";
}
export interface SuggestedZoomRegion {
@@ -89,25 +73,11 @@ function normalizeTelemetrySample(
};
}
function applyCursorTypeInRange(
samples: CursorTelemetryPoint[],
startMs: number,
endMs: number,
cursorType: NonNullable<CursorTelemetryPoint["cursorType"]>,
) {
for (const sample of samples) {
if (sample.timeMs < startMs || sample.timeMs > endMs) continue;
if (!sample.cursorType) {
sample.cursorType = cursorType;
}
}
}
export function normalizeCursorTelemetry(
telemetry: CursorTelemetryPoint[],
totalMs: number,
): CursorTelemetryPoint[] {
const normalized = [...telemetry]
return [...telemetry]
.filter(
(sample) =>
Number.isFinite(sample.timeMs) &&
@@ -116,188 +86,16 @@ export function normalizeCursorTelemetry(
)
.sort((a, b) => a.timeMs - b.timeMs)
.map((sample) => normalizeTelemetrySample(sample, totalMs));
const interactions = detectInteractionCandidates(normalized);
for (const candidate of interactions) {
if (candidate.kind === "text-selection") {
applyCursorTypeInRange(
normalized,
candidate.centerTimeMs - 140,
candidate.centerTimeMs + 1200,
"text",
);
continue;
}
if (candidate.kind === "text-field-click" || candidate.kind === "text-focus-like") {
applyCursorTypeInRange(
normalized,
candidate.centerTimeMs - 100,
candidate.centerTimeMs + 900,
"text",
);
continue;
}
}
for (const sample of normalized) {
if (sample.interactionType !== "click" && sample.interactionType !== "double-click") {
continue;
}
const mouseUp = normalized.find(
(candidate) =>
candidate.timeMs > sample.timeMs && candidate.interactionType === "mouseup",
);
if (!mouseUp) {
continue;
}
const dragDuration = mouseUp.timeMs - sample.timeMs;
const dragDistance = Math.hypot(mouseUp.cx - sample.cx, mouseUp.cy - sample.cy);
if (dragDuration >= 160 && dragDistance > 0.015) {
const isTextDrag =
Math.abs(mouseUp.cx - sample.cx) > Math.abs(mouseUp.cy - sample.cy) * 1.8;
applyCursorTypeInRange(
normalized,
sample.timeMs,
mouseUp.timeMs,
isTextDrag ? "text" : "closed-hand",
);
}
}
return normalized;
}
export function detectZoomDwellCandidates(samples: CursorTelemetryPoint[]): ZoomDwellCandidate[] {
if (samples.length < 2) {
return [];
}
const dwellCandidates: ZoomDwellCandidate[] = [];
let runStart = 0;
const pushRunIfDwell = (startIndex: number, endIndexExclusive: number) => {
if (endIndexExclusive - startIndex < 2) {
return;
}
const start = samples[startIndex];
const end = samples[endIndexExclusive - 1];
const runDuration = end.timeMs - start.timeMs;
if (runDuration < MIN_DWELL_DURATION_MS || runDuration > MAX_DWELL_DURATION_MS) {
return;
}
const runSamples = samples.slice(startIndex, endIndexExclusive);
const avgCx = runSamples.reduce((sum, sample) => sum + sample.cx, 0) / runSamples.length;
const avgCy = runSamples.reduce((sum, sample) => sum + sample.cy, 0) / runSamples.length;
dwellCandidates.push({
centerTimeMs: Math.round((start.timeMs + end.timeMs) / 2),
focus: { cx: avgCx, cy: avgCy },
strength: runDuration,
});
};
for (let index = 1; index < samples.length; index += 1) {
const prev = samples[index - 1];
const curr = samples[index];
const distance = Math.hypot(curr.cx - prev.cx, curr.cy - prev.cy);
if (distance > DWELL_MOVE_THRESHOLD) {
pushRunIfDwell(runStart, index);
runStart = index;
}
}
pushRunIfDwell(runStart, samples.length);
return dwellCandidates;
}
export function detectInteractionCandidates(
samples: CursorTelemetryPoint[],
): CursorInteractionCandidate[] {
// --- Phase 1: Explicit interaction events (from uiohook telemetry) ---
const clickEvents = samples.filter((sample) => isExplicitClickType(sample.interactionType));
const explicitInteractionCandidates: CursorInteractionCandidate[] = [];
for (const clickSample of clickEvents) {
// Classify what happened AFTER this click by analyzing cursor trajectory
const kind = classifyPostClickBehavior(samples, clickSample);
const baseStrength =
kind === "double-click-like"
? 1500
: kind === "dropdown-open"
? 1200
: kind === "text-selection"
? 1300
: kind === "text-field-click"
? 1100
: 900;
explicitInteractionCandidates.push({
centerTimeMs: Math.round(clickSample.timeMs),
focus: { cx: clickSample.cx, cy: clickSample.cy },
strength: baseStrength,
kind,
source: "explicit",
});
}
// --- Phase 2: Dwell-based heuristic candidates ---
const dwellCandidates = detectZoomDwellCandidates(samples).map<CursorInteractionCandidate>(
(candidate) => {
if (candidate.strength >= 1100) {
return { ...candidate, kind: "text-focus-like", source: "heuristic" };
}
if (candidate.strength <= 800) {
return { ...candidate, kind: "click-like", source: "heuristic" };
}
return { ...candidate, kind: "dwell", source: "heuristic" };
},
);
// --- Phase 3: Synthetic double-click detection from dwell pairs ---
const doubleClickCandidates: CursorInteractionCandidate[] = [];
const sortedByTime = [...dwellCandidates].sort((a, b) => a.centerTimeMs - b.centerTimeMs);
for (let index = 1; index < sortedByTime.length; index += 1) {
const prev = sortedByTime[index - 1];
const curr = sortedByTime[index];
const timeGap = curr.centerTimeMs - prev.centerTimeMs;
const spatialGap = Math.hypot(curr.focus.cx - prev.focus.cx, curr.focus.cy - prev.focus.cy);
const bothShort = prev.strength <= 900 && curr.strength <= 900;
if (bothShort && timeGap <= 450 && spatialGap <= 0.035) {
doubleClickCandidates.push({
centerTimeMs: Math.round((prev.centerTimeMs + curr.centerTimeMs) / 2),
focus: {
cx: (prev.focus.cx + curr.focus.cx) / 2,
cy: (prev.focus.cy + curr.focus.cy) / 2,
},
strength: prev.strength + curr.strength + 500,
kind: "double-click-like",
source: "heuristic",
});
}
}
return [...explicitInteractionCandidates, ...dwellCandidates, ...doubleClickCandidates];
}
/**
* Groups a sorted list of click timestamps into clusters where consecutive
* clicks are no more than `mergeGapMs` apart. Returns an array of
* `{ firstMs, lastMs, focus }` objects, one per cluster. The focus is taken
* from the click with the highest interaction strength, falling back to the
* centroid of all clicks in the cluster.
* `{ firstMs, lastMs, focus }` objects, one per cluster. Every click is weighted
* equally, so the focus is the centroid of all click coordinates in the cluster.
*/
function buildClickClusters(
clicks: CursorInteractionCandidate[],
clicks: ClickCandidate[],
mergeGapMs: number,
): Array<{ firstMs: number; lastMs: number; focus: ZoomFocus }> {
if (clicks.length === 0) {
@@ -309,8 +107,6 @@ function buildClickClusters(
let clusterStart = sorted[0].centerTimeMs;
let clusterEnd = sorted[0].centerTimeMs;
let bestStrength = sorted[0].strength;
let bestFocus = sorted[0].focus;
let sumCx = sorted[0].focus.cx;
let sumCy = sorted[0].focus.cy;
let count = 1;
@@ -322,10 +118,6 @@ function buildClickClusters(
if (gap <= mergeGapMs) {
// Extend current cluster
clusterEnd = Math.max(clusterEnd, click.centerTimeMs);
if (click.strength > bestStrength) {
bestStrength = click.strength;
bestFocus = click.focus;
}
sumCx += click.focus.cx;
sumCy += click.focus.cy;
count += 1;
@@ -334,12 +126,10 @@ function buildClickClusters(
clusters.push({
firstMs: clusterStart,
lastMs: clusterEnd,
focus: bestFocus ?? { cx: sumCx / count, cy: sumCy / count },
focus: { cx: sumCx / count, cy: sumCy / count },
});
clusterStart = click.centerTimeMs;
clusterEnd = click.centerTimeMs;
bestStrength = click.strength;
bestFocus = click.focus;
sumCx = click.focus.cx;
sumCy = click.focus.cy;
count = 1;
@@ -350,7 +140,7 @@ function buildClickClusters(
clusters.push({
firstMs: clusterStart,
lastMs: clusterEnd,
focus: bestFocus ?? { cx: sumCx / count, cy: sumCy / count },
focus: { cx: sumCx / count, cy: sumCy / count },
});
return clusters;
@@ -389,10 +179,12 @@ export function buildInteractionZoomSuggestions(params: {
return { status: "no-telemetry", suggestions: [] };
}
// Only use explicit click events (uiohook telemetry) – ignore dwell heuristics
const clickCandidates = detectInteractionCandidates(normalizedSamples).filter(
(candidate) => candidate.source === "explicit",
);
const clickCandidates: ClickCandidate[] = normalizedSamples
.filter((sample) => isExplicitClickType(sample.interactionType))
.map((sample) => ({
centerTimeMs: Math.round(sample.timeMs),
focus: { cx: sample.cx, cy: sample.cy },
}));
if (clickCandidates.length === 0) {
return { status: "no-interactions", suggestions: [] };
@@ -437,85 +229,3 @@ export function buildInteractionZoomSuggestions(params: {
return { status: "ok", suggestions };
}
/**
* Analyzes cursor movement after a click to classify the interaction pattern.
*
* - **dropdown-open**: click followed by slow downward cursor movement (browsing items)
* - **text-selection**: click followed by primarily horizontal drag movement
* - **text-field-click**: click followed by cursor staying mostly still (dwell)
* - **double-click-like**: explicit double-click interaction type
* - **click-like**: generic click with no recognizable post-click pattern
*/
function classifyPostClickBehavior(
samples: CursorTelemetryPoint[],
clickSample: CursorTelemetryPoint,
): CursorInteractionCandidate["kind"] {
// Explicit double-click from uiohook
if (clickSample.interactionType === "double-click") {
return "double-click-like";
}
const clickTime = clickSample.timeMs;
// Check for mouseup shortly after (drag detection)
const mouseUpAfter = samples.find(
(s) =>
s.interactionType === "mouseup" && s.timeMs > clickTime && s.timeMs - clickTime < 3000,
);
if (mouseUpAfter) {
const dragDx = Math.abs(mouseUpAfter.cx - clickSample.cx);
const dragDy = Math.abs(mouseUpAfter.cy - clickSample.cy);
const dragDuration = mouseUpAfter.timeMs - clickTime;
// Text selection: horizontal drag > 3% of screen, mostly horizontal, duration 200ms+
if (dragDuration >= 200 && dragDx > 0.03 && dragDx > dragDy * 1.8) {
return "text-selection";
}
}
// Analyze trajectory in the 400ms-2000ms window after click
const moveSamples = samples.filter(
(s) =>
s.timeMs > clickTime + 100 &&
s.timeMs <= clickTime + 2000 &&
(s.interactionType === "move" || !s.interactionType),
);
if (moveSamples.length < 3) {
// Very few move samples after click = cursor stayed still = text field click
return "text-field-click";
}
// Compute displacement from click position
let maxDist = 0;
let totalAbsDy = 0;
let totalAbsDx = 0;
for (const s of moveSamples) {
const dist = Math.hypot(s.cx - clickSample.cx, s.cy - clickSample.cy);
maxDist = Math.max(maxDist, dist);
totalAbsDx += Math.abs(s.cx - clickSample.cx);
totalAbsDy += Math.abs(s.cy - clickSample.cy);
}
// Cursor barely moved after click: text field click (dwell)
if (maxDist < 0.02) {
return "text-field-click";
}
// Primarily downward movement after click: dropdown open
const lastMoveSample = moveSamples[moveSamples.length - 1];
const netDy = lastMoveSample.cy - clickSample.cy;
if (netDy > 0.03 && totalAbsDy > totalAbsDx * 1.5) {
return "dropdown-open";
}
// Primarily horizontal movement: text selection (fallback if no mouseup)
if (totalAbsDx > 0.03 && totalAbsDx > totalAbsDy * 1.8) {
return "text-selection";
}
return "click-like";
}