feat(extensions): integrate extension hooks into editor playback and settings

- Wire render hooks and cursor effects into VideoPlayback canvas pipeline
- Add extension settings panels to SettingsPanel
- Persist extension-related editor preferences and project state
This commit is contained in:
webadderall
2026-04-12 01:37:28 +10:00
parent d3fab9d83e
commit 34ead66e2a
4 changed files with 664 additions and 8 deletions
+238 -1
View File
@@ -14,6 +14,8 @@ import { Switch } from "@/components/ui/switch";
import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath";
import { cn } from "@/lib/utils";
import { extensionHost, type FrameInstance } from "@/lib/extensions";
import type { ExtensionSettingField } from "@/lib/extensions";
import type { BuiltInWallpaper } from "@/lib/wallpapers";
import { BUILT_IN_WALLPAPERS, getAvailableWallpapers, isVideoWallpaperSource } from "@/lib/wallpapers";
import { type AspectRatio } from "@/utils/aspectRatioUtils";
@@ -114,7 +116,9 @@ export type EditorEffectSection =
| "webcam"
| "zoom"
| "frame"
| "crop";
| "crop"
| "extensions"
| `ext:${string}`;
function isHexWallpaper(value: string): boolean {
return /^#(?:[0-9a-f]{3}){1,2}$/i.test(value);
@@ -142,6 +146,129 @@ function SectionLabel({ children }: { children: React.ReactNode }) {
);
}
/**
* Renders extension-contributed settings fields (toggle, slider, select, color, text).
*/
function ExtensionSettingsSection({ extensionId, label, fields }: {
extensionId: string;
label: string;
fields: ExtensionSettingField[];
}) {
const [, forceUpdate] = useState(0);
return (
<div className="flex flex-col gap-1.5 mt-2 pt-2 border-t border-white/[0.06]">
<p className="text-[10px] font-semibold uppercase tracking-widest text-slate-500">{label}</p>
{fields.map((field) => {
const value = extensionHost.getExtensionSetting(extensionId, field.id) ?? field.defaultValue;
if (field.type === 'toggle') {
return (
<div key={field.id} className="flex items-center justify-between rounded-lg bg-white/[0.03] px-2.5 py-1.5">
<span className="text-[11px] text-slate-300">{field.label}</span>
<Switch
checked={Boolean(value)}
onCheckedChange={(checked) => {
extensionHost.setExtensionSetting(extensionId, field.id, checked);
forceUpdate(n => n + 1);
}}
className="data-[state=checked]:bg-[#2563EB] scale-75"
/>
</div>
);
}
if (field.type === 'slider') {
return (
<div key={field.id} className="flex items-center justify-between gap-2 rounded-lg bg-white/[0.03] px-2.5 py-1.5">
<span className="text-[11px] text-slate-300 flex-shrink-0">{field.label}</span>
<div className="flex items-center gap-1.5">
<input
type="range"
min={field.min ?? 0}
max={field.max ?? 1}
step={field.step ?? 0.01}
value={typeof value === 'number' ? value : field.defaultValue as number}
onChange={(e) => {
extensionHost.setExtensionSetting(extensionId, field.id, parseFloat(e.target.value));
forceUpdate(n => n + 1);
}}
className="w-20 h-1 accent-[#2563EB]"
/>
<span className="text-[10px] text-slate-500 w-8 text-right font-mono">
{(typeof value === 'number' ? value : 0).toFixed(1)}
</span>
</div>
</div>
);
}
if (field.type === 'select' && field.options) {
return (
<div key={field.id} className="flex items-center justify-between gap-2 rounded-lg bg-white/[0.03] px-2.5 py-1.5">
<span className="text-[11px] text-slate-300 flex-shrink-0">{field.label}</span>
<Select
value={String(value)}
onValueChange={(v) => {
extensionHost.setExtensionSetting(extensionId, field.id, v);
forceUpdate(n => n + 1);
}}
>
<SelectTrigger className="h-6 w-24 text-[10px] border-white/10 bg-white/[0.03]">
<SelectValue />
</SelectTrigger>
<SelectContent>
{field.options.map(opt => (
<SelectItem key={opt.value} value={opt.value} className="text-[10px]">
{opt.label}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
);
}
if (field.type === 'color') {
return (
<div key={field.id} className="flex items-center justify-between gap-2 rounded-lg bg-white/[0.03] px-2.5 py-1.5">
<span className="text-[11px] text-slate-300 flex-shrink-0">{field.label}</span>
<input
type="color"
value={String(value)}
onChange={(e) => {
extensionHost.setExtensionSetting(extensionId, field.id, e.target.value);
forceUpdate(n => n + 1);
}}
className="w-7 h-5 rounded border border-white/10 cursor-pointer bg-transparent"
/>
</div>
);
}
if (field.type === 'text') {
return (
<div key={field.id} className="flex items-center justify-between gap-2 rounded-lg bg-white/[0.03] px-2.5 py-1.5">
<span className="text-[11px] text-slate-300 flex-shrink-0">{field.label}</span>
<input
type="text"
value={String(value)}
onChange={(e) => {
extensionHost.setExtensionSetting(extensionId, field.id, e.target.value);
forceUpdate(n => n + 1);
}}
className="w-24 h-6 rounded bg-white/[0.06] border border-white/10 px-1.5 text-[10px] text-slate-200"
/>
</div>
);
}
return null;
})}
</div>
);
}
interface SettingsPanelProps {
panelMode?: "editor" | "background";
activeEffectSection?: EditorEffectSection;
@@ -213,6 +340,8 @@ interface SettingsPanelProps {
onClearWebcam?: () => void;
padding?: number;
onPaddingChange?: (padding: number) => void;
frame?: string | null;
onFrameChange?: (frameId: string | null) => void;
cropRegion?: CropRegion;
onCropChange?: (region: CropRegion) => void;
aspectRatio: AspectRatio;
@@ -562,6 +691,8 @@ export function SettingsPanel({
onClearWebcam,
padding = 50,
onPaddingChange,
frame = null,
onFrameChange,
cropRegion,
onCropChange,
aspectRatio,
@@ -670,6 +801,38 @@ export function SettingsPanel({
GRADIENTS.includes(selected) ? selected : GRADIENTS[0],
);
const removeBackgroundEnabled = aspectRatio === "native" && padding === 0;
// Device frames from extension system
const [availableFrames, setAvailableFrames] = useState<FrameInstance[]>([]);
useEffect(() => {
const update = () => setAvailableFrames(extensionHost.getFrames());
update();
return extensionHost.onChange(update);
}, []);
// Extension-contributed settings panels
const [extensionPanels, setExtensionPanels] = useState<ReturnType<typeof extensionHost.getSettingsPanels>>([]);
useEffect(() => {
const update = () => setExtensionPanels(extensionHost.getSettingsPanels());
update();
return extensionHost.onChange(update);
}, []);
const renderExtensionPanelsForSections = (...sections: string[]) =>
extensionPanels
.filter((panel) => {
const parentSection = panel.panel.parentSection;
return parentSection ? sections.includes(parentSection) : false;
})
.map((panel) => (
<ExtensionSettingsSection
key={`${panel.extensionId}/${panel.panel.id}`}
extensionId={panel.extensionId}
label={panel.panel.label}
fields={panel.panel.fields}
/>
));
const [backgroundTab, setBackgroundTab] = useState<BackgroundTab>(() =>
getBackgroundTabForWallpaper(selected),
);
@@ -932,6 +1095,7 @@ export function SettingsPanel({
onShadowChange?.(initialEditorPreferences.shadowIntensity);
onBorderRadiusChange?.(initialEditorPreferences.borderRadius);
onPaddingChange?.(initialEditorPreferences.padding);
onFrameChange?.(null);
onAspectRatioChange?.(initialEditorPreferences.aspectRatio);
removeBackgroundStateRef.current = null;
};
@@ -1443,6 +1607,53 @@ export function SettingsPanel({
className="data-[state=checked]:bg-[#2563EB] scale-75"
/>
</div>
{/* Frame Picker */}
{availableFrames.length > 0 && (
<div className="flex flex-col gap-1.5 mt-1">
<div className="flex items-center justify-between">
<span className="text-[10px] text-slate-400">Frame</span>
{frame && (
<button
type="button"
onClick={() => onFrameChange?.(null)}
className="text-[9px] text-[#2563EB] hover:opacity-80"
>
Remove
</button>
)}
</div>
<div className="grid grid-cols-3 gap-1.5">
{availableFrames.map((f) => {
const isSelected = frame === f.id;
return (
<button
key={f.id}
type="button"
onClick={() => onFrameChange?.(isSelected ? null : f.id)}
className={cn(
"flex flex-col items-center gap-1 p-1.5 rounded-lg border transition-all text-center",
isSelected
? "border-[#2563EB]/50 bg-[#2563EB]/10 ring-1 ring-[#2563EB]/30"
: "border-white/[0.06] bg-white/[0.02] hover:bg-white/[0.05]",
)}
>
<div className="w-full aspect-video rounded bg-black/30 overflow-hidden flex items-center justify-center">
<img
src={f.thumbnailPath}
alt={f.label}
className="w-full h-full object-contain"
draggable={false}
/>
</div>
<span className="text-[8px] text-slate-400 truncate w-full leading-tight">
{f.label}
</span>
</button>
);
})}
</div>
</div>
)}
</div>
</section>
);
@@ -1744,6 +1955,7 @@ export function SettingsPanel({
formatValue={(value) => `${Math.round(value * 100)}%`}
parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100}
/>
{renderExtensionPanelsForSections("captions")}
</div>
</section>
);
@@ -1755,6 +1967,7 @@ export function SettingsPanel({
{zoomSectionContent}
{frameSectionContent}
{cropSectionContent}
{renderExtensionPanelsForSections("scene", "appearance", "zoom", "frame", "crop")}
</div>
);
@@ -1909,6 +2122,7 @@ export function SettingsPanel({
}}
/>
</div>
{renderExtensionPanelsForSections("cursor")}
</section>
);
case "webcam":
@@ -2088,9 +2302,32 @@ export function SettingsPanel({
</div>
</div>
</div>
{renderExtensionPanelsForSections("webcam")}
</div>
</section>
);
default: {
// Handle extension-contributed standalone section pages (ext:extensionId/panelId)
if (activeEffectSection?.startsWith('ext:')) {
const panels = extensionPanels.filter(
p => !p.panel.parentSection && `ext:${p.extensionId}/${p.panel.id}` === activeEffectSection,
);
if (panels.length > 0) {
const p = panels[0];
return (
<section className="flex flex-col gap-2">
<SectionLabel>{p.panel.label}</SectionLabel>
<ExtensionSettingsSection
extensionId={p.extensionId}
label={p.panel.label}
fields={p.panel.fields}
/>
</section>
);
}
}
return sceneSectionContent;
}
}
})();
+419 -7
View File
@@ -92,6 +92,18 @@ import {
formatAspectRatioForCSS,
} from "@/utils/aspectRatioUtils";
import { AnnotationOverlay } from "./AnnotationOverlay";
import { extensionHost } from "@/lib/extensions";
import {
mapCursorToCanvasNormalized,
mapSmoothedCursorToCanvasNormalized,
} from "@/lib/extensions/cursorCoordinates";
import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform";
import {
notifyCursorInteraction,
executeExtensionCursorEffects,
executeExtensionRenderHooks,
clearCursorEffects,
} from "@/lib/extensions/renderHooks";
import {
DEFAULT_CURSOR_CLICK_BOUNCE,
DEFAULT_CURSOR_CLICK_BOUNCE_DURATION,
@@ -138,6 +150,41 @@ function createPlaybackAnimationState(): PlaybackAnimationState {
};
}
function getCursorPositionAtTime(
telemetry: CursorTelemetryPoint[],
timeMs: number,
params?: {
maskRect?: { x: number; y: number; width: number; height: number } | null;
canvasWidth: number;
canvasHeight: number;
},
): { cx: number; cy: number; interactionType?: string } | null {
if (telemetry.length === 0) {
return null;
}
let closest = telemetry[0];
let minDist = Math.abs(telemetry[0].timeMs - timeMs);
for (let index = 1; index < telemetry.length; index++) {
const point = telemetry[index];
const distance = Math.abs(point.timeMs - timeMs);
if (distance < minDist) {
minDist = distance;
closest = point;
}
if (point.timeMs > timeMs) {
break;
}
}
return mapCursorToCanvasNormalized({
cx: closest.cx,
cy: closest.cy,
interactionType: closest.interactionType,
}, params ?? { canvasWidth: 1, canvasHeight: 1 });
}
function getEffectiveNativeAspectRatio(
dimensions: { width: number; height: number } | null | undefined,
cropRegion?: import("./types").CropRegion,
@@ -187,6 +234,7 @@ interface VideoPlaybackProps {
connectedZoomEasing?: ZoomTransitionEasing;
borderRadius?: number;
padding?: number;
frame?: string | null;
cropRegion?: import("./types").CropRegion;
webcam?: WebcamOverlaySettings;
webcamVideoPath?: string | null;
@@ -262,6 +310,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
connectedZoomEasing = DEFAULT_CONNECTED_ZOOM_EASING,
borderRadius = 0,
padding = 50,
frame = null,
cropRegion,
webcam,
webcamVideoPath,
@@ -333,6 +382,9 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
}>({ x: 0, y: 0, width: 0, height: 0 });
const cropBoundsRef = useRef({ startX: 0, endX: 0, startY: 0, endY: 0 });
const maskGraphicsRef = useRef<Graphics | null>(null);
const frameSpriteRef = useRef<Sprite | null>(null);
const frameContainerRef = useRef<Container | null>(null);
const frameIdRef = useRef<string | null>(frame);
const isPlayingRef = useRef(isPlaying);
const isSeekingRef = useRef(false);
const allowPlaybackRef = useRef(false);
@@ -357,6 +409,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const connectedZoomEasingRef = useRef(connectedZoomEasing);
const videoReadyRafRef = useRef<number | null>(null);
const cursorOverlayRef = useRef<PixiCursorOverlay | null>(null);
const cursorEffectsCanvasRef = useRef<HTMLCanvasElement | null>(null);
const cursorTelemetryRef = useRef<CursorTelemetryPoint[]>([]);
const showCursorRef = useRef(showCursor);
const cursorSizeRef = useRef(cursorSize);
@@ -366,6 +419,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
const cursorClickBounceRef = useRef(cursorClickBounce);
const cursorClickBounceDurationRef = useRef(cursorClickBounceDuration);
const cursorSwayRef = useRef(cursorSway);
const lastEmittedClickTimeMsRef = useRef(-1);
// Spring animation state for smooth zoom transitions
const springScaleRef = useRef<SpringState>(createSpringState(1));
@@ -567,6 +621,16 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
};
}
// Look up device frame insets so layout centers the full frame (video + bezels)
let frameInsets: { top: number; right: number; bottom: number; left: number } | null = null;
if (frame) {
const frames = extensionHost.getFrames();
const frameData = frames.find((f) => f.id === frame);
if (frameData?.screenInsets) {
frameInsets = frameData.screenInsets;
}
}
const result = layoutVideoContentUtil({
container,
app,
@@ -577,6 +641,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
lockedVideoDimensions: lockedVideoDimensionsRef.current,
borderRadius,
padding,
frameInsets,
});
if (result) {
@@ -587,6 +652,57 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
baseMaskRef.current = result.maskRect;
cropBoundsRef.current = result.cropBounds;
// Sync extension cursor effects canvas resolution with renderer
const effectsCanvas = cursorEffectsCanvasRef.current;
if (effectsCanvas) {
const w = result.stageSize.width;
const h = result.stageSize.height;
if (effectsCanvas.width !== w || effectsCanvas.height !== h) {
effectsCanvas.width = w;
effectsCanvas.height = h;
}
}
// Push layout info to extension host for query APIs
extensionHost.setVideoLayout({
maskRect: { x: result.maskRect.x, y: result.maskRect.y, width: result.maskRect.width, height: result.maskRect.height },
canvasWidth: result.stageSize.width,
canvasHeight: result.stageSize.height,
borderRadius,
padding,
});
extensionHost.setShadowConfig({
enabled: Boolean(showShadow) && shadowIntensity > 0,
intensity: shadowIntensity,
});
// Position device frame sprite to fill the stage
const frameSprite = frameSpriteRef.current;
if (frameSprite && frame) {
const frames = extensionHost.getFrames();
const frameData = frames.find((f) => f.id === frame);
if (frameData) {
const maskRect = result.maskRect;
const insets = frameData.screenInsets;
if (insets) {
// Frame is larger than screen area — compute full frame size from insets
const screenW = maskRect.width;
const screenH = maskRect.height;
const frameW = screenW / (1 - insets.left - insets.right);
const frameH = screenH / (1 - insets.top - insets.bottom);
const frameX = maskRect.x - insets.left * frameW;
const frameY = maskRect.y - insets.top * frameH;
frameSprite.position.set(frameX, frameY);
frameSprite.width = frameW;
frameSprite.height = frameH;
} else {
frameSprite.position.set(maskRect.x, maskRect.y);
frameSprite.width = maskRect.width;
frameSprite.height = maskRect.height;
}
}
}
// Reset camera container to identity
cameraContainer.scale.set(1);
cameraContainer.position.set(0, 0);
@@ -601,7 +717,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
updateOverlayForRegion(activeRegion);
applyWebcamBubbleLayout(animationStateRef.current.appliedScale || 1);
}
}, [updateOverlayForRegion, cropRegion, borderRadius, padding, applyWebcamBubbleLayout]);
}, [updateOverlayForRegion, cropRegion, borderRadius, padding, frame, showShadow, shadowIntensity, applyWebcamBubbleLayout]);
useEffect(() => {
const video = videoRef.current;
@@ -616,6 +732,81 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
layoutVideoContentRef.current = layoutVideoContent;
}, [layoutVideoContent]);
// Sync device frame ref
useEffect(() => {
frameIdRef.current = frame;
extensionHost.setActiveFrame(frame ?? null);
}, [frame]);
// Manage device frame sprite
useEffect(() => {
const frameContainer = frameContainerRef.current;
if (!frameContainer) return;
// Clear existing frame sprite
if (frameSpriteRef.current) {
frameContainer.removeChild(frameSpriteRef.current);
frameSpriteRef.current.destroy();
frameSpriteRef.current = null;
}
if (!frame) {
layoutVideoContentRef.current?.();
return;
}
let cancelled = false;
function tryLoadFrame() {
if (cancelled) return;
const container = frameContainerRef.current;
if (!container) return false;
const frames = extensionHost.getFrames();
const frameData = frames.find((f) => f.id === frame);
if (!frameData) return false;
if (frameData.draw) {
// Resolution-independent: draw at a reasonable size, Pixi handles the rest
const drawW = 1920;
const drawH = 1080;
const canvas = document.createElement('canvas');
canvas.width = drawW;
canvas.height = drawH;
const ctx = canvas.getContext('2d');
if (ctx) frameData.draw(ctx, drawW, drawH);
if (cancelled || frameIdRef.current !== frame) return true;
const texture = Texture.from(canvas);
const sprite = new Sprite(texture);
frameSpriteRef.current = sprite;
container.addChild(sprite);
layoutVideoContentRef.current?.();
} else {
const img = new Image();
img.onload = () => {
if (cancelled || frameIdRef.current !== frame) return;
const texture = Texture.from(img);
const sprite = new Sprite(texture);
frameSpriteRef.current = sprite;
container.addChild(sprite);
layoutVideoContentRef.current?.();
};
img.src = frameData.filePath;
}
return true;
}
// Try immediately; if extension hasn't registered frames yet,
// listen for changes and retry once they become available.
if (!tryLoadFrame()) {
const unsub = extensionHost.onChange(() => {
if (tryLoadFrame()) unsub();
});
return () => { cancelled = true; unsub(); };
}
return () => { cancelled = true; };
}, [frame]);
const selectedZoom = useMemo(() => {
if (!selectedZoomId) return null;
return zoomRegions.find((region) => region.id === selectedZoomId) ?? null;
@@ -778,6 +969,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
useEffect(() => {
isPlayingRef.current = isPlaying;
extensionHost.emitEvent({ type: isPlaying ? 'playback:play' : 'playback:pause', timeMs: currentTimeRef.current });
// Snap springs to current position when pausing so scrubbing is instant
if (!isPlaying) {
resetSpringState(springScaleRef.current);
@@ -855,6 +1047,16 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
useEffect(() => {
cursorTelemetryRef.current = cursorTelemetry;
// Push to extension host for query APIs
extensionHost.setCursorTelemetry(
cursorTelemetry.map(p => ({
timeMs: p.timeMs,
cx: p.cx,
cy: p.cy,
interactionType: p.interactionType,
pressure: (p as any).pressure,
})),
);
}, [cursorTelemetry]);
useEffect(() => {
@@ -898,8 +1100,15 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
}, [cursorSway]);
useEffect(() => {
currentTimeRef.current = currentTime * 1000;
}, [currentTime]);
const timeMs = currentTime * 1000;
currentTimeRef.current = timeMs;
const videoInfo = extensionHost.getVideoInfoSnapshot();
extensionHost.setPlaybackState({
currentTimeMs: timeMs,
durationMs: videoInfo?.durationMs ?? 0,
isPlaying,
});
}, [currentTime, isPlaying]);
useEffect(() => {
if (!pixiReady || !videoReady) return;
@@ -1115,6 +1324,11 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
videoContainerRef.current = videoContainer;
cameraContainer.addChild(videoContainer);
// Device frame overlay container — sits above video but below cursor
const frameContainer = new Container();
frameContainerRef.current = frameContainer;
cameraContainer.addChild(frameContainer);
const cursorContainer = new Container();
cursorContainerRef.current = cursorContainer;
cameraContainer.addChild(cursorContainer);
@@ -1164,6 +1378,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
appRef.current = null;
cameraContainerRef.current = null;
videoContainerRef.current = null;
frameContainerRef.current = null;
frameSpriteRef.current = null;
cursorContainerRef.current = null;
videoSpriteRef.current = null;
};
@@ -1423,6 +1639,14 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
state.focusY = targetFocus.cy;
state.progress = targetProgress;
// Push zoom state to extension host for query APIs
extensionHost.setZoomState({
scale: targetScaleFactor,
focusX: targetFocus.cx,
focusY: targetFocus.cy,
progress: targetProgress,
});
const projectedTransform = computeZoomTransform({
stageSize: stageSizeRef.current,
baseMask: baseMaskRef.current,
@@ -1479,17 +1703,182 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
);
applyWebcamBubbleLayout(animationStateRef.current.appliedScale || 1);
// Update cursor overlay
const timeMs = currentTimeRef.current;
const effectsCanvas = cursorEffectsCanvasRef.current;
const extensionCanvasWidth = effectsCanvas?.width || stageSizeRef.current.width;
const extensionCanvasHeight = effectsCanvas?.height || stageSizeRef.current.height;
let smoothedCursorForHooks: {
cx: number;
cy: number;
trail: Array<{ cx: number; cy: number }>;
} | null = null;
// Update cursor overlay + emit cursor events
const cursorOverlay = cursorOverlayRef.current;
if (cursorOverlay) {
const timeMs = currentTimeRef.current;
const telemetry = cursorTelemetryRef.current;
cursorOverlay.update(
cursorTelemetryRef.current,
telemetry,
timeMs,
baseMaskRef.current,
showCursorRef.current,
!isPlayingRef.current || isSeekingRef.current,
);
smoothedCursorForHooks = mapSmoothedCursorToCanvasNormalized(
cursorOverlay.getSmoothedCursorSnapshot(),
{
maskRect: baseMaskRef.current,
canvasWidth: extensionCanvasWidth,
canvasHeight: extensionCanvasHeight,
},
);
extensionHost.setSmoothedCursor(
smoothedCursorForHooks
? {
timeMs,
cx: smoothedCursorForHooks.cx,
cy: smoothedCursorForHooks.cy,
trail: smoothedCursorForHooks.trail,
}
: null,
);
// Emit cursor:click events for extensions
if (isPlayingRef.current && telemetry.length > 0) {
for (let i = telemetry.length - 1; i >= 0; i--) {
const p = telemetry[i];
if (p.timeMs > timeMs) continue;
if (p.timeMs < timeMs - 100) break;
if (
p.interactionType &&
p.interactionType !== 'move' &&
p.timeMs !== lastEmittedClickTimeMsRef.current
) {
const extensionCursor = mapCursorToCanvasNormalized(
{
cx: p.cx,
cy: p.cy,
interactionType: p.interactionType,
},
{
maskRect: baseMaskRef.current,
canvasWidth: extensionCanvasWidth,
canvasHeight: extensionCanvasHeight,
},
);
lastEmittedClickTimeMsRef.current = p.timeMs;
extensionHost.emitEvent({
type: 'cursor:click',
timeMs: p.timeMs,
data: extensionCursor,
});
if (extensionCursor) {
notifyCursorInteraction(
p.timeMs,
extensionCursor.cx,
extensionCursor.cy,
p.interactionType,
);
}
}
break;
}
}
}
if (!cursorOverlay) {
extensionHost.setSmoothedCursor(null);
}
if (effectsCanvas && effectsCanvas.width > 0 && effectsCanvas.height > 0) {
const ctx2d = effectsCanvas.getContext("2d");
if (ctx2d) {
ctx2d.clearRect(0, 0, effectsCanvas.width, effectsCanvas.height);
const maskRect = baseMaskRef.current;
const animationState = animationStateRef.current;
const videoInfo = extensionHost.getVideoInfoSnapshot();
const rawCursor = getCursorPositionAtTime(cursorTelemetryRef.current, timeMs, {
maskRect,
canvasWidth: effectsCanvas.width,
canvasHeight: effectsCanvas.height,
});
const hookParams = {
width: effectsCanvas.width,
height: effectsCanvas.height,
timeMs,
durationMs: videoInfo?.durationMs ?? 0,
cursor: smoothedCursorForHooks
? {
cx: smoothedCursorForHooks.cx,
cy: smoothedCursorForHooks.cy,
interactionType: rawCursor?.interactionType,
}
: rawCursor,
smoothedCursor: smoothedCursorForHooks,
videoLayout:
maskRect.width > 0 && maskRect.height > 0
? {
maskRect: {
x: maskRect.x,
y: maskRect.y,
width: maskRect.width,
height: maskRect.height,
},
borderRadius,
padding,
}
: undefined,
zoom: {
scale: animationState.scale,
focusX: animationState.focusX,
focusY: animationState.focusY,
progress: animationState.progress,
},
shadow: {
enabled: Boolean(showShadow) && shadowIntensity > 0,
intensity: shadowIntensity,
},
sceneTransform: {
scale: animationState.appliedScale,
x: animationState.x,
y: animationState.y,
},
};
ctx2d.save();
applyCanvasSceneTransform(ctx2d, {
scale: animationState.appliedScale,
x: animationState.x,
y: animationState.y,
});
executeExtensionRenderHooks("post-video", ctx2d, hookParams);
executeExtensionRenderHooks("post-zoom", ctx2d, hookParams);
executeExtensionRenderHooks("post-cursor", ctx2d, hookParams);
if (isSeekingRef.current) {
clearCursorEffects();
} else {
executeExtensionCursorEffects(
ctx2d,
timeMs,
effectsCanvas.width,
effectsCanvas.height,
{
zoom: hookParams.zoom,
sceneTransform: hookParams.sceneTransform,
videoLayout: hookParams.videoLayout,
},
);
}
ctx2d.restore();
executeExtensionRenderHooks("post-webcam", ctx2d, hookParams);
executeExtensionRenderHooks("post-annotations", ctx2d, hookParams);
executeExtensionRenderHooks("final", ctx2d, hookParams);
}
}
};
@@ -1499,7 +1888,16 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
app.ticker.remove(ticker);
}
};
}, [pixiReady, videoReady, clampFocusToStage, applyWebcamBubbleLayout]);
}, [
pixiReady,
videoReady,
clampFocusToStage,
applyWebcamBubbleLayout,
borderRadius,
padding,
showShadow,
shadowIntensity,
]);
useEffect(() => {
const overlay = cursorOverlayRef.current;
@@ -1530,6 +1928,14 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
) => {
const video = e.currentTarget;
onDurationChange(video.duration);
// Push video info to extension host for query APIs
extensionHost.setVideoInfo({
width: video.videoWidth,
height: video.videoHeight,
durationMs: Number.isFinite(video.duration) ? video.duration * 1000 : 0,
fps: 60, // Not available from HTMLVideoElement; default to 60
});
const targetTime = clampMediaTimeToDuration(
currentTime,
Number.isFinite(video.duration) ? video.duration : null,
@@ -1729,6 +2135,12 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
: "none",
}}
/>
{/* Canvas overlay for extension cursor effects (drawn via Canvas 2D API) */}
<canvas
ref={cursorEffectsCanvasRef}
className="absolute inset-0 w-full h-full pointer-events-none"
style={{ zIndex: 1 }}
/>
{/* Only render overlay after PIXI and video are fully initialized */}
{pixiReady && videoReady && (
<div
@@ -32,6 +32,7 @@ type PersistedEditorControls = Pick<
| "cursorSway"
| "borderRadius"
| "padding"
| "frame"
| "webcam"
| "aspectRatio"
| "exportEncodingMode"
@@ -84,6 +85,7 @@ export const DEFAULT_EDITOR_PREFERENCES: EditorPreferences = {
cursorSway: DEFAULT_EDITOR_CONTROLS.cursorSway,
borderRadius: DEFAULT_EDITOR_CONTROLS.borderRadius,
padding: DEFAULT_EDITOR_CONTROLS.padding,
frame: DEFAULT_EDITOR_CONTROLS.frame,
webcam: DEFAULT_EDITOR_CONTROLS.webcam,
aspectRatio: DEFAULT_EDITOR_CONTROLS.aspectRatio,
exportEncodingMode: DEFAULT_EDITOR_CONTROLS.exportEncodingMode,
@@ -166,6 +168,7 @@ function normalizeEditorControls(
cursorSway: raw.cursorSway ?? fallback.cursorSway,
borderRadius: raw.borderRadius ?? fallback.borderRadius,
padding: raw.padding ?? fallback.padding,
frame: raw.frame !== undefined ? raw.frame : fallback.frame,
webcam: raw.webcam ?? fallback.webcam,
aspectRatio: raw.aspectRatio ?? fallback.aspectRatio,
exportEncodingMode: raw.exportEncodingMode ?? fallback.exportEncodingMode,
@@ -215,6 +218,7 @@ function normalizeEditorControls(
cursorSway: normalized.cursorSway,
borderRadius: normalized.borderRadius,
padding: normalized.padding,
frame: normalized.frame,
webcam: normalized.webcam,
aspectRatio: normalized.aspectRatio,
exportEncodingMode: normalized.exportEncodingMode,
@@ -92,6 +92,8 @@ export interface ProjectEditorState {
cursorSway: number;
borderRadius: number;
padding: number;
/** Selected frame ID (e.g. "recordly.frames/browser-dark"), or null for none */
frame: string | null;
cropRegion: CropRegion;
zoomRegions: ZoomRegion[];
trimRegions: TrimRegion[];
@@ -655,6 +657,7 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
: DEFAULT_CURSOR_SWAY,
borderRadius: typeof editor.borderRadius === "number" ? editor.borderRadius : 12.5,
padding: isFiniteNumber(editor.padding) ? clamp(editor.padding, 0, 100) : 20,
frame: typeof editor.frame === "string" ? editor.frame : null,
cropRegion: {
x: cropX,
y: cropY,