mirror of
https://github.com/webadderallorg/Recordly.git
synced 2026-09-25 15:25:44 +00:00
feat(extensions): integrate extension hooks into editor playback and settings
- Wire render hooks and cursor effects into VideoPlayback canvas pipeline - Add extension settings panels to SettingsPanel - Persist extension-related editor preferences and project state
This commit is contained in:
@@ -14,6 +14,8 @@ import { Switch } from "@/components/ui/switch";
|
||||
import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
|
||||
import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { extensionHost, type FrameInstance } from "@/lib/extensions";
|
||||
import type { ExtensionSettingField } from "@/lib/extensions";
|
||||
import type { BuiltInWallpaper } from "@/lib/wallpapers";
|
||||
import { BUILT_IN_WALLPAPERS, getAvailableWallpapers, isVideoWallpaperSource } from "@/lib/wallpapers";
|
||||
import { type AspectRatio } from "@/utils/aspectRatioUtils";
|
||||
@@ -114,7 +116,9 @@ export type EditorEffectSection =
|
||||
| "webcam"
|
||||
| "zoom"
|
||||
| "frame"
|
||||
| "crop";
|
||||
| "crop"
|
||||
| "extensions"
|
||||
| `ext:${string}`;
|
||||
|
||||
function isHexWallpaper(value: string): boolean {
|
||||
return /^#(?:[0-9a-f]{3}){1,2}$/i.test(value);
|
||||
@@ -142,6 +146,129 @@ function SectionLabel({ children }: { children: React.ReactNode }) {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders extension-contributed settings fields (toggle, slider, select, color, text).
|
||||
*/
|
||||
function ExtensionSettingsSection({ extensionId, label, fields }: {
|
||||
extensionId: string;
|
||||
label: string;
|
||||
fields: ExtensionSettingField[];
|
||||
}) {
|
||||
const [, forceUpdate] = useState(0);
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-1.5 mt-2 pt-2 border-t border-white/[0.06]">
|
||||
<p className="text-[10px] font-semibold uppercase tracking-widest text-slate-500">{label}</p>
|
||||
{fields.map((field) => {
|
||||
const value = extensionHost.getExtensionSetting(extensionId, field.id) ?? field.defaultValue;
|
||||
|
||||
if (field.type === 'toggle') {
|
||||
return (
|
||||
<div key={field.id} className="flex items-center justify-between rounded-lg bg-white/[0.03] px-2.5 py-1.5">
|
||||
<span className="text-[11px] text-slate-300">{field.label}</span>
|
||||
<Switch
|
||||
checked={Boolean(value)}
|
||||
onCheckedChange={(checked) => {
|
||||
extensionHost.setExtensionSetting(extensionId, field.id, checked);
|
||||
forceUpdate(n => n + 1);
|
||||
}}
|
||||
className="data-[state=checked]:bg-[#2563EB] scale-75"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (field.type === 'slider') {
|
||||
return (
|
||||
<div key={field.id} className="flex items-center justify-between gap-2 rounded-lg bg-white/[0.03] px-2.5 py-1.5">
|
||||
<span className="text-[11px] text-slate-300 flex-shrink-0">{field.label}</span>
|
||||
<div className="flex items-center gap-1.5">
|
||||
<input
|
||||
type="range"
|
||||
min={field.min ?? 0}
|
||||
max={field.max ?? 1}
|
||||
step={field.step ?? 0.01}
|
||||
value={typeof value === 'number' ? value : field.defaultValue as number}
|
||||
onChange={(e) => {
|
||||
extensionHost.setExtensionSetting(extensionId, field.id, parseFloat(e.target.value));
|
||||
forceUpdate(n => n + 1);
|
||||
}}
|
||||
className="w-20 h-1 accent-[#2563EB]"
|
||||
/>
|
||||
<span className="text-[10px] text-slate-500 w-8 text-right font-mono">
|
||||
{(typeof value === 'number' ? value : 0).toFixed(1)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (field.type === 'select' && field.options) {
|
||||
return (
|
||||
<div key={field.id} className="flex items-center justify-between gap-2 rounded-lg bg-white/[0.03] px-2.5 py-1.5">
|
||||
<span className="text-[11px] text-slate-300 flex-shrink-0">{field.label}</span>
|
||||
<Select
|
||||
value={String(value)}
|
||||
onValueChange={(v) => {
|
||||
extensionHost.setExtensionSetting(extensionId, field.id, v);
|
||||
forceUpdate(n => n + 1);
|
||||
}}
|
||||
>
|
||||
<SelectTrigger className="h-6 w-24 text-[10px] border-white/10 bg-white/[0.03]">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{field.options.map(opt => (
|
||||
<SelectItem key={opt.value} value={opt.value} className="text-[10px]">
|
||||
{opt.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (field.type === 'color') {
|
||||
return (
|
||||
<div key={field.id} className="flex items-center justify-between gap-2 rounded-lg bg-white/[0.03] px-2.5 py-1.5">
|
||||
<span className="text-[11px] text-slate-300 flex-shrink-0">{field.label}</span>
|
||||
<input
|
||||
type="color"
|
||||
value={String(value)}
|
||||
onChange={(e) => {
|
||||
extensionHost.setExtensionSetting(extensionId, field.id, e.target.value);
|
||||
forceUpdate(n => n + 1);
|
||||
}}
|
||||
className="w-7 h-5 rounded border border-white/10 cursor-pointer bg-transparent"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (field.type === 'text') {
|
||||
return (
|
||||
<div key={field.id} className="flex items-center justify-between gap-2 rounded-lg bg-white/[0.03] px-2.5 py-1.5">
|
||||
<span className="text-[11px] text-slate-300 flex-shrink-0">{field.label}</span>
|
||||
<input
|
||||
type="text"
|
||||
value={String(value)}
|
||||
onChange={(e) => {
|
||||
extensionHost.setExtensionSetting(extensionId, field.id, e.target.value);
|
||||
forceUpdate(n => n + 1);
|
||||
}}
|
||||
className="w-24 h-6 rounded bg-white/[0.06] border border-white/10 px-1.5 text-[10px] text-slate-200"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
interface SettingsPanelProps {
|
||||
panelMode?: "editor" | "background";
|
||||
activeEffectSection?: EditorEffectSection;
|
||||
@@ -213,6 +340,8 @@ interface SettingsPanelProps {
|
||||
onClearWebcam?: () => void;
|
||||
padding?: number;
|
||||
onPaddingChange?: (padding: number) => void;
|
||||
frame?: string | null;
|
||||
onFrameChange?: (frameId: string | null) => void;
|
||||
cropRegion?: CropRegion;
|
||||
onCropChange?: (region: CropRegion) => void;
|
||||
aspectRatio: AspectRatio;
|
||||
@@ -562,6 +691,8 @@ export function SettingsPanel({
|
||||
onClearWebcam,
|
||||
padding = 50,
|
||||
onPaddingChange,
|
||||
frame = null,
|
||||
onFrameChange,
|
||||
cropRegion,
|
||||
onCropChange,
|
||||
aspectRatio,
|
||||
@@ -670,6 +801,38 @@ export function SettingsPanel({
|
||||
GRADIENTS.includes(selected) ? selected : GRADIENTS[0],
|
||||
);
|
||||
const removeBackgroundEnabled = aspectRatio === "native" && padding === 0;
|
||||
|
||||
// Device frames from extension system
|
||||
const [availableFrames, setAvailableFrames] = useState<FrameInstance[]>([]);
|
||||
useEffect(() => {
|
||||
const update = () => setAvailableFrames(extensionHost.getFrames());
|
||||
update();
|
||||
return extensionHost.onChange(update);
|
||||
}, []);
|
||||
|
||||
// Extension-contributed settings panels
|
||||
const [extensionPanels, setExtensionPanels] = useState<ReturnType<typeof extensionHost.getSettingsPanels>>([]);
|
||||
useEffect(() => {
|
||||
const update = () => setExtensionPanels(extensionHost.getSettingsPanels());
|
||||
update();
|
||||
return extensionHost.onChange(update);
|
||||
}, []);
|
||||
|
||||
const renderExtensionPanelsForSections = (...sections: string[]) =>
|
||||
extensionPanels
|
||||
.filter((panel) => {
|
||||
const parentSection = panel.panel.parentSection;
|
||||
return parentSection ? sections.includes(parentSection) : false;
|
||||
})
|
||||
.map((panel) => (
|
||||
<ExtensionSettingsSection
|
||||
key={`${panel.extensionId}/${panel.panel.id}`}
|
||||
extensionId={panel.extensionId}
|
||||
label={panel.panel.label}
|
||||
fields={panel.panel.fields}
|
||||
/>
|
||||
));
|
||||
|
||||
const [backgroundTab, setBackgroundTab] = useState<BackgroundTab>(() =>
|
||||
getBackgroundTabForWallpaper(selected),
|
||||
);
|
||||
@@ -932,6 +1095,7 @@ export function SettingsPanel({
|
||||
onShadowChange?.(initialEditorPreferences.shadowIntensity);
|
||||
onBorderRadiusChange?.(initialEditorPreferences.borderRadius);
|
||||
onPaddingChange?.(initialEditorPreferences.padding);
|
||||
onFrameChange?.(null);
|
||||
onAspectRatioChange?.(initialEditorPreferences.aspectRatio);
|
||||
removeBackgroundStateRef.current = null;
|
||||
};
|
||||
@@ -1443,6 +1607,53 @@ export function SettingsPanel({
|
||||
className="data-[state=checked]:bg-[#2563EB] scale-75"
|
||||
/>
|
||||
</div>
|
||||
{/* Frame Picker */}
|
||||
{availableFrames.length > 0 && (
|
||||
<div className="flex flex-col gap-1.5 mt-1">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-[10px] text-slate-400">Frame</span>
|
||||
{frame && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onFrameChange?.(null)}
|
||||
className="text-[9px] text-[#2563EB] hover:opacity-80"
|
||||
>
|
||||
Remove
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<div className="grid grid-cols-3 gap-1.5">
|
||||
{availableFrames.map((f) => {
|
||||
const isSelected = frame === f.id;
|
||||
return (
|
||||
<button
|
||||
key={f.id}
|
||||
type="button"
|
||||
onClick={() => onFrameChange?.(isSelected ? null : f.id)}
|
||||
className={cn(
|
||||
"flex flex-col items-center gap-1 p-1.5 rounded-lg border transition-all text-center",
|
||||
isSelected
|
||||
? "border-[#2563EB]/50 bg-[#2563EB]/10 ring-1 ring-[#2563EB]/30"
|
||||
: "border-white/[0.06] bg-white/[0.02] hover:bg-white/[0.05]",
|
||||
)}
|
||||
>
|
||||
<div className="w-full aspect-video rounded bg-black/30 overflow-hidden flex items-center justify-center">
|
||||
<img
|
||||
src={f.thumbnailPath}
|
||||
alt={f.label}
|
||||
className="w-full h-full object-contain"
|
||||
draggable={false}
|
||||
/>
|
||||
</div>
|
||||
<span className="text-[8px] text-slate-400 truncate w-full leading-tight">
|
||||
{f.label}
|
||||
</span>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
@@ -1744,6 +1955,7 @@ export function SettingsPanel({
|
||||
formatValue={(value) => `${Math.round(value * 100)}%`}
|
||||
parseInput={(text) => parseFloat(text.replace(/%$/, "")) / 100}
|
||||
/>
|
||||
{renderExtensionPanelsForSections("captions")}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
@@ -1755,6 +1967,7 @@ export function SettingsPanel({
|
||||
{zoomSectionContent}
|
||||
{frameSectionContent}
|
||||
{cropSectionContent}
|
||||
{renderExtensionPanelsForSections("scene", "appearance", "zoom", "frame", "crop")}
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -1909,6 +2122,7 @@ export function SettingsPanel({
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{renderExtensionPanelsForSections("cursor")}
|
||||
</section>
|
||||
);
|
||||
case "webcam":
|
||||
@@ -2088,9 +2302,32 @@ export function SettingsPanel({
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{renderExtensionPanelsForSections("webcam")}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
default: {
|
||||
// Handle extension-contributed standalone section pages (ext:extensionId/panelId)
|
||||
if (activeEffectSection?.startsWith('ext:')) {
|
||||
const panels = extensionPanels.filter(
|
||||
p => !p.panel.parentSection && `ext:${p.extensionId}/${p.panel.id}` === activeEffectSection,
|
||||
);
|
||||
if (panels.length > 0) {
|
||||
const p = panels[0];
|
||||
return (
|
||||
<section className="flex flex-col gap-2">
|
||||
<SectionLabel>{p.panel.label}</SectionLabel>
|
||||
<ExtensionSettingsSection
|
||||
extensionId={p.extensionId}
|
||||
label={p.panel.label}
|
||||
fields={p.panel.fields}
|
||||
/>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
}
|
||||
return sceneSectionContent;
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
|
||||
@@ -92,6 +92,18 @@ import {
|
||||
formatAspectRatioForCSS,
|
||||
} from "@/utils/aspectRatioUtils";
|
||||
import { AnnotationOverlay } from "./AnnotationOverlay";
|
||||
import { extensionHost } from "@/lib/extensions";
|
||||
import {
|
||||
mapCursorToCanvasNormalized,
|
||||
mapSmoothedCursorToCanvasNormalized,
|
||||
} from "@/lib/extensions/cursorCoordinates";
|
||||
import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform";
|
||||
import {
|
||||
notifyCursorInteraction,
|
||||
executeExtensionCursorEffects,
|
||||
executeExtensionRenderHooks,
|
||||
clearCursorEffects,
|
||||
} from "@/lib/extensions/renderHooks";
|
||||
import {
|
||||
DEFAULT_CURSOR_CLICK_BOUNCE,
|
||||
DEFAULT_CURSOR_CLICK_BOUNCE_DURATION,
|
||||
@@ -138,6 +150,41 @@ function createPlaybackAnimationState(): PlaybackAnimationState {
|
||||
};
|
||||
}
|
||||
|
||||
function getCursorPositionAtTime(
|
||||
telemetry: CursorTelemetryPoint[],
|
||||
timeMs: number,
|
||||
params?: {
|
||||
maskRect?: { x: number; y: number; width: number; height: number } | null;
|
||||
canvasWidth: number;
|
||||
canvasHeight: number;
|
||||
},
|
||||
): { cx: number; cy: number; interactionType?: string } | null {
|
||||
if (telemetry.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let closest = telemetry[0];
|
||||
let minDist = Math.abs(telemetry[0].timeMs - timeMs);
|
||||
|
||||
for (let index = 1; index < telemetry.length; index++) {
|
||||
const point = telemetry[index];
|
||||
const distance = Math.abs(point.timeMs - timeMs);
|
||||
if (distance < minDist) {
|
||||
minDist = distance;
|
||||
closest = point;
|
||||
}
|
||||
if (point.timeMs > timeMs) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return mapCursorToCanvasNormalized({
|
||||
cx: closest.cx,
|
||||
cy: closest.cy,
|
||||
interactionType: closest.interactionType,
|
||||
}, params ?? { canvasWidth: 1, canvasHeight: 1 });
|
||||
}
|
||||
|
||||
function getEffectiveNativeAspectRatio(
|
||||
dimensions: { width: number; height: number } | null | undefined,
|
||||
cropRegion?: import("./types").CropRegion,
|
||||
@@ -187,6 +234,7 @@ interface VideoPlaybackProps {
|
||||
connectedZoomEasing?: ZoomTransitionEasing;
|
||||
borderRadius?: number;
|
||||
padding?: number;
|
||||
frame?: string | null;
|
||||
cropRegion?: import("./types").CropRegion;
|
||||
webcam?: WebcamOverlaySettings;
|
||||
webcamVideoPath?: string | null;
|
||||
@@ -262,6 +310,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
connectedZoomEasing = DEFAULT_CONNECTED_ZOOM_EASING,
|
||||
borderRadius = 0,
|
||||
padding = 50,
|
||||
frame = null,
|
||||
cropRegion,
|
||||
webcam,
|
||||
webcamVideoPath,
|
||||
@@ -333,6 +382,9 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
}>({ x: 0, y: 0, width: 0, height: 0 });
|
||||
const cropBoundsRef = useRef({ startX: 0, endX: 0, startY: 0, endY: 0 });
|
||||
const maskGraphicsRef = useRef<Graphics | null>(null);
|
||||
const frameSpriteRef = useRef<Sprite | null>(null);
|
||||
const frameContainerRef = useRef<Container | null>(null);
|
||||
const frameIdRef = useRef<string | null>(frame);
|
||||
const isPlayingRef = useRef(isPlaying);
|
||||
const isSeekingRef = useRef(false);
|
||||
const allowPlaybackRef = useRef(false);
|
||||
@@ -357,6 +409,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
const connectedZoomEasingRef = useRef(connectedZoomEasing);
|
||||
const videoReadyRafRef = useRef<number | null>(null);
|
||||
const cursorOverlayRef = useRef<PixiCursorOverlay | null>(null);
|
||||
const cursorEffectsCanvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const cursorTelemetryRef = useRef<CursorTelemetryPoint[]>([]);
|
||||
const showCursorRef = useRef(showCursor);
|
||||
const cursorSizeRef = useRef(cursorSize);
|
||||
@@ -366,6 +419,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
const cursorClickBounceRef = useRef(cursorClickBounce);
|
||||
const cursorClickBounceDurationRef = useRef(cursorClickBounceDuration);
|
||||
const cursorSwayRef = useRef(cursorSway);
|
||||
const lastEmittedClickTimeMsRef = useRef(-1);
|
||||
|
||||
// Spring animation state for smooth zoom transitions
|
||||
const springScaleRef = useRef<SpringState>(createSpringState(1));
|
||||
@@ -567,6 +621,16 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
};
|
||||
}
|
||||
|
||||
// Look up device frame insets so layout centers the full frame (video + bezels)
|
||||
let frameInsets: { top: number; right: number; bottom: number; left: number } | null = null;
|
||||
if (frame) {
|
||||
const frames = extensionHost.getFrames();
|
||||
const frameData = frames.find((f) => f.id === frame);
|
||||
if (frameData?.screenInsets) {
|
||||
frameInsets = frameData.screenInsets;
|
||||
}
|
||||
}
|
||||
|
||||
const result = layoutVideoContentUtil({
|
||||
container,
|
||||
app,
|
||||
@@ -577,6 +641,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
lockedVideoDimensions: lockedVideoDimensionsRef.current,
|
||||
borderRadius,
|
||||
padding,
|
||||
frameInsets,
|
||||
});
|
||||
|
||||
if (result) {
|
||||
@@ -587,6 +652,57 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
baseMaskRef.current = result.maskRect;
|
||||
cropBoundsRef.current = result.cropBounds;
|
||||
|
||||
// Sync extension cursor effects canvas resolution with renderer
|
||||
const effectsCanvas = cursorEffectsCanvasRef.current;
|
||||
if (effectsCanvas) {
|
||||
const w = result.stageSize.width;
|
||||
const h = result.stageSize.height;
|
||||
if (effectsCanvas.width !== w || effectsCanvas.height !== h) {
|
||||
effectsCanvas.width = w;
|
||||
effectsCanvas.height = h;
|
||||
}
|
||||
}
|
||||
|
||||
// Push layout info to extension host for query APIs
|
||||
extensionHost.setVideoLayout({
|
||||
maskRect: { x: result.maskRect.x, y: result.maskRect.y, width: result.maskRect.width, height: result.maskRect.height },
|
||||
canvasWidth: result.stageSize.width,
|
||||
canvasHeight: result.stageSize.height,
|
||||
borderRadius,
|
||||
padding,
|
||||
});
|
||||
extensionHost.setShadowConfig({
|
||||
enabled: Boolean(showShadow) && shadowIntensity > 0,
|
||||
intensity: shadowIntensity,
|
||||
});
|
||||
|
||||
// Position device frame sprite to fill the stage
|
||||
const frameSprite = frameSpriteRef.current;
|
||||
if (frameSprite && frame) {
|
||||
const frames = extensionHost.getFrames();
|
||||
const frameData = frames.find((f) => f.id === frame);
|
||||
if (frameData) {
|
||||
const maskRect = result.maskRect;
|
||||
const insets = frameData.screenInsets;
|
||||
if (insets) {
|
||||
// Frame is larger than screen area — compute full frame size from insets
|
||||
const screenW = maskRect.width;
|
||||
const screenH = maskRect.height;
|
||||
const frameW = screenW / (1 - insets.left - insets.right);
|
||||
const frameH = screenH / (1 - insets.top - insets.bottom);
|
||||
const frameX = maskRect.x - insets.left * frameW;
|
||||
const frameY = maskRect.y - insets.top * frameH;
|
||||
frameSprite.position.set(frameX, frameY);
|
||||
frameSprite.width = frameW;
|
||||
frameSprite.height = frameH;
|
||||
} else {
|
||||
frameSprite.position.set(maskRect.x, maskRect.y);
|
||||
frameSprite.width = maskRect.width;
|
||||
frameSprite.height = maskRect.height;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reset camera container to identity
|
||||
cameraContainer.scale.set(1);
|
||||
cameraContainer.position.set(0, 0);
|
||||
@@ -601,7 +717,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
updateOverlayForRegion(activeRegion);
|
||||
applyWebcamBubbleLayout(animationStateRef.current.appliedScale || 1);
|
||||
}
|
||||
}, [updateOverlayForRegion, cropRegion, borderRadius, padding, applyWebcamBubbleLayout]);
|
||||
}, [updateOverlayForRegion, cropRegion, borderRadius, padding, frame, showShadow, shadowIntensity, applyWebcamBubbleLayout]);
|
||||
|
||||
useEffect(() => {
|
||||
const video = videoRef.current;
|
||||
@@ -616,6 +732,81 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
layoutVideoContentRef.current = layoutVideoContent;
|
||||
}, [layoutVideoContent]);
|
||||
|
||||
// Sync device frame ref
|
||||
useEffect(() => {
|
||||
frameIdRef.current = frame;
|
||||
extensionHost.setActiveFrame(frame ?? null);
|
||||
}, [frame]);
|
||||
|
||||
// Manage device frame sprite
|
||||
useEffect(() => {
|
||||
const frameContainer = frameContainerRef.current;
|
||||
if (!frameContainer) return;
|
||||
|
||||
// Clear existing frame sprite
|
||||
if (frameSpriteRef.current) {
|
||||
frameContainer.removeChild(frameSpriteRef.current);
|
||||
frameSpriteRef.current.destroy();
|
||||
frameSpriteRef.current = null;
|
||||
}
|
||||
|
||||
if (!frame) {
|
||||
layoutVideoContentRef.current?.();
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
function tryLoadFrame() {
|
||||
if (cancelled) return;
|
||||
const container = frameContainerRef.current;
|
||||
if (!container) return false;
|
||||
const frames = extensionHost.getFrames();
|
||||
const frameData = frames.find((f) => f.id === frame);
|
||||
if (!frameData) return false;
|
||||
|
||||
if (frameData.draw) {
|
||||
// Resolution-independent: draw at a reasonable size, Pixi handles the rest
|
||||
const drawW = 1920;
|
||||
const drawH = 1080;
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = drawW;
|
||||
canvas.height = drawH;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (ctx) frameData.draw(ctx, drawW, drawH);
|
||||
if (cancelled || frameIdRef.current !== frame) return true;
|
||||
const texture = Texture.from(canvas);
|
||||
const sprite = new Sprite(texture);
|
||||
frameSpriteRef.current = sprite;
|
||||
container.addChild(sprite);
|
||||
layoutVideoContentRef.current?.();
|
||||
} else {
|
||||
const img = new Image();
|
||||
img.onload = () => {
|
||||
if (cancelled || frameIdRef.current !== frame) return;
|
||||
const texture = Texture.from(img);
|
||||
const sprite = new Sprite(texture);
|
||||
frameSpriteRef.current = sprite;
|
||||
container.addChild(sprite);
|
||||
layoutVideoContentRef.current?.();
|
||||
};
|
||||
img.src = frameData.filePath;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Try immediately; if extension hasn't registered frames yet,
|
||||
// listen for changes and retry once they become available.
|
||||
if (!tryLoadFrame()) {
|
||||
const unsub = extensionHost.onChange(() => {
|
||||
if (tryLoadFrame()) unsub();
|
||||
});
|
||||
return () => { cancelled = true; unsub(); };
|
||||
}
|
||||
|
||||
return () => { cancelled = true; };
|
||||
}, [frame]);
|
||||
|
||||
const selectedZoom = useMemo(() => {
|
||||
if (!selectedZoomId) return null;
|
||||
return zoomRegions.find((region) => region.id === selectedZoomId) ?? null;
|
||||
@@ -778,6 +969,7 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
|
||||
useEffect(() => {
|
||||
isPlayingRef.current = isPlaying;
|
||||
extensionHost.emitEvent({ type: isPlaying ? 'playback:play' : 'playback:pause', timeMs: currentTimeRef.current });
|
||||
// Snap springs to current position when pausing so scrubbing is instant
|
||||
if (!isPlaying) {
|
||||
resetSpringState(springScaleRef.current);
|
||||
@@ -855,6 +1047,16 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
|
||||
useEffect(() => {
|
||||
cursorTelemetryRef.current = cursorTelemetry;
|
||||
// Push to extension host for query APIs
|
||||
extensionHost.setCursorTelemetry(
|
||||
cursorTelemetry.map(p => ({
|
||||
timeMs: p.timeMs,
|
||||
cx: p.cx,
|
||||
cy: p.cy,
|
||||
interactionType: p.interactionType,
|
||||
pressure: (p as any).pressure,
|
||||
})),
|
||||
);
|
||||
}, [cursorTelemetry]);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -898,8 +1100,15 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
}, [cursorSway]);
|
||||
|
||||
useEffect(() => {
|
||||
currentTimeRef.current = currentTime * 1000;
|
||||
}, [currentTime]);
|
||||
const timeMs = currentTime * 1000;
|
||||
currentTimeRef.current = timeMs;
|
||||
const videoInfo = extensionHost.getVideoInfoSnapshot();
|
||||
extensionHost.setPlaybackState({
|
||||
currentTimeMs: timeMs,
|
||||
durationMs: videoInfo?.durationMs ?? 0,
|
||||
isPlaying,
|
||||
});
|
||||
}, [currentTime, isPlaying]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!pixiReady || !videoReady) return;
|
||||
@@ -1115,6 +1324,11 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
videoContainerRef.current = videoContainer;
|
||||
cameraContainer.addChild(videoContainer);
|
||||
|
||||
// Device frame overlay container — sits above video but below cursor
|
||||
const frameContainer = new Container();
|
||||
frameContainerRef.current = frameContainer;
|
||||
cameraContainer.addChild(frameContainer);
|
||||
|
||||
const cursorContainer = new Container();
|
||||
cursorContainerRef.current = cursorContainer;
|
||||
cameraContainer.addChild(cursorContainer);
|
||||
@@ -1164,6 +1378,8 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
appRef.current = null;
|
||||
cameraContainerRef.current = null;
|
||||
videoContainerRef.current = null;
|
||||
frameContainerRef.current = null;
|
||||
frameSpriteRef.current = null;
|
||||
cursorContainerRef.current = null;
|
||||
videoSpriteRef.current = null;
|
||||
};
|
||||
@@ -1423,6 +1639,14 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
state.focusY = targetFocus.cy;
|
||||
state.progress = targetProgress;
|
||||
|
||||
// Push zoom state to extension host for query APIs
|
||||
extensionHost.setZoomState({
|
||||
scale: targetScaleFactor,
|
||||
focusX: targetFocus.cx,
|
||||
focusY: targetFocus.cy,
|
||||
progress: targetProgress,
|
||||
});
|
||||
|
||||
const projectedTransform = computeZoomTransform({
|
||||
stageSize: stageSizeRef.current,
|
||||
baseMask: baseMaskRef.current,
|
||||
@@ -1479,17 +1703,182 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
);
|
||||
applyWebcamBubbleLayout(animationStateRef.current.appliedScale || 1);
|
||||
|
||||
// Update cursor overlay
|
||||
const timeMs = currentTimeRef.current;
|
||||
const effectsCanvas = cursorEffectsCanvasRef.current;
|
||||
const extensionCanvasWidth = effectsCanvas?.width || stageSizeRef.current.width;
|
||||
const extensionCanvasHeight = effectsCanvas?.height || stageSizeRef.current.height;
|
||||
let smoothedCursorForHooks: {
|
||||
cx: number;
|
||||
cy: number;
|
||||
trail: Array<{ cx: number; cy: number }>;
|
||||
} | null = null;
|
||||
|
||||
// Update cursor overlay + emit cursor events
|
||||
const cursorOverlay = cursorOverlayRef.current;
|
||||
if (cursorOverlay) {
|
||||
const timeMs = currentTimeRef.current;
|
||||
const telemetry = cursorTelemetryRef.current;
|
||||
cursorOverlay.update(
|
||||
cursorTelemetryRef.current,
|
||||
telemetry,
|
||||
timeMs,
|
||||
baseMaskRef.current,
|
||||
showCursorRef.current,
|
||||
!isPlayingRef.current || isSeekingRef.current,
|
||||
);
|
||||
|
||||
smoothedCursorForHooks = mapSmoothedCursorToCanvasNormalized(
|
||||
cursorOverlay.getSmoothedCursorSnapshot(),
|
||||
{
|
||||
maskRect: baseMaskRef.current,
|
||||
canvasWidth: extensionCanvasWidth,
|
||||
canvasHeight: extensionCanvasHeight,
|
||||
},
|
||||
);
|
||||
extensionHost.setSmoothedCursor(
|
||||
smoothedCursorForHooks
|
||||
? {
|
||||
timeMs,
|
||||
cx: smoothedCursorForHooks.cx,
|
||||
cy: smoothedCursorForHooks.cy,
|
||||
trail: smoothedCursorForHooks.trail,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
|
||||
// Emit cursor:click events for extensions
|
||||
if (isPlayingRef.current && telemetry.length > 0) {
|
||||
for (let i = telemetry.length - 1; i >= 0; i--) {
|
||||
const p = telemetry[i];
|
||||
if (p.timeMs > timeMs) continue;
|
||||
if (p.timeMs < timeMs - 100) break;
|
||||
if (
|
||||
p.interactionType &&
|
||||
p.interactionType !== 'move' &&
|
||||
p.timeMs !== lastEmittedClickTimeMsRef.current
|
||||
) {
|
||||
const extensionCursor = mapCursorToCanvasNormalized(
|
||||
{
|
||||
cx: p.cx,
|
||||
cy: p.cy,
|
||||
interactionType: p.interactionType,
|
||||
},
|
||||
{
|
||||
maskRect: baseMaskRef.current,
|
||||
canvasWidth: extensionCanvasWidth,
|
||||
canvasHeight: extensionCanvasHeight,
|
||||
},
|
||||
);
|
||||
lastEmittedClickTimeMsRef.current = p.timeMs;
|
||||
extensionHost.emitEvent({
|
||||
type: 'cursor:click',
|
||||
timeMs: p.timeMs,
|
||||
data: extensionCursor,
|
||||
});
|
||||
if (extensionCursor) {
|
||||
notifyCursorInteraction(
|
||||
p.timeMs,
|
||||
extensionCursor.cx,
|
||||
extensionCursor.cy,
|
||||
p.interactionType,
|
||||
);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!cursorOverlay) {
|
||||
extensionHost.setSmoothedCursor(null);
|
||||
}
|
||||
|
||||
if (effectsCanvas && effectsCanvas.width > 0 && effectsCanvas.height > 0) {
|
||||
const ctx2d = effectsCanvas.getContext("2d");
|
||||
if (ctx2d) {
|
||||
ctx2d.clearRect(0, 0, effectsCanvas.width, effectsCanvas.height);
|
||||
|
||||
const maskRect = baseMaskRef.current;
|
||||
const animationState = animationStateRef.current;
|
||||
const videoInfo = extensionHost.getVideoInfoSnapshot();
|
||||
const rawCursor = getCursorPositionAtTime(cursorTelemetryRef.current, timeMs, {
|
||||
maskRect,
|
||||
canvasWidth: effectsCanvas.width,
|
||||
canvasHeight: effectsCanvas.height,
|
||||
});
|
||||
const hookParams = {
|
||||
width: effectsCanvas.width,
|
||||
height: effectsCanvas.height,
|
||||
timeMs,
|
||||
durationMs: videoInfo?.durationMs ?? 0,
|
||||
cursor: smoothedCursorForHooks
|
||||
? {
|
||||
cx: smoothedCursorForHooks.cx,
|
||||
cy: smoothedCursorForHooks.cy,
|
||||
interactionType: rawCursor?.interactionType,
|
||||
}
|
||||
: rawCursor,
|
||||
smoothedCursor: smoothedCursorForHooks,
|
||||
videoLayout:
|
||||
maskRect.width > 0 && maskRect.height > 0
|
||||
? {
|
||||
maskRect: {
|
||||
x: maskRect.x,
|
||||
y: maskRect.y,
|
||||
width: maskRect.width,
|
||||
height: maskRect.height,
|
||||
},
|
||||
borderRadius,
|
||||
padding,
|
||||
}
|
||||
: undefined,
|
||||
zoom: {
|
||||
scale: animationState.scale,
|
||||
focusX: animationState.focusX,
|
||||
focusY: animationState.focusY,
|
||||
progress: animationState.progress,
|
||||
},
|
||||
shadow: {
|
||||
enabled: Boolean(showShadow) && shadowIntensity > 0,
|
||||
intensity: shadowIntensity,
|
||||
},
|
||||
sceneTransform: {
|
||||
scale: animationState.appliedScale,
|
||||
x: animationState.x,
|
||||
y: animationState.y,
|
||||
},
|
||||
};
|
||||
|
||||
ctx2d.save();
|
||||
applyCanvasSceneTransform(ctx2d, {
|
||||
scale: animationState.appliedScale,
|
||||
x: animationState.x,
|
||||
y: animationState.y,
|
||||
});
|
||||
executeExtensionRenderHooks("post-video", ctx2d, hookParams);
|
||||
executeExtensionRenderHooks("post-zoom", ctx2d, hookParams);
|
||||
executeExtensionRenderHooks("post-cursor", ctx2d, hookParams);
|
||||
|
||||
if (isSeekingRef.current) {
|
||||
clearCursorEffects();
|
||||
} else {
|
||||
executeExtensionCursorEffects(
|
||||
ctx2d,
|
||||
timeMs,
|
||||
effectsCanvas.width,
|
||||
effectsCanvas.height,
|
||||
{
|
||||
zoom: hookParams.zoom,
|
||||
sceneTransform: hookParams.sceneTransform,
|
||||
videoLayout: hookParams.videoLayout,
|
||||
},
|
||||
);
|
||||
}
|
||||
ctx2d.restore();
|
||||
|
||||
executeExtensionRenderHooks("post-webcam", ctx2d, hookParams);
|
||||
executeExtensionRenderHooks("post-annotations", ctx2d, hookParams);
|
||||
|
||||
executeExtensionRenderHooks("final", ctx2d, hookParams);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1499,7 +1888,16 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
app.ticker.remove(ticker);
|
||||
}
|
||||
};
|
||||
}, [pixiReady, videoReady, clampFocusToStage, applyWebcamBubbleLayout]);
|
||||
}, [
|
||||
pixiReady,
|
||||
videoReady,
|
||||
clampFocusToStage,
|
||||
applyWebcamBubbleLayout,
|
||||
borderRadius,
|
||||
padding,
|
||||
showShadow,
|
||||
shadowIntensity,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
const overlay = cursorOverlayRef.current;
|
||||
@@ -1530,6 +1928,14 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
) => {
|
||||
const video = e.currentTarget;
|
||||
onDurationChange(video.duration);
|
||||
|
||||
// Push video info to extension host for query APIs
|
||||
extensionHost.setVideoInfo({
|
||||
width: video.videoWidth,
|
||||
height: video.videoHeight,
|
||||
durationMs: Number.isFinite(video.duration) ? video.duration * 1000 : 0,
|
||||
fps: 60, // Not available from HTMLVideoElement; default to 60
|
||||
});
|
||||
const targetTime = clampMediaTimeToDuration(
|
||||
currentTime,
|
||||
Number.isFinite(video.duration) ? video.duration : null,
|
||||
@@ -1729,6 +2135,12 @@ const VideoPlayback = forwardRef<VideoPlaybackRef, VideoPlaybackProps>(
|
||||
: "none",
|
||||
}}
|
||||
/>
|
||||
{/* Canvas overlay for extension cursor effects (drawn via Canvas 2D API) */}
|
||||
<canvas
|
||||
ref={cursorEffectsCanvasRef}
|
||||
className="absolute inset-0 w-full h-full pointer-events-none"
|
||||
style={{ zIndex: 1 }}
|
||||
/>
|
||||
{/* Only render overlay after PIXI and video are fully initialized */}
|
||||
{pixiReady && videoReady && (
|
||||
<div
|
||||
|
||||
@@ -32,6 +32,7 @@ type PersistedEditorControls = Pick<
|
||||
| "cursorSway"
|
||||
| "borderRadius"
|
||||
| "padding"
|
||||
| "frame"
|
||||
| "webcam"
|
||||
| "aspectRatio"
|
||||
| "exportEncodingMode"
|
||||
@@ -84,6 +85,7 @@ export const DEFAULT_EDITOR_PREFERENCES: EditorPreferences = {
|
||||
cursorSway: DEFAULT_EDITOR_CONTROLS.cursorSway,
|
||||
borderRadius: DEFAULT_EDITOR_CONTROLS.borderRadius,
|
||||
padding: DEFAULT_EDITOR_CONTROLS.padding,
|
||||
frame: DEFAULT_EDITOR_CONTROLS.frame,
|
||||
webcam: DEFAULT_EDITOR_CONTROLS.webcam,
|
||||
aspectRatio: DEFAULT_EDITOR_CONTROLS.aspectRatio,
|
||||
exportEncodingMode: DEFAULT_EDITOR_CONTROLS.exportEncodingMode,
|
||||
@@ -166,6 +168,7 @@ function normalizeEditorControls(
|
||||
cursorSway: raw.cursorSway ?? fallback.cursorSway,
|
||||
borderRadius: raw.borderRadius ?? fallback.borderRadius,
|
||||
padding: raw.padding ?? fallback.padding,
|
||||
frame: raw.frame !== undefined ? raw.frame : fallback.frame,
|
||||
webcam: raw.webcam ?? fallback.webcam,
|
||||
aspectRatio: raw.aspectRatio ?? fallback.aspectRatio,
|
||||
exportEncodingMode: raw.exportEncodingMode ?? fallback.exportEncodingMode,
|
||||
@@ -215,6 +218,7 @@ function normalizeEditorControls(
|
||||
cursorSway: normalized.cursorSway,
|
||||
borderRadius: normalized.borderRadius,
|
||||
padding: normalized.padding,
|
||||
frame: normalized.frame,
|
||||
webcam: normalized.webcam,
|
||||
aspectRatio: normalized.aspectRatio,
|
||||
exportEncodingMode: normalized.exportEncodingMode,
|
||||
|
||||
@@ -92,6 +92,8 @@ export interface ProjectEditorState {
|
||||
cursorSway: number;
|
||||
borderRadius: number;
|
||||
padding: number;
|
||||
/** Selected frame ID (e.g. "recordly.frames/browser-dark"), or null for none */
|
||||
frame: string | null;
|
||||
cropRegion: CropRegion;
|
||||
zoomRegions: ZoomRegion[];
|
||||
trimRegions: TrimRegion[];
|
||||
@@ -655,6 +657,7 @@ export function normalizeProjectEditor(editor: Partial<ProjectEditorState>): Pro
|
||||
: DEFAULT_CURSOR_SWAY,
|
||||
borderRadius: typeof editor.borderRadius === "number" ? editor.borderRadius : 12.5,
|
||||
padding: isFiniteNumber(editor.padding) ? clamp(editor.padding, 0, 100) : 20,
|
||||
frame: typeof editor.frame === "string" ? editor.frame : null,
|
||||
cropRegion: {
|
||||
x: cropX,
|
||||
y: cropY,
|
||||
|
||||
Reference in New Issue
Block a user