import { ArrowLeft, Gift, PartyPopper, RotateCcw, SkipForward, } from 'lucide-react'; import { type CSSProperties, type PointerEvent as ReactPointerEvent, useCallback, useEffect, useRef, useState, } from 'react'; import type { BabyObjectMatchDraft, BabyObjectMatchItemAsset, BabyObjectMatchVisualAsset, BabyObjectMatchVisualAssetKind, } from '../../../packages/shared/src/contracts/edutainmentBabyObject'; import type { MocapHandInput, MocapInputCommand, UseMocapInputResult, } from '../../services/useMocapInput'; import { useMocapInput } from '../../services/useMocapInput'; import { ResolvedAssetImage } from '../ResolvedAssetImage'; const BABY_OBJECT_MATCH_SUCCESS_TARGET = 20; const BABY_OBJECT_MATCH_GIFT_APPEAR_DURATION_MS = 620; const BABY_OBJECT_MATCH_GIFT_OPEN_DURATION_MS = 640; const BABY_OBJECT_MATCH_ITEM_APPEAR_DURATION_MS = 620; const BABY_OBJECT_MATCH_FEEDBACK_DURATION_MS = 1180; const BABY_OBJECT_MATCH_INTRO_SHOW_DURATION_MS = 2000; const BABY_OBJECT_MATCH_INTRO_FLY_DURATION_MS = 720; const BABY_OBJECT_MATCH_INTRO_READY_PAUSE_MS = 1000; const BABY_OBJECT_MATCH_ITEM_CENTER: RuntimeHandPoint = { x: 0.5, y: 0.37 }; const BABY_OBJECT_MATCH_ITEM_GRAB_RADIUS = 0.14; // 篮子仍只认主体附近,但在上一版核心区基础上扩大约 50%,避免贴近篮子后仍难以命中。 const BABY_OBJECT_MATCH_BASKET_DROP_Y = 0.62; const BABY_OBJECT_MATCH_LEFT_BASKET_MAX_X = 0.36; const BABY_OBJECT_MATCH_RIGHT_BASKET_MIN_X = 0.64; type BabyObjectMatchRuntimeShellProps = { draft: BabyObjectMatchDraft; embedded?: boolean; enableMocapInput?: boolean; mocapInput?: UseMocapInputResult | null; random?: BabyObjectMatchRandom; onBack?: () => void; onNextLevel?: () => void; }; type BasketSide = 'left' | 'right'; type RuntimePhase = | 'intro-left-showing' | 'intro-left-flying' | 'intro-left-ready' | 'intro-right-showing' | 'intro-right-flying' | 'intro-right-ready' | 'gift-entering' | 'gift-opening' | 'item-appearing' | 'active' | 'correct' | 'wrong' | 'complete'; type RuntimeRound = { item: BabyObjectMatchItemAsset; baskets: Record; }; type RuntimeIntroShowcase = { side: BasketSide; item: BabyObjectMatchItemAsset; isFlying: boolean; }; type RuntimeHandPoint = { x: number; y: number; }; type RuntimeHandRole = 'left' | 'right'; type RuntimeHands = Record; type HeldItemState = { hand: RuntimeHandRole; }; type BabyObjectMatchRandom = () => number; type BabyObjectMatchStageStyle = CSSProperties & Partial< Record< | '--baby-object-ui-frame-image' | '--baby-object-gift-box-image' | '--baby-object-basket-image' | '--baby-object-smoke-image', string > >; function pickRandomIndex(length: number, random: BabyObjectMatchRandom) { if (length <= 1) { return 0; } return Math.min(length - 1, Math.floor(random() * length)); } function buildRuntimeRound( draft: BabyObjectMatchDraft, random: BabyObjectMatchRandom, ): RuntimeRound { const items = draft.itemAssets; const item = items[pickRandomIndex(items.length, random)] ?? items[0]!; return { item, baskets: { left: items[0]!, right: items[1]!, }, }; } function mocapHandToRuntimePoint( hand: MocapHandInput | null | undefined, skeletonWrist: RuntimeHandPoint | null | undefined, ): RuntimeHandPoint | null { if (skeletonWrist) { return clampRuntimePoint(skeletonWrist); } if (!hand) { return null; } // 骨架 wrist 缺失时再回退到手部 landmarks 的 wrist,最后才使用手部派生点。 const point = hand.wrist ?? hand; return clampRuntimePoint({ x: point.x, y: point.y }); } function clampRuntimePoint(point: RuntimeHandPoint): RuntimeHandPoint { return { x: Math.max(0, Math.min(1, point.x)), y: Math.max(0, Math.min(1, point.y)), }; } function isRuntimePointTouchingItem(point: RuntimeHandPoint) { const dx = point.x - BABY_OBJECT_MATCH_ITEM_CENTER.x; const dy = point.y - BABY_OBJECT_MATCH_ITEM_CENTER.y; return Math.sqrt(dx * dx + dy * dy) <= BABY_OBJECT_MATCH_ITEM_GRAB_RADIUS; } function isRuntimeControlPointerTarget(target: EventTarget | null) { return ( target instanceof Element && target.closest( 'button, a, input, select, textarea, [role="button"], [data-baby-object-runtime-control="true"]', ) !== null ); } function resolveBasketSideForPoint(point: RuntimeHandPoint): BasketSide | null { if (point.y < BABY_OBJECT_MATCH_BASKET_DROP_Y) { return null; } if (point.x <= BABY_OBJECT_MATCH_LEFT_BASKET_MAX_X) { return 'left'; } if (point.x >= BABY_OBJECT_MATCH_RIGHT_BASKET_MIN_X) { return 'right'; } return null; } function resolveMocapRuntimeHands(command: MocapInputCommand): RuntimeHands { // 本地 mocap 当前按摄像头视角输出 handedness,这里换回用户身体视角用于显示双手。 return { left: mocapHandToRuntimePoint( command.rightHand, command.bodyJoints?.rightWrist, ), right: mocapHandToRuntimePoint( command.leftHand, command.bodyJoints?.leftWrist, ), }; } function buildMocapPacketKey( command: MocapInputCommand, rawPacketPreview: UseMocapInputResult['rawPacketPreview'], ) { return rawPacketPreview?.receivedAtMs !== undefined ? `${rawPacketPreview.receivedAtMs}:${rawPacketPreview.text}` : JSON.stringify(command); } function findVisualAsset( draft: BabyObjectMatchDraft, kind: BabyObjectMatchVisualAssetKind, ): BabyObjectMatchVisualAsset | null { return ( draft.visualPackage?.assets.find((asset) => asset.assetKind === kind) ?? null ); } function buildCssImageValue(src: string) { return `url("${src.replace(/"/gu, '\\"')}")`; } function resolveIntroShowcase( phase: RuntimePhase, draft: BabyObjectMatchDraft, ): RuntimeIntroShowcase | null { if (phase === 'intro-left-showing' || phase === 'intro-left-flying') { const item = draft.itemAssets[0]; return item ? { side: 'left', item, isFlying: phase === 'intro-left-flying' } : null; } if (phase === 'intro-right-showing' || phase === 'intro-right-flying') { const item = draft.itemAssets[1]; return item ? { side: 'right', item, isFlying: phase === 'intro-right-flying' } : null; } return null; } function isBasketOptionReadyInIntro(side: BasketSide, phase: RuntimePhase) { if (!phase.startsWith('intro-')) { return true; } if (side === 'left') { return ( phase === 'intro-left-ready' || phase === 'intro-right-showing' || phase === 'intro-right-flying' || phase === 'intro-right-ready' ); } return phase === 'intro-right-ready'; } export function BabyObjectMatchRuntimeShell({ draft, embedded = false, enableMocapInput = true, mocapInput = null, random, onBack, onNextLevel, }: BabyObjectMatchRuntimeShellProps) { const randomRef = useRef( random ?? (() => Math.random()), ); const introTimerRef = useRef(null); const feedbackTimerRef = useRef(null); const handledMocapPacketKeyRef = useRef(null); const latestMocapPacketKeyRef = useRef(null); const [phase, setPhase] = useState('intro-left-showing'); const [successCount, setSuccessCount] = useState(0); const [round, setRound] = useState(() => buildRuntimeRound(draft, randomRef.current), ); const [feedbackText, setFeedbackText] = useState(null); const [lastTargetSide, setLastTargetSide] = useState(null); const [runtimeHands, setRuntimeHands] = useState({ left: null, right: null, }); const [heldItem, setHeldItem] = useState(null); const liveMocapInput = useMocapInput({ enabled: enableMocapInput && !mocapInput, }); const resolvedMocapInput = mocapInput ?? liveMocapInput; const backgroundAsset = findVisualAsset(draft, 'background'); const uiFrameAsset = findVisualAsset(draft, 'ui-frame'); const giftBoxAsset = findVisualAsset(draft, 'gift-box'); const basketAsset = findVisualAsset(draft, 'basket'); const smokeAsset = findVisualAsset(draft, 'smoke-puff'); const stageStyle: BabyObjectMatchStageStyle = { ...(uiFrameAsset ? { '--baby-object-ui-frame-image': buildCssImageValue( uiFrameAsset.imageSrc, ), } : {}), ...(giftBoxAsset ? { '--baby-object-gift-box-image': buildCssImageValue( giftBoxAsset.imageSrc, ), } : {}), ...(basketAsset ? { '--baby-object-basket-image': buildCssImageValue( basketAsset.imageSrc, ), } : {}), ...(smokeAsset ? { '--baby-object-smoke-image': buildCssImageValue( smokeAsset.imageSrc, ), } : {}), }; const progressText = `${successCount}/${BABY_OBJECT_MATCH_SUCCESS_TARGET}`; const isComplete = phase === 'complete'; const currentItem = round?.item ?? null; const isJudgementOpen = phase === 'active'; const introShowcase = resolveIntroShowcase(phase, draft); const heldPoint = heldItem ? runtimeHands[heldItem.hand] : null; const shouldShowCurrentItem = currentItem && (phase === 'item-appearing' || phase === 'active' || phase === 'correct' || phase === 'wrong'); const shouldShowGift = phase === 'gift-entering' || phase === 'gift-opening'; const shouldShowSmoke = phase === 'gift-opening' || phase === 'item-appearing'; useEffect(() => { randomRef.current = random ?? (() => Math.random()); }, [random]); useEffect(() => { latestMocapPacketKeyRef.current = resolvedMocapInput.latestCommand ? buildMocapPacketKey( resolvedMocapInput.latestCommand, resolvedMocapInput.rawPacketPreview, ) : null; }, [resolvedMocapInput.latestCommand, resolvedMocapInput.rawPacketPreview]); const clearFeedbackTimer = useCallback(() => { if (feedbackTimerRef.current !== null) { window.clearTimeout(feedbackTimerRef.current); feedbackTimerRef.current = null; } }, []); const clearIntroTimer = useCallback(() => { if (introTimerRef.current !== null) { window.clearTimeout(introTimerRef.current); introTimerRef.current = null; } }, []); const resetInputPaths = useCallback(() => { handledMocapPacketKeyRef.current = null; setHeldItem(null); }, []); useEffect(() => { clearIntroTimer(); if (phase === 'intro-left-showing') { introTimerRef.current = window.setTimeout(() => { introTimerRef.current = null; setPhase('intro-left-flying'); }, BABY_OBJECT_MATCH_INTRO_SHOW_DURATION_MS); return clearIntroTimer; } if (phase === 'intro-left-flying') { introTimerRef.current = window.setTimeout(() => { introTimerRef.current = null; setPhase('intro-left-ready'); }, BABY_OBJECT_MATCH_INTRO_FLY_DURATION_MS); return clearIntroTimer; } if (phase === 'intro-left-ready') { introTimerRef.current = window.setTimeout(() => { introTimerRef.current = null; setPhase('intro-right-showing'); }, BABY_OBJECT_MATCH_INTRO_READY_PAUSE_MS); return clearIntroTimer; } if (phase === 'intro-right-showing') { introTimerRef.current = window.setTimeout(() => { introTimerRef.current = null; setPhase('intro-right-flying'); }, BABY_OBJECT_MATCH_INTRO_SHOW_DURATION_MS); return clearIntroTimer; } if (phase === 'intro-right-flying') { introTimerRef.current = window.setTimeout(() => { introTimerRef.current = null; setPhase('intro-right-ready'); }, BABY_OBJECT_MATCH_INTRO_FLY_DURATION_MS); return clearIntroTimer; } if (phase === 'intro-right-ready') { introTimerRef.current = window.setTimeout(() => { introTimerRef.current = null; setPhase('gift-entering'); }, BABY_OBJECT_MATCH_INTRO_READY_PAUSE_MS); return clearIntroTimer; } if (phase === 'gift-entering') { introTimerRef.current = window.setTimeout(() => { introTimerRef.current = null; setPhase('gift-opening'); }, BABY_OBJECT_MATCH_GIFT_APPEAR_DURATION_MS); return clearIntroTimer; } if (phase === 'gift-opening') { introTimerRef.current = window.setTimeout(() => { introTimerRef.current = null; setPhase('item-appearing'); }, BABY_OBJECT_MATCH_GIFT_OPEN_DURATION_MS); return clearIntroTimer; } if (phase === 'item-appearing') { introTimerRef.current = window.setTimeout(() => { introTimerRef.current = null; resetInputPaths(); handledMocapPacketKeyRef.current = latestMocapPacketKeyRef.current; setPhase('active'); }, BABY_OBJECT_MATCH_ITEM_APPEAR_DURATION_MS); return clearIntroTimer; } return clearIntroTimer; }, [clearIntroTimer, phase, resetInputPaths]); const resetRuntime = useCallback(() => { clearIntroTimer(); clearFeedbackTimer(); resetInputPaths(); setSuccessCount(0); setRound(buildRuntimeRound(draft, randomRef.current)); setFeedbackText(null); setLastTargetSide(null); setPhase('intro-left-showing'); }, [clearFeedbackTimer, clearIntroTimer, draft, resetInputPaths]); const finishFeedback = useCallback( (nextSuccessCount: number, wasCorrect: boolean) => { clearIntroTimer(); clearFeedbackTimer(); feedbackTimerRef.current = window.setTimeout(() => { feedbackTimerRef.current = null; if (wasCorrect) { if (nextSuccessCount >= BABY_OBJECT_MATCH_SUCCESS_TARGET) { setFeedbackText('恭喜你!小朋友!'); setRound(null); setPhase('complete'); return; } setRound(buildRuntimeRound(draft, randomRef.current)); setFeedbackText(null); setLastTargetSide(null); resetInputPaths(); setPhase('gift-entering'); return; } setFeedbackText(null); setLastTargetSide(null); resetInputPaths(); setPhase('active'); }, BABY_OBJECT_MATCH_FEEDBACK_DURATION_MS); }, [clearFeedbackTimer, clearIntroTimer, draft, resetInputPaths], ); const sendItemToBasket = useCallback( (side: BasketSide) => { if (!isJudgementOpen || !round) { return; } const isCorrect = round.baskets[side].itemId === round.item.itemId; setLastTargetSide(side); if (isCorrect) { const nextSuccessCount = successCount + 1; setSuccessCount(nextSuccessCount); setFeedbackText('真棒'); setPhase('correct'); finishFeedback(nextSuccessCount, true); return; } setFeedbackText('再想一想吧'); setPhase('wrong'); finishFeedback(successCount, false); }, [finishFeedback, isJudgementOpen, round, successCount], ); useEffect( () => () => { clearIntroTimer(); clearFeedbackTimer(); }, [clearFeedbackTimer, clearIntroTimer], ); useEffect(() => { const command = resolvedMocapInput.latestCommand; if (!command || isComplete) { return; } const packetKey = buildMocapPacketKey( command, resolvedMocapInput.rawPacketPreview, ); if (handledMocapPacketKeyRef.current === packetKey) { return; } handledMocapPacketKeyRef.current = packetKey; const nextHands = resolveMocapRuntimeHands(command); setRuntimeHands(nextHands); if (!isJudgementOpen) { resetInputPaths(); return; } const currentHeldItem = heldItem; if (currentHeldItem) { const heldHandPoint = nextHands[currentHeldItem.hand]; const targetSide = heldHandPoint ? resolveBasketSideForPoint(heldHandPoint) : null; if (!targetSide) { return; } sendItemToBasket(targetSide); resetInputPaths(); return; } for (const hand of ['left', 'right'] as const) { const point = nextHands[hand]; if (!point || !isRuntimePointTouchingItem(point)) { continue; } setHeldItem({ hand }); return; } }, [ heldItem, isComplete, isJudgementOpen, resetInputPaths, resolvedMocapInput.latestCommand, resolvedMocapInput.rawPacketPreview, sendItemToBasket, ]); const getPointerUnitPoint = ( event: ReactPointerEvent, element: HTMLElement, ): RuntimeHandPoint => { const rect = element.getBoundingClientRect(); const width = rect.width || 1; const height = rect.height || 1; return clampRuntimePoint({ x: (event.clientX - rect.left) / width, y: (event.clientY - rect.top) / height, }); }; const handlePointerDown = (event: ReactPointerEvent) => { if (isRuntimeControlPointerTarget(event.target)) { return; } if (!isJudgementOpen) { return; } if (event.button !== 0 && event.button !== 2) { return; } const hand: RuntimeHandRole = event.button === 2 ? 'right' : 'left'; const point = getPointerUnitPoint(event, event.currentTarget); setRuntimeHands((current) => ({ ...current, [hand]: point })); if (isRuntimePointTouchingItem(point)) { setHeldItem({ hand }); } event.preventDefault(); if (typeof event.currentTarget.setPointerCapture === 'function') { event.currentTarget.setPointerCapture(event.pointerId); } }; const handlePointerMove = (event: ReactPointerEvent) => { if (isRuntimeControlPointerTarget(event.target)) { return; } if (!isJudgementOpen) { return; } if (event.buttons !== 1 && event.buttons !== 2) { return; } const hand: RuntimeHandRole = event.buttons === 2 ? 'right' : 'left'; const point = getPointerUnitPoint(event, event.currentTarget); setRuntimeHands((current) => ({ ...current, [hand]: point })); if (!heldItem && isRuntimePointTouchingItem(point)) { setHeldItem({ hand }); return; } if (!heldItem || heldItem.hand !== hand) { return; } const targetSide = resolveBasketSideForPoint(point); if (targetSide) { sendItemToBasket(targetSide); resetInputPaths(); } }; const handlePointerUp = (event: ReactPointerEvent) => { if ( typeof event.currentTarget.hasPointerCapture === 'function' && event.currentTarget.hasPointerCapture(event.pointerId) ) { event.currentTarget.releasePointerCapture(event.pointerId); } }; return (
event.preventDefault()} > {backgroundAsset ? (
); } export default BabyObjectMatchRuntimeShell;