import type { CSSProperties, PointerEvent as ReactPointerEvent } from 'react'; import { useCallback, useEffect, useRef, useState } from 'react'; import type { BabyObjectMatchDraft } from '../../../packages/shared/src/contracts/edutainmentBabyObject'; import { BABY_OBJECT_MATCH_EDUTAINMENT_TAG, BABY_OBJECT_MATCH_TEMPLATE_ID, BABY_OBJECT_MATCH_TEMPLATE_NAME, } from '../../../packages/shared/src/contracts/edutainmentBabyObject'; import type { MocapConnectionStatus, MocapHandInput, MocapInputCommand, MocapPointInput, } from '../../services/useMocapInput'; import { useMocapInput } from '../../services/useMocapInput'; import { BabyObjectMatchRuntimeShell } from '../edutainment-runtime/BabyObjectMatchRuntimeShell'; import { applyChildMotionWarmupCompletion, CHILD_MOTION_CENTER_X, CHILD_MOTION_FINISH_DURATION_MS, CHILD_MOTION_HOLD_DURATION_MS, CHILD_MOTION_NARRATION_DURATION_MS, type ChildMotionPoint, type ChildMotionWarmupCalibration, type ChildMotionWarmupStepId, createEmptyChildMotionCalibration, getChildMotionTargetX, getChildMotionWarmupStep, hasCompletedChildMotionWarmupInRuntime, isAvatarOnWarmupTarget, markChildMotionWarmupCompletedInRuntime, resolveNextChildMotionWarmupStep, } from './childMotionWarmupModel'; type DragHand = 'left' | 'right'; type CameraAccessState = 'idle' | 'requesting' | 'ready' | 'blocked'; type MotionSourceState = 'connecting' | 'ready' | 'waiting' | 'offline'; type WarmupStepPhase = 'intro' | 'active' | 'complete'; type WarmupMocapGestureIntent = | 'greeting' | 'left-hand' | 'right-hand'; type WarmupBodyHandSide = 'left' | 'right'; type WarmupHandIndicators = Record; const CHILD_MOTION_BABY_OBJECT_DEMO_DRAFT: BabyObjectMatchDraft = { draftId: 'child-motion-demo-baby-object-draft', profileId: 'child-motion-demo-baby-object-profile', templateId: BABY_OBJECT_MATCH_TEMPLATE_ID, templateName: BABY_OBJECT_MATCH_TEMPLATE_NAME, workTitle: '宝贝识物', workDescription: '苹果和香蕉识物分类', itemNames: ['苹果', '香蕉'], itemAssets: [ { itemId: 'child-motion-demo-baby-object-apple', itemName: '苹果', imageSrc: 'data:image/svg+xml;utf8,%3Csvg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 512 512%22%3E%3Crect width=%22512%22 height=%22512%22 rx=%2296%22 fill=%22%23fff1d6%22/%3E%3Ccircle cx=%22256%22 cy=%22266%22 r=%22122%22 fill=%22%23ef5b5b%22/%3E%3Cpath d=%22M250 148c20-50 58-66 102-54-18 45-52 70-102 54Z%22 fill=%22%2351a45f%22/%3E%3Cpath d=%22M256 150c-8-34 2-62 28-84%22 stroke=%22%23734822%22 stroke-width=%2218%22 stroke-linecap=%22round%22 fill=%22none%22/%3E%3Ccircle cx=%22216%22 cy=%22226%22 r=%2218%22 fill=%22%23fff%22 opacity=%22.65%22/%3E%3C/svg%3E', assetObjectId: null, generationProvider: 'placeholder', prompt: '苹果', }, { itemId: 'child-motion-demo-baby-object-banana', itemName: '香蕉', imageSrc: 'data:image/svg+xml;utf8,%3Csvg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 512 512%22%3E%3Crect width=%22512%22 height=%22512%22 rx=%2296%22 fill=%22%23e9f7ff%22/%3E%3Cpath d=%22M142 302c128 74 228 38 278-122 14 144-84 244-226 220-52-9-84-38-52-98Z%22 fill=%22%23ffd75d%22/%3E%3Cpath d=%22M406 180c6-20 18-34 38-44%22 stroke=%22%238b5b22%22 stroke-width=%2218%22 stroke-linecap=%22round%22/%3E%3Cpath d=%22M158 310c70 40 152 42 218-38%22 stroke=%22%23fff2a7%22 stroke-width=%2220%22 stroke-linecap=%22round%22 fill=%22none%22 opacity=%22.72%22/%3E%3C/svg%3E', assetObjectId: null, generationProvider: 'placeholder', prompt: '香蕉', }, ], visualPackage: null, themeTags: [BABY_OBJECT_MATCH_EDUTAINMENT_TAG, '宝贝识物'], publicationStatus: 'published', createdAt: '2026-05-11T00:00:00.000Z', updatedAt: '2026-05-11T00:00:00.000Z', publishedAt: '2026-05-11T00:00:00.000Z', }; const WARMUP_MOCAP_WAVE_MIN_X_RANGE = 0.055; const WARMUP_ARM_SWING_MIN_POINTS = 5; const WARMUP_ARM_SWING_MIN_VERTICAL_RANGE = 0.08; const WARMUP_ARM_SWING_MIN_ANGLE_RANGE_DEG = 28; const WARMUP_ARM_SWING_MIN_REACH = 0.12; const WARMUP_ARM_SWING_MIN_OUTWARD_X = 0.1; const WARMUP_ARM_SWING_DIRECTION_EPSILON = 0.012; const WARMUP_GREETING_WAVE_MIN_POINTS = 5; const WARMUP_GREETING_WAVE_MIN_X_RANGE = 0.075; const WARMUP_GREETING_WAVE_MIN_DIRECTION_CHANGES = 1; const WARMUP_GREETING_WAVE_DIRECTION_EPSILON = 0.008; const WARMUP_GREETING_WRIST_ABOVE_ELBOW_MARGIN = 0.04; const WARMUP_GREETING_WRIST_ABOVE_SHOULDER_MARGIN = 0.08; const WARMUP_STEP_INTRO_DELAY_MS = 1000; const WARMUP_SUBTITLE_LINE_DELAY_MS = 2000; const WARMUP_STEP_COMPLETE_PAUSE_MS = 820; const WARMUP_TOTAL_STEPS = 11; const AVATAR_MOCAP_DEAD_ZONE = 0.012; const AVATAR_MOCAP_SMOOTHING = 0.28; const AVATAR_MOCAP_MAX_STEP = 0.035; function clampMotionUnit(value: number) { return Math.max(0, Math.min(1, value)); } function normalizePointerPoint( event: ReactPointerEvent, element: HTMLElement, ): ChildMotionPoint { const rect = element.getBoundingClientRect(); const width = rect.width || 1; const height = rect.height || 1; return { x: clampMotionUnit((event.clientX - rect.left) / width), y: clampMotionUnit((event.clientY - rect.top) / height), }; } function formatPercent(value: number | null) { if (value === null) { return '--'; } return `${Math.round(value * 100)}%`; } function formatAvatarLeftPercent(value: number) { return `${Math.round(clampMotionUnit(value) * 1000) / 10}%`; } function createEmptyWarmupHandIndicators(): WarmupHandIndicators { return { left: null, right: null, }; } function resolveMocapHandWithBodySide( command: MocapInputCommand, side: WarmupBodyHandSide, ) { // 本地 mocap 的 handedness 目前是摄像头视角:画面右侧手对应用户身体左手。 return side === 'left' ? command.rightHand : command.leftHand; } function resolveMocapSkeletonWristWithBodySide( command: MocapInputCommand, side: WarmupBodyHandSide, ) { const joints = command.bodyJoints; return side === 'left' ? joints?.rightWrist : joints?.leftWrist; } function resolveWarmupHandIndicatorsFromMocap( command: MocapInputCommand, ): WarmupHandIndicators { return { left: mocapHandToWarmupIndicatorPoint( resolveMocapHandWithBodySide(command, 'left'), resolveMocapSkeletonWristWithBodySide(command, 'left'), ), right: mocapHandToWarmupIndicatorPoint( resolveMocapHandWithBodySide(command, 'right'), resolveMocapSkeletonWristWithBodySide(command, 'right'), ), }; } function resolveMocapJointWithBodySide( command: MocapInputCommand, side: WarmupBodyHandSide, joint: 'shoulder' | 'elbow', ) { const joints = command.bodyJoints; if (side === 'left') { return joint === 'shoulder' ? joints?.rightShoulder : joints?.rightElbow; } return joint === 'shoulder' ? joints?.leftShoulder : joints?.leftElbow; } function mocapHandToChildMotionPoint( hand: MocapHandInput | null | undefined, command?: MocapInputCommand, bodySide?: WarmupBodyHandSide, ): ChildMotionPoint | null { if (!hand) { return null; } const armMetrics = command && bodySide ? resolveWarmupArmMetrics(hand, command, bodySide) : null; return { x: clampMotionUnit(hand.x), y: clampMotionUnit(hand.y), isRaised: command ? isWarmupGreetingHandRaised(hand, command, bodySide) : undefined, isArmExtended: armMetrics?.isExtended, armAngleDeg: armMetrics?.angleDeg, armReach: armMetrics?.reach, }; } function mocapHandToWarmupIndicatorPoint( hand: MocapHandInput | null | undefined, skeletonWrist: MocapPointInput | null | undefined, ): ChildMotionPoint | null { // 骨架手腕节点比手掌识别结果更稳定;热身指示器优先跟随骨架手腕。 const point = skeletonWrist ?? hand?.wrist ?? hand; if (!point) { return null; } return { x: clampMotionUnit(point.x), y: clampMotionUnit(point.y), }; } function appendWarmupMocapPoint( points: ChildMotionPoint[], point: ChildMotionPoint, ) { return [...points, point].slice(-16); } function getMotionSourceState( mocapStatus: MocapConnectionStatus, latestCommand: MocapInputCommand | null, ): MotionSourceState { if (mocapStatus === 'connecting' || mocapStatus === 'idle') { return 'connecting'; } if (mocapStatus === 'connected') { return latestCommand && (Boolean(latestCommand.bodyCenter) || Boolean(latestCommand.hands?.length) || latestCommand.actions.length > 0) ? 'ready' : 'waiting'; } return 'offline'; } function getMotionSourceText(state: MotionSourceState) { if (state === 'ready') { return '动作数据已连接'; } if (state === 'waiting') { return '动作数据已连接,等待识别'; } if (state === 'offline') { return '动作数据暂不可用,已保留本地调试'; } return '正在连接动作数据'; } function countWarmupVerticalDirectionChanges(points: ChildMotionPoint[]) { let previousDirection = 0; let directionChanges = 0; for (let index = 1; index < points.length; index += 1) { const delta = points[index]!.y - points[index - 1]!.y; if (Math.abs(delta) < WARMUP_ARM_SWING_DIRECTION_EPSILON) { continue; } const direction = Math.sign(delta); if (previousDirection !== 0 && direction !== previousDirection) { directionChanges += 1; } previousDirection = direction; } return directionChanges; } function hasWarmupArmSwingPath(points: ChildMotionPoint[]) { const extendedPoints = points.filter((point) => point.isArmExtended); if (extendedPoints.length < WARMUP_ARM_SWING_MIN_POINTS) { return false; } const xValues = extendedPoints.map((point) => point.x); const yValues = extendedPoints.map((point) => point.y); const angleValues = extendedPoints .map((point) => point.armAngleDeg) .filter((angle): angle is number => typeof angle === 'number'); const xRange = Math.max(...xValues) - Math.min(...xValues); const yRange = Math.max(...yValues) - Math.min(...yValues); const angleRange = angleValues.length > 0 ? Math.max(...angleValues) - Math.min(...angleValues) : 0; return ( xRange >= WARMUP_MOCAP_WAVE_MIN_X_RANGE && yRange >= WARMUP_ARM_SWING_MIN_VERTICAL_RANGE && angleRange >= WARMUP_ARM_SWING_MIN_ANGLE_RANGE_DEG && countWarmupVerticalDirectionChanges(extendedPoints) >= 1 ); } function countWarmupHorizontalDirectionChanges(points: ChildMotionPoint[]) { let previousDirection = 0; let directionChanges = 0; for (let index = 1; index < points.length; index += 1) { const delta = points[index]!.x - points[index - 1]!.x; if (Math.abs(delta) < WARMUP_GREETING_WAVE_DIRECTION_EPSILON) { continue; } const direction = Math.sign(delta); if (previousDirection !== 0 && direction !== previousDirection) { directionChanges += 1; } previousDirection = direction; } return directionChanges; } function hasWarmupGreetingWavePath(points: ChildMotionPoint[]) { const raisedPoints = points.filter((point) => point.isRaised); if (raisedPoints.length < WARMUP_GREETING_WAVE_MIN_POINTS) { return false; } const xValues = raisedPoints.map((point) => point.x); const xRange = Math.max(...xValues) - Math.min(...xValues); return ( xRange >= WARMUP_GREETING_WAVE_MIN_X_RANGE && countWarmupHorizontalDirectionChanges(raisedPoints) >= WARMUP_GREETING_WAVE_MIN_DIRECTION_CHANGES ); } function isWarmupGreetingHandRaised( hand: MocapHandInput, command: MocapInputCommand, bodySide?: WarmupBodyHandSide, ) { const wrist = hand.wrist ?? { x: hand.x, y: hand.y }; const elbow = bodySide ? resolveMocapJointWithBodySide(command, bodySide, 'elbow') : hand.side === 'left' ? command.bodyJoints?.leftElbow : hand.side === 'right' ? command.bodyJoints?.rightElbow : null; if (elbow) { return wrist.y <= elbow.y + WARMUP_GREETING_WRIST_ABOVE_ELBOW_MARGIN; } const shoulder = bodySide ? resolveMocapJointWithBodySide(command, bodySide, 'shoulder') : hand.side === 'left' ? command.bodyJoints?.leftShoulder : hand.side === 'right' ? command.bodyJoints?.rightShoulder : null; if (shoulder) { return wrist.y <= shoulder.y + WARMUP_GREETING_WRIST_ABOVE_SHOULDER_MARGIN; } return false; } function getWarmupPointDistance(left: MocapPointInput, right: MocapPointInput) { return Math.hypot(left.x - right.x, left.y - right.y); } function resolveWarmupArmMetrics( hand: MocapHandInput, command: MocapInputCommand, bodySide: WarmupBodyHandSide, ) { const wrist = hand.wrist ?? { x: hand.x, y: hand.y }; const shoulder = resolveMocapJointWithBodySide(command, bodySide, 'shoulder'); if (!shoulder) { return null; } const elbow = resolveMocapJointWithBodySide(command, bodySide, 'elbow'); const reach = getWarmupPointDistance(shoulder, wrist); const outwardX = bodySide === 'left' ? shoulder.x - wrist.x : wrist.x - shoulder.x; const upperArmReach = elbow ? getWarmupPointDistance(shoulder, elbow) : null; const angleDeg = (Math.atan2(shoulder.y - wrist.y, Math.abs(wrist.x - shoulder.x)) * 180) / Math.PI; const isNotDrooping = elbow ? wrist.y <= elbow.y + WARMUP_GREETING_WRIST_ABOVE_ELBOW_MARGIN : wrist.y <= shoulder.y + WARMUP_GREETING_WRIST_ABOVE_SHOULDER_MARGIN; const isExtended = outwardX >= WARMUP_ARM_SWING_MIN_OUTWARD_X && reach >= WARMUP_ARM_SWING_MIN_REACH && (!upperArmReach || reach >= upperArmReach * 1.2) && isNotDrooping; return { angleDeg, reach, isExtended, }; } function resolveAvatarXFromMocap(command: MocapInputCommand) { const bodyCenterX = command.bodyCenter?.x; if (typeof bodyCenterX !== 'number' || !Number.isFinite(bodyCenterX)) { return null; } return clampMotionUnit(bodyCenterX); } function resolveDampedAvatarX(current: number, target: number) { const clampedCurrent = clampMotionUnit(current); const clampedTarget = clampMotionUnit(target); const delta = clampedTarget - clampedCurrent; if (Math.abs(delta) <= AVATAR_MOCAP_DEAD_ZONE) { return clampedCurrent; } const smoothedDelta = delta * AVATAR_MOCAP_SMOOTHING; const limitedDelta = Math.sign(smoothedDelta) * Math.min(Math.abs(smoothedDelta), AVATAR_MOCAP_MAX_STEP); return clampMotionUnit(clampedCurrent + limitedDelta); } function resolveWarmupMocapGestureIntent( stepId: ChildMotionWarmupStepId, paths: { leftHandPath: ChildMotionPoint[]; rightHandPath: ChildMotionPoint[]; primaryHandPath: ChildMotionPoint[]; }, ): WarmupMocapGestureIntent | null { if (stepId === 'wave_greeting') { if ( hasWarmupGreetingWavePath(paths.leftHandPath) || hasWarmupGreetingWavePath(paths.rightHandPath) || hasWarmupGreetingWavePath(paths.primaryHandPath) ) { return 'greeting'; } } if ( stepId === 'wave_left_hand' && hasWarmupArmSwingPath(paths.leftHandPath) ) { return 'left-hand'; } if ( stepId === 'wave_right_hand' && hasWarmupArmSwingPath(paths.rightHandPath) ) { return 'right-hand'; } return null; } function getHoldProgress( stepId: ChildMotionWarmupStepId, avatarX: number, holdStartedAt: number | null, nowMs: number, ) { const step = getChildMotionWarmupStep(stepId); if (!isAvatarOnWarmupTarget(step, avatarX) || holdStartedAt === null) { return 0; } return Math.min(1, (nowMs - holdStartedAt) / CHILD_MOTION_HOLD_DURATION_MS); } function getStepIndex(stepId: ChildMotionWarmupStepId) { const order: ChildMotionWarmupStepId[] = [ 'center_arrive', 'wave_greeting', 'warmup_intro', 'move_left', 'return_center_1', 'move_right', 'return_center_2', 'wave_left_hand', 'wave_right_hand', 'warmup_finish', 'level_select', ]; return Math.max(0, order.indexOf(stepId)); } function ChildMotionAvatar({ avatarX, isJumping, }: { avatarX: number; isJumping: boolean; }) { return (
); } function ChildMotionRing({ targetX, progress, }: { targetX: number; progress: number; }) { return (
0 ? 'child-motion-ring--active' : ''}`} data-testid="child-motion-ring" style={ { left: `${targetX * 100}%`, '--child-motion-ring-progress': `${Math.round(progress * 360)}deg`, } as CSSProperties } aria-label="绿色圆环" >
); } function ChildMotionGestureGuide({ stepId, leftHandPath, rightHandPath, }: { stepId: ChildMotionWarmupStepId; leftHandPath: ChildMotionPoint[]; rightHandPath: ChildMotionPoint[]; }) { const isLeft = stepId === 'wave_left_hand'; const isRight = stepId === 'wave_right_hand'; const isGreeting = stepId === 'wave_greeting'; const activePath = isLeft ? leftHandPath : isRight ? rightHandPath : []; return ( ); } function ChildMotionHandIndicators({ hands, }: { hands: WarmupHandIndicators; }) { return (