GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(edutainment): refresh baby object match flow

Browse Source
This commit is contained in:
五香丸子
2026-05-16 11:29:28 +08:00
parent 49ffa6b901
commit 45daca3647
24 changed files with 6616 additions and 659 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
src/components/child-motion-demo/ChildMotionWarmupDemo.tsx
View File

@@ -1,5 +1,5 @@
import type { CSSProperties, PointerEvent as ReactPointerEvent } from 'react';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useCallback, useEffect, useRef, useState } from 'react';
import type { BabyObjectMatchDraft } from '../../../packages/shared/src/contracts/edutainmentBabyObject';
import {
@@ -40,9 +40,9 @@ type WarmupStepPhase = 'intro' | 'active' | 'complete';
type WarmupMocapGestureIntent =
| 'greeting'
| 'left-hand'
| 'right-hand'
| 'jump';
| 'right-hand';
type WarmupBodyHandSide = 'left' | 'right';
type WarmupHandIndicators = Record<DragHand, ChildMotionPoint | null>;
const CHILD_MOTION_BABY_OBJECT_DEMO_DRAFT: BabyObjectMatchDraft = {
draftId: 'child-motion-demo-baby-object-draft',
@@ -94,7 +94,9 @@ const WARMUP_GREETING_WAVE_DIRECTION_EPSILON = 0.008;
const WARMUP_GREETING_WRIST_ABOVE_ELBOW_MARGIN = 0.04;
const WARMUP_GREETING_WRIST_ABOVE_SHOULDER_MARGIN = 0.08;
const WARMUP_STEP_INTRO_DELAY_MS = 1000;
const WARMUP_SUBTITLE_LINE_DELAY_MS = 2000;
const WARMUP_STEP_COMPLETE_PAUSE_MS = 820;
const WARMUP_TOTAL_STEPS = 11;
const AVATAR_MOCAP_DEAD_ZONE = 0.012;
const AVATAR_MOCAP_SMOOTHING = 0.28;
const AVATAR_MOCAP_MAX_STEP = 0.035;
@@ -128,6 +130,13 @@ function formatAvatarLeftPercent(value: number) {
return `${Math.round(clampMotionUnit(value) * 1000) / 10}%`;
}
function createEmptyWarmupHandIndicators(): WarmupHandIndicators {
return {
left: null,
right: null,
};
}
function resolveMocapHandWithBodySide(
command: MocapInputCommand,
side: WarmupBodyHandSide,
@@ -136,6 +145,29 @@ function resolveMocapHandWithBodySide(
return side === 'left' ? command.rightHand : command.leftHand;
}
function resolveMocapSkeletonWristWithBodySide(
command: MocapInputCommand,
side: WarmupBodyHandSide,
) {
const joints = command.bodyJoints;
return side === 'left' ? joints?.rightWrist : joints?.leftWrist;
}
function resolveWarmupHandIndicatorsFromMocap(
command: MocapInputCommand,
): WarmupHandIndicators {
return {
left: mocapHandToWarmupIndicatorPoint(
resolveMocapHandWithBodySide(command, 'left'),
resolveMocapSkeletonWristWithBodySide(command, 'left'),
),
right: mocapHandToWarmupIndicatorPoint(
resolveMocapHandWithBodySide(command, 'right'),
resolveMocapSkeletonWristWithBodySide(command, 'right'),
),
};
}
function resolveMocapJointWithBodySide(
command: MocapInputCommand,
side: WarmupBodyHandSide,
@@ -175,6 +207,22 @@ function mocapHandToChildMotionPoint(
};
}
function mocapHandToWarmupIndicatorPoint(
hand: MocapHandInput | null | undefined,
skeletonWrist: MocapPointInput | null | undefined,
): ChildMotionPoint | null {
// 骨架手腕节点比手掌识别结果更稳定；热身指示器优先跟随骨架手腕。
const point = skeletonWrist ?? hand?.wrist ?? hand;
if (!point) {
return null;
}
return {
x: clampMotionUnit(point.x),
y: clampMotionUnit(point.y),
};
}
function appendWarmupMocapPoint(
points: ChildMotionPoint[],
point: ChildMotionPoint,
@@ -218,13 +266,6 @@ function getMotionSourceText(state: MotionSourceState) {
return '正在连接动作数据';
}
function hasWarmupMocapAction(
command: MocapInputCommand,
expectedActions: string[],
) {
return command.actions.some((action) => expectedActions.includes(action));
}
function countWarmupVerticalDirectionChanges(points: ChildMotionPoint[]) {
let previousDirection = 0;
let directionChanges = 0;
@@ -403,7 +444,6 @@ function resolveDampedAvatarX(current: number, target: number) {
function resolveWarmupMocapGestureIntent(
stepId: ChildMotionWarmupStepId,
command: MocapInputCommand,
paths: {
leftHandPath: ChildMotionPoint[];
rightHandPath: ChildMotionPoint[];
@@ -434,19 +474,6 @@ function resolveWarmupMocapGestureIntent(
return 'right-hand';
}
if (
stepId === 'jump_once' &&
hasWarmupMocapAction(command, [
'jump',
'jump_once',
'hop',
'跳跃',
'原地跳',
])
) {
return 'jump';
}
return null;
}
@@ -475,7 +502,6 @@ function getStepIndex(stepId: ChildMotionWarmupStepId) {
'return_center_2',
'wave_left_hand',
'wave_right_hand',
'jump_once',
'warmup_finish',
'level_select',
];
@@ -546,11 +572,15 @@ function ChildMotionGestureGuide({
const isLeft = stepId === 'wave_left_hand';
const isRight = stepId === 'wave_right_hand';
const isGreeting = stepId === 'wave_greeting';
const isJump = stepId === 'jump_once';
const activePath = isLeft ? leftHandPath : isRight ? rightHandPath : [];
return (
<div className="child-motion-gesture-guide" aria-hidden="true">
<div
className={`child-motion-gesture-guide ${
isGreeting ? 'child-motion-gesture-guide--greeting' : ''
}`}
aria-hidden="true"
>
{isGreeting ? (
<span className="child-motion-gesture-guide__wave-cat">
<span className="child-motion-gesture-guide__wave-cat-body" />
@@ -561,8 +591,14 @@ function ChildMotionGestureGuide({
{isLeft || isRight ? (
<>
<span
className={`child-motion-gesture-guide__arm child-motion-gesture-guide__arm--${isLeft ? 'left' : 'right'}`}
/>
className={`child-motion-gesture-guide__arm-swing child-motion-gesture-guide__arm-swing--${isLeft ? 'left' : 'right'}`}
data-testid={`child-motion-arm-swing-guide-${isLeft ? 'left' : 'right'}`}
>
<span className="child-motion-gesture-guide__arm-swing-track" />
<span className="child-motion-gesture-guide__arm-swing-paw">
<span className="child-motion-gesture-guide__arm-swing-paw-asset" />
</span>
</span>
{activePath.map((point, index) => (
<span
key={`${isLeft ? 'left' : 'right'}-${index}`}
@@ -576,9 +612,40 @@ function ChildMotionGestureGuide({
))}
</>
) : null}
{isJump ? (
<span className="child-motion-gesture-guide__jump">跳</span>
) : null}
</div>
);
}
function ChildMotionHandIndicators({
hands,
}: {
hands: WarmupHandIndicators;
}) {
return (
<div
className="baby-object-runtime__hands child-motion-hand-indicators"
aria-hidden="true"
>
{(['left', 'right'] as const).map((hand) => {
const point = hands[hand];
if (!point) {
return null;
}
return (
<div
key={hand}
className={`baby-object-runtime__hand baby-object-runtime__hand--${hand} child-motion-hand-indicator`}
data-testid={`child-motion-${hand}-hand-indicator`}
style={
{
'--baby-object-hand-x': `${point.x * 100}%`,
'--baby-object-hand-y': `${point.y * 100}%`,
} as CSSProperties
}
/>
);
})}
</div>
);
}
@@ -606,10 +673,6 @@ function ChildMotionCalibrationPanel({
<span>右手</span>
<strong>{calibration.rightHandPath.length}</strong>
</div>
<div>
<span>跳跃</span>
<strong>{formatPercent(calibration.jumpSpace)}</strong>
</div>
</div>
);
}
@@ -630,11 +693,15 @@ export function ChildMotionWarmupDemo() {
const [nowMs, setNowMs] = useState(() => Date.now());
const [leftHandPath, setLeftHandPath] = useState<ChildMotionPoint[]>([]);
const [rightHandPath, setRightHandPath] = useState<ChildMotionPoint[]>([]);
const [handIndicators, setHandIndicators] = useState(
createEmptyWarmupHandIndicators,
);
const [activeHand, setActiveHand] = useState<DragHand | null>(null);
const [isJumping, setIsJumping] = useState(false);
const [justCompletedText, setJustCompletedText] = useState<string | null>(
null,
);
const [subtitleLineIndex, setSubtitleLineIndex] = useState(0);
const [cameraAccessState, setCameraAccessState] = useState<CameraAccessState>(
() =>
typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia
@@ -657,7 +724,9 @@ export function ChildMotionWarmupDemo() {
step.kind === 'finish',
});
const stepIndex = getStepIndex(stepId);
const progressPercent = Math.round((stepIndex / 12) * 100);
const progressPercent = Math.round(
(stepIndex / (WARMUP_TOTAL_STEPS - 1)) * 100,
);
const holdProgress = getHoldProgress(stepId, avatarX, holdStartedAt, nowMs);
const isStepActive = stepPhase === 'active';
const shouldShowStepCues = stepPhase !== 'intro';
@@ -681,12 +750,14 @@ export function ChildMotionWarmupDemo() {
);
const nextStep = resolveNextChildMotionWarmupStep(stepId);
if (stepId === 'jump_once') {
if (stepId === 'warmup_finish') {
markChildMotionWarmupCompletedInRuntime();
}
const completionText =
stepId === 'warmup_finish' || stepId === 'jump_once' ? null : '真棒';
stepId === 'wave_greeting' || stepId === 'warmup_finish'
? null
: '真棒';
setJustCompletedText(completionText);
setStepPhase('complete');
setHoldStartedAt(null);
@@ -803,6 +874,7 @@ export function ChildMotionWarmupDemo() {
setHoldStartedAt(null);
setLeftHandPath([]);
setRightHandPath([]);
setSubtitleLineIndex(0);
handledMocapPacketKeyRef.current = null;
if (step.kind === 'levelSelect') {
@@ -819,6 +891,20 @@ export function ChildMotionWarmupDemo() {
return () => window.clearTimeout(timeout);
}, [step.kind, stepId]);
useEffect(() => {
if (step.spokenLines.length <= 1) {
return;
}
setSubtitleLineIndex(0);
const timeout = window.setTimeout(() => {
setSubtitleLineIndex((current) =>
Math.min(current + 1, step.spokenLines.length - 1),
);
}, WARMUP_SUBTITLE_LINE_DELAY_MS);
return () => window.clearTimeout(timeout);
}, [step.spokenLines, stepId]);
useEffect(() => {
if (step.kind !== 'position' || !isStepActive) {
return;
@@ -925,7 +1011,7 @@ export function ChildMotionWarmupDemo() {
setRightHandPath(nextRightHandPath);
}
const intent = resolveWarmupMocapGestureIntent(stepId, command, {
const intent = resolveWarmupMocapGestureIntent(stepId, {
leftHandPath: nextLeftHandPath,
rightHandPath: nextRightHandPath,
primaryHandPath: nextPrimaryHandPath,
@@ -934,13 +1020,6 @@ export function ChildMotionWarmupDemo() {
return;
}
if (intent === 'jump') {
setIsJumping(true);
window.setTimeout(() => setIsJumping(false), 360);
completeStep({ type: 'jump', jumpSpace: 0.14 });
return;
}
if (intent === 'right-hand') {
const path = [...nextRightHandPath, rightPoint].filter(
(point): point is ChildMotionPoint => Boolean(point),
@@ -965,6 +1044,21 @@ export function ChildMotionWarmupDemo() {
stepId,
]);
useEffect(() => {
if (stepPhase === 'complete' || !mocapInput.latestCommand) {
return;
}
setHandIndicators(
resolveWarmupHandIndicatorsFromMocap(mocapInput.latestCommand),
);
}, [
mocapInput.latestCommand,
mocapInput.rawPacketPreview?.receivedAtMs,
mocapInput.rawPacketPreview?.text,
stepPhase,
]);
useEffect(() => {
if (stepPhase === 'complete' || !mocapInput.latestCommand) {
return;
@@ -1008,15 +1102,12 @@ export function ChildMotionWarmupDemo() {
event.preventDefault();
setIsJumping(true);
window.setTimeout(() => setIsJumping(false), 360);
if (stepId === 'jump_once' && isStepActive) {
completeStep({ type: 'jump', jumpSpace: 0.14 });
}
}
};
window.addEventListener('keydown', handleKeyDown);
return () => window.removeEventListener('keydown', handleKeyDown);
}, [completeStep, isStepActive, stepId, stepPhase]);
}, [stepPhase]);
useEffect(() => {
const handleKeyUp = (event: KeyboardEvent) => {
@@ -1040,20 +1131,29 @@ export function ChildMotionWarmupDemo() {
return;
}
if (event.button !== 0 && event.button !== 2) {
if (
event.button !== 0 &&
event.button !== 2 &&
event.buttons !== 1 &&
event.buttons !== 2
) {
return;
}
event.preventDefault();
const nextHand: DragHand = event.button === 2 ? 'right' : 'left';
const nextHand: DragHand =
event.button === 2 || event.buttons === 2 ? 'right' : 'left';
setActiveHand(nextHand);
const point = normalizePointerPoint(event, event.currentTarget);
setHandIndicators((current) => ({ ...current, [nextHand]: point }));
if (nextHand === 'left') {
setLeftHandPath([point]);
} else {
setRightHandPath([point]);
}
event.currentTarget.setPointerCapture(event.pointerId);
if (typeof event.currentTarget.setPointerCapture === 'function') {
event.currentTarget.setPointerCapture(event.pointerId);
}
};
const handleStagePointerMove = (event: ReactPointerEvent<HTMLElement>) => {
@@ -1062,6 +1162,7 @@ export function ChildMotionWarmupDemo() {
}
const point = normalizePointerPoint(event, event.currentTarget);
setHandIndicators((current) => ({ ...current, [activeHand]: point }));
const appendPoint = (points: ChildMotionPoint[]) =>
[...points, point].slice(-16);
if (activeHand === 'left') {
@@ -1076,7 +1177,10 @@ export function ChildMotionWarmupDemo() {
return;
}
if (event.currentTarget.hasPointerCapture(event.pointerId)) {
if (
typeof event.currentTarget.hasPointerCapture === 'function' &&
event.currentTarget.hasPointerCapture(event.pointerId)
) {
event.currentTarget.releasePointerCapture(event.pointerId);
}
const hand = activeHand;
@@ -1110,10 +1214,8 @@ export function ChildMotionWarmupDemo() {
setIsBabyObjectRuntimeOpen(true);
};
const lineText = useMemo(
() => step.spokenLines.join('，'),
[step.spokenLines],
);
const shouldHideStepTitle = stepId === 'center_arrive';
const subtitleText = step.spokenLines[subtitleLineIndex] ?? step.spokenLines[0];
if (isBabyObjectRuntimeOpen) {
return (
@@ -1170,6 +1272,7 @@ export function ChildMotionWarmupDemo() {
rightHandPath={rightHandPath}
/>
) : null}
<ChildMotionHandIndicators hands={handIndicators} />
<ChildMotionAvatar avatarX={avatarX} isJumping={isJumping} />
{justCompletedText ? (
<div className="child-motion-floating-reward">
@@ -1178,10 +1281,16 @@ export function ChildMotionWarmupDemo() {
) : null}
<div className="child-motion-hud child-motion-hud--top">
<span className="child-motion-step-count">{`${Math.min(stepIndex + 1, 12)}/12`}</span>
<div>
<h1>{step.title}</h1>
<p>{lineText}</p>
<span className="child-motion-step-count">{`${Math.min(stepIndex + 1, WARMUP_TOTAL_STEPS)}/${WARMUP_TOTAL_STEPS}`}</span>
<div
className={
shouldHideStepTitle
? 'child-motion-hud__copy child-motion-hud__copy--subtitle-only'
: 'child-motion-hud__copy'
}
>
{shouldHideStepTitle ? null : <h1>{step.title}</h1>}
<p>{subtitleText}</p>
</div>
<span className="child-motion-progress">{progressPercent}%</span>
</div>