GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

feat: connect child motion warmup to mocap input
Some checks failed
CI / verify (push) Has been cancelled
Details

Browse Source
This commit is contained in:
五香丸子
2026-05-10 21:22:50 +08:00
parent 7f2461313e
commit 54c2d6de47
6 changed files with 846 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

272
src/components/child-motion-demo/ChildMotionWarmupDemo.test.tsx
View File

@@ -1,6 +1,6 @@
/* @vitest-environment jsdom */
import { fireEvent, render, screen } from '@testing-library/react';
import { act, fireEvent, render, screen } from '@testing-library/react';
import { afterEach, beforeEach, expect, test, vi } from 'vitest';
import { ChildMotionWarmupDemo } from './ChildMotionWarmupDemo';
@@ -9,9 +9,40 @@ import {
resetChildMotionWarmupRuntimeSession,
} from './childMotionWarmupModel';
const mocapMock = vi.hoisted(() => ({
status: 'connected' as 'idle' | 'connecting' | 'connected' | 'error',
command: null as null | {
actions: string[];
hands?: Array<{ x: number; y: number; state: string; side: string }>;
primaryHand?: { x: number; y: number; state: string; side: string } | null;
leftHand?: { x: number; y: number; state: string; side: string } | null;
rightHand?: { x: number; y: number; state: string; side: string } | null;
bodyCenter?: { x: number; y: number } | null;
},
receivedAtMs: 1,
}));
vi.mock('../../services/useMocapInput', () => ({
useMocapInput: ({ enabled }: { enabled: boolean }) => ({
status: enabled ? mocapMock.status : 'idle',
latestCommand: enabled ? mocapMock.command : null,
rawPacketPreview:
enabled && mocapMock.command
? {
text: JSON.stringify(mocapMock.command),
receivedAtMs: mocapMock.receivedAtMs,
}
: null,
error: null,
}),
}));
beforeEach(() => {
resetChildMotionWarmupRuntimeSession();
vi.restoreAllMocks();
mocapMock.status = 'connected';
mocapMock.command = null;
mocapMock.receivedAtMs = 1;
Object.defineProperty(navigator, 'mediaDevices', {
configurable: true,
value: undefined,
@@ -19,6 +50,7 @@ beforeEach(() => {
});
afterEach(() => {
vi.useRealTimers();
vi.restoreAllMocks();
});
@@ -57,6 +89,241 @@ test('developer keyboard input moves the avatar and triggers jump state', () =>
expect(avatar.className).toContain('child-motion-avatar--jumping');
});
test('mocap body center keeps the warmup flow on the motion data source', async () => {
vi.useFakeTimers();
mocapMock.command = {
actions: [],
bodyCenter: { x: 0.5, y: 0.6 },
hands: [],
primaryHand: null,
leftHand: null,
rightHand: null,
};
const { rerender, unmount } = render(<ChildMotionWarmupDemo />);
expect(screen.queryByText('摄像头暂不可用，已切换到本地演示')).toBeNull();
expect(screen.queryByText('动作数据已连接，等待识别')).toBeNull();
expect(screen.getByTestId('child-motion-avatar').getAttribute('style')).toContain(
'left: 50%',
);
await act(async () => {
vi.advanceTimersByTime(2100);
await vi.runOnlyPendingTimersAsync();
});
await vi.waitFor(() => {
expect(screen.getByText('打个招呼')).toBeTruthy();
});
mocapMock.command = {
actions: ['open_palm'],
bodyCenter: { x: 0.5, y: 0.6 },
hands: [{ x: 0.48, y: 0.34, state: 'open_palm', side: 'left' }],
primaryHand: { x: 0.48, y: 0.34, state: 'open_palm', side: 'left' },
leftHand: { x: 0.48, y: 0.34, state: 'open_palm', side: 'left' },
rightHand: null,
};
mocapMock.receivedAtMs += 1;
await act(async () => {
rerender(<ChildMotionWarmupDemo />);
vi.advanceTimersByTime(1000);
await vi.runOnlyPendingTimersAsync();
});
await vi.waitFor(() => {
expect(screen.getByText('准备热身')).toBeTruthy();
});
await act(async () => {
vi.advanceTimersByTime(1000);
await vi.runOnlyPendingTimersAsync();
});
await vi.waitFor(() => {
expect(screen.getByRole('heading', { name: '向左一步' })).toBeTruthy();
});
mocapMock.command = {
actions: [],
bodyCenter: { x: 0.34, y: 0.6 },
hands: [],
primaryHand: null,
leftHand: null,
rightHand: null,
};
mocapMock.receivedAtMs += 1;
await act(async () => {
rerender(<ChildMotionWarmupDemo />);
});
await vi.waitFor(() => {
expect(screen.getByTestId('child-motion-avatar').getAttribute('style')).toContain(
'left: 34%',
);
});
await act(async () => {
vi.advanceTimersByTime(2100);
await vi.runOnlyPendingTimersAsync();
});
await vi.waitFor(() => {
expect(screen.getByRole('heading', { name: '回到中间来' })).toBeTruthy();
});
await act(async () => {
unmount();
});
vi.useRealTimers();
});
test('mocap open palm completes the greeting wave step', async () => {
vi.useFakeTimers();
const { rerender, unmount } = render(<ChildMotionWarmupDemo />);
await act(async () => {
vi.advanceTimersByTime(2100);
await vi.runOnlyPendingTimersAsync();
});
await vi.waitFor(() => {
expect(screen.getByText('打个招呼')).toBeTruthy();
});
mocapMock.command = {
actions: ['open_palm'],
hands: [{ x: 0.46, y: 0.34, state: 'open_palm', side: 'left' }],
primaryHand: { x: 0.46, y: 0.34, state: 'open_palm', side: 'left' },
leftHand: { x: 0.46, y: 0.34, state: 'open_palm', side: 'left' },
rightHand: null,
};
mocapMock.receivedAtMs += 1;
await act(async () => {
rerender(<ChildMotionWarmupDemo />);
});
await vi.waitFor(() => {
expect(screen.getByText('准备热身')).toBeTruthy();
});
await act(async () => {
unmount();
});
vi.useRealTimers();
});
test('mocap hand tracks complete left and right wave steps only after movement is visible', async () => {
vi.useFakeTimers();
const { rerender, unmount } = render(<ChildMotionWarmupDemo />);
const advancePositionStep = async (key: string, code: string) => {
await act(async () => {
fireEvent.keyDown(window, { key, code });
});
await act(async () => {
vi.advanceTimersByTime(2100);
await vi.runOnlyPendingTimersAsync();
});
await act(async () => {
fireEvent.keyUp(window, { key, code });
});
};
await act(async () => {
vi.advanceTimersByTime(2100);
await vi.runOnlyPendingTimersAsync();
});
await vi.waitFor(() => {
expect(screen.getByText('打个招呼')).toBeTruthy();
});
mocapMock.command = {
actions: ['open_palm'],
hands: [{ x: 0.48, y: 0.34, state: 'open_palm', side: 'left' }],
primaryHand: { x: 0.48, y: 0.34, state: 'open_palm', side: 'left' },
leftHand: { x: 0.48, y: 0.34, state: 'open_palm', side: 'left' },
rightHand: null,
};
mocapMock.receivedAtMs += 1;
await act(async () => {
rerender(<ChildMotionWarmupDemo />);
});
await act(async () => {
vi.advanceTimersByTime(1000);
await vi.runOnlyPendingTimersAsync();
});
await advancePositionStep('a', 'KeyA');
await act(async () => {
vi.advanceTimersByTime(120);
await vi.runOnlyPendingTimersAsync();
vi.advanceTimersByTime(2100);
await vi.runOnlyPendingTimersAsync();
});
await advancePositionStep('d', 'KeyD');
await act(async () => {
vi.advanceTimersByTime(120);
await vi.runOnlyPendingTimersAsync();
vi.advanceTimersByTime(2100);
await vi.runOnlyPendingTimersAsync();
});
await vi.waitFor(() => {
expect(screen.getByRole('heading', { name: '挥动左手' })).toBeTruthy();
});
mocapMock.command = {
actions: [],
leftHand: { x: 0.3, y: 0.38, state: 'unknown', side: 'left' },
primaryHand: { x: 0.3, y: 0.38, state: 'unknown', side: 'left' },
rightHand: null,
};
mocapMock.receivedAtMs += 1;
await act(async () => {
rerender(<ChildMotionWarmupDemo />);
});
mocapMock.command = {
actions: [],
leftHand: { x: 0.39, y: 0.36, state: 'unknown', side: 'left' },
primaryHand: { x: 0.39, y: 0.36, state: 'unknown', side: 'left' },
rightHand: null,
};
mocapMock.receivedAtMs += 1;
await act(async () => {
rerender(<ChildMotionWarmupDemo />);
});
mocapMock.command = {
actions: [],
leftHand: { x: 0.31, y: 0.34, state: 'unknown', side: 'left' },
primaryHand: { x: 0.31, y: 0.34, state: 'unknown', side: 'left' },
rightHand: null,
};
mocapMock.receivedAtMs += 1;
await act(async () => {
rerender(<ChildMotionWarmupDemo />);
});
await vi.waitFor(() => {
expect(screen.getByRole('heading', { name: '挥动右手' })).toBeTruthy();
});
mocapMock.command = {
actions: ['right_hand_wave'],
leftHand: null,
primaryHand: { x: 0.64, y: 0.35, state: 'unknown', side: 'right' },
rightHand: { x: 0.64, y: 0.35, state: 'unknown', side: 'right' },
};
mocapMock.receivedAtMs += 1;
await act(async () => {
rerender(<ChildMotionWarmupDemo />);
});
await vi.waitFor(() => {
expect(screen.getByRole('heading', { name: '原地跳一下' })).toBeTruthy();
});
await act(async () => {
vi.advanceTimersByTime(720);
await vi.runOnlyPendingTimersAsync();
unmount();
});
vi.useRealTimers();
});
test('connects camera stream and releases it on unmount', async () => {
const stopTrack = vi.fn();
const stream = {
@@ -79,7 +346,8 @@ test('connects camera stream and releases it on unmount', async () => {
const { unmount } = render(<ChildMotionWarmupDemo />);
expect(await screen.findByText('正在连接摄像头')).toBeTruthy();
expect(screen.queryByText('摄像头暂不可用，已切换到本地演示')).toBeNull();
expect(screen.getByText('动作数据已连接，等待识别')).toBeTruthy();
await vi.waitFor(() => {
expect(getUserMedia).toHaveBeenCalledWith({
audio: false,

304
src/components/child-motion-demo/ChildMotionWarmupDemo.tsx
View File

@@ -4,6 +4,12 @@ import type {
} from 'react';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import type {
MocapConnectionStatus,
MocapHandInput,
MocapInputCommand,
} from '../../services/useMocapInput';
import { useMocapInput } from '../../services/useMocapInput';
import {
applyChildMotionWarmupCompletion,
CHILD_MOTION_CENTER_X,
@@ -24,6 +30,11 @@ import {
type DragHand = 'left' | 'right';
type CameraAccessState = 'idle' | 'requesting' | 'ready' | 'blocked';
type MotionSourceState = 'connecting' | 'ready' | 'waiting' | 'offline';
type WarmupMocapGestureIntent = 'greeting' | 'left-hand' | 'right-hand' | 'jump';
const WARMUP_MOCAP_WAVE_MIN_POINTS = 3;
const WARMUP_MOCAP_WAVE_MIN_X_RANGE = 0.055;
function clampMotionUnit(value: number) {
return Math.max(0, Math.min(1, value));
@@ -50,6 +61,163 @@ function formatPercent(value: number | null) {
return `${Math.round(value * 100)}%`;
}
function mocapHandToChildMotionPoint(
hand: MocapHandInput | null | undefined,
): ChildMotionPoint | null {
if (!hand) {
return null;
}
return {
x: clampMotionUnit(hand.x),
y: clampMotionUnit(hand.y),
};
}
function appendWarmupMocapPoint(
points: ChildMotionPoint[],
point: ChildMotionPoint,
) {
return [...points, point].slice(-16);
}
function getMotionSourceState(
mocapStatus: MocapConnectionStatus,
latestCommand: MocapInputCommand | null,
): MotionSourceState {
if (mocapStatus === 'connecting' || mocapStatus === 'idle') {
return 'connecting';
}
if (mocapStatus === 'connected') {
return latestCommand &&
(Boolean(latestCommand.bodyCenter) ||
Boolean(latestCommand.hands?.length) ||
latestCommand.actions.length > 0)
? 'ready'
: 'waiting';
}
return 'offline';
}
function getMotionSourceText(state: MotionSourceState) {
if (state === 'ready') {
return '动作数据已连接';
}
if (state === 'waiting') {
return '动作数据已连接，等待识别';
}
if (state === 'offline') {
return '动作数据暂不可用，已保留本地调试';
}
return '正在连接动作数据';
}
function hasWarmupMocapAction(
command: MocapInputCommand,
expectedActions: string[],
) {
return command.actions.some((action) => expectedActions.includes(action));
}
function hasWarmupMocapWavePath(points: ChildMotionPoint[]) {
if (points.length < WARMUP_MOCAP_WAVE_MIN_POINTS) {
return false;
}
const xValues = points.map((point) => point.x);
return (
Math.max(...xValues) - Math.min(...xValues) >=
WARMUP_MOCAP_WAVE_MIN_X_RANGE
);
}
function resolveAvatarXFromMocap(command: MocapInputCommand) {
return command.bodyCenter?.x ?? null;
}
function resolveWarmupMocapGestureIntent(
stepId: ChildMotionWarmupStepId,
command: MocapInputCommand,
paths: {
leftHandPath: ChildMotionPoint[];
rightHandPath: ChildMotionPoint[];
primaryHandPath: ChildMotionPoint[];
},
): WarmupMocapGestureIntent | null {
if (stepId === 'wave_greeting') {
if (
hasWarmupMocapAction(command, [
'wave',
'wave_greeting',
'hand_wave',
'hello',
'greeting',
'open_palm',
'handwave',
'wavehand',
'招手',
'挥手',
]) ||
command.hands?.some((hand) => hand.state === 'open_palm') ||
hasWarmupMocapWavePath(paths.leftHandPath) ||
hasWarmupMocapWavePath(paths.rightHandPath) ||
hasWarmupMocapWavePath(paths.primaryHandPath)
) {
return 'greeting';
}
}
if (
stepId === 'wave_left_hand' &&
(hasWarmupMocapAction(command, [
'left_wave',
'wave_left',
'left_hand_wave',
'wave_left_hand',
'left_handwave',
'lefthand_wave',
'lefthandwave',
'左手挥手',
'挥动左手',
]) ||
hasWarmupMocapWavePath(paths.leftHandPath))
) {
return 'left-hand';
}
if (
stepId === 'wave_right_hand' &&
(hasWarmupMocapAction(command, [
'right_wave',
'wave_right',
'right_hand_wave',
'wave_right_hand',
'right_handwave',
'righthand_wave',
'righthandwave',
'右手挥手',
'挥动右手',
]) ||
hasWarmupMocapWavePath(paths.rightHandPath))
) {
return 'right-hand';
}
if (
stepId === 'jump_once' &&
hasWarmupMocapAction(command, ['jump', 'jump_once', 'hop', '跳跃', '原地跳'])
) {
return 'jump';
}
return null;
}
function getHoldProgress(
stepId: ChildMotionWarmupStepId,
avatarX: number,
@@ -230,12 +398,25 @@ export function ChildMotionWarmupDemo() {
const holdCompletionRef = useRef(false);
const cameraVideoRef = useRef<HTMLVideoElement | null>(null);
const cameraStreamRef = useRef<MediaStream | null>(null);
const handledMocapPacketKeyRef = useRef<string | null>(null);
const step = getChildMotionWarmupStep(stepId);
const mocapInput = useMocapInput({
enabled:
step.kind === 'position' ||
step.kind === 'gesture' ||
step.kind === 'narration' ||
step.kind === 'finish',
});
const stepIndex = getStepIndex(stepId);
const progressPercent = Math.round((stepIndex / 12) * 100);
const holdProgress = getHoldProgress(stepId, avatarX, holdStartedAt, nowMs);
const targetX = step.target ? getChildMotionTargetX(step.target) : null;
const motionSourceState = getMotionSourceState(
mocapInput.status,
mocapInput.latestCommand,
);
const motionSourceText = getMotionSourceText(motionSourceState);
const completeStep = useCallback(
(completion: Parameters<typeof applyChildMotionWarmupCompletion>[2]) => {
@@ -382,6 +563,115 @@ export function ChildMotionWarmupDemo() {
return () => window.clearTimeout(timeout);
}, [completeStep, step.kind]);
useEffect(() => {
if (step.kind !== 'gesture' || !mocapInput.latestCommand) {
return;
}
const command = mocapInput.latestCommand;
const packetKey =
mocapInput.rawPacketPreview?.receivedAtMs !== undefined
? `${mocapInput.rawPacketPreview.receivedAtMs}:${mocapInput.rawPacketPreview.text}`
: JSON.stringify(command);
if (handledMocapPacketKeyRef.current === packetKey) {
return;
}
const primaryPoint = mocapHandToChildMotionPoint(command.primaryHand);
const primaryHandSide = command.primaryHand?.side ?? 'unknown';
const fallbackPrimaryToLeft =
Boolean(primaryPoint) &&
!command.leftHand &&
(primaryHandSide === 'left' ||
primaryHandSide === 'unknown' ||
stepId === 'wave_left_hand' ||
stepId === 'wave_greeting');
const fallbackPrimaryToRight =
Boolean(primaryPoint) &&
!command.rightHand &&
(primaryHandSide === 'right' ||
stepId === 'wave_right_hand');
const leftPoint =
mocapHandToChildMotionPoint(command.leftHand) ??
(fallbackPrimaryToLeft ? primaryPoint : null);
const rightPoint =
mocapHandToChildMotionPoint(command.rightHand) ??
(fallbackPrimaryToRight ? primaryPoint : null);
const nextLeftHandPath = leftPoint
? appendWarmupMocapPoint(leftHandPath, leftPoint)
: leftHandPath;
const nextRightHandPath = rightPoint
? appendWarmupMocapPoint(rightHandPath, rightPoint)
: rightHandPath;
const nextPrimaryHandPath = primaryPoint
? command.primaryHand?.side === 'right'
? nextRightHandPath
: nextLeftHandPath
: [];
handledMocapPacketKeyRef.current = packetKey;
if (leftPoint) {
setLeftHandPath(nextLeftHandPath);
}
if (rightPoint) {
setRightHandPath(nextRightHandPath);
}
const intent = resolveWarmupMocapGestureIntent(stepId, command, {
leftHandPath: nextLeftHandPath,
rightHandPath: nextRightHandPath,
primaryHandPath: nextPrimaryHandPath,
});
if (!intent) {
return;
}
if (intent === 'jump') {
setIsJumping(true);
window.setTimeout(() => setIsJumping(false), 360);
completeStep({ type: 'jump', jumpSpace: 0.14 });
return;
}
if (intent === 'right-hand') {
const path = [...nextRightHandPath, rightPoint ?? primaryPoint].filter(
(point): point is ChildMotionPoint => Boolean(point),
);
completeStep({ type: 'right-hand', path: path.slice(-16) });
return;
}
const path = [...nextLeftHandPath, leftPoint ?? primaryPoint].filter(
(point): point is ChildMotionPoint => Boolean(point),
);
completeStep({ type: 'left-hand', path: path.slice(-16) });
}, [
completeStep,
leftHandPath,
mocapInput.latestCommand,
mocapInput.rawPacketPreview?.receivedAtMs,
mocapInput.rawPacketPreview?.text,
rightHandPath,
step.kind,
stepId,
]);
useEffect(() => {
if (!mocapInput.latestCommand) {
return;
}
const nextAvatarX = resolveAvatarXFromMocap(mocapInput.latestCommand);
if (nextAvatarX === null) {
return;
}
setAvatarX(nextAvatarX);
}, [
mocapInput.latestCommand,
mocapInput.rawPacketPreview?.receivedAtMs,
mocapInput.rawPacketPreview?.text,
]);
useEffect(() => {
const handleKeyDown = (event: KeyboardEvent) => {
if (event.repeat) {
@@ -521,14 +811,12 @@ export function ChildMotionWarmupDemo() {
muted
playsInline
/>
{cameraAccessState === 'requesting' ? (
<div className="child-motion-camera-state" aria-live="polite">
正在连接摄像头
</div>
) : null}
{cameraAccessState === 'blocked' ? (
<div className="child-motion-camera-state" aria-live="polite">
摄像头暂不可用，已切换到本地演示
{motionSourceState !== 'ready' ? (
<div
className={`child-motion-camera-state child-motion-camera-state--${motionSourceState}`}
aria-live="polite"
>
{motionSourceText}
</div>
) : null}
<div className="child-motion-floor" aria-hidden="true" />