feat: connect child motion warmup to mocap input

2026-05-10 21:22:50 +08:00
parent 7f2461313e
commit 54c2d6de47
6 changed files with 846 additions and 55 deletions
--- a/.hermes/shared-memory/decision-log.md
+++ b/.hermes/shared-memory/decision-log.md
@@ -24,6 +24,14 @@
 - 验证方式：执行 `npm run test -- src\services\input-devices\runtimeDragInputController.test.ts`、`npm run test -- src\components\puzzle-runtime\PuzzleRuntimeShell.test.tsx`、`npm run typecheck` 和编码检查。
 - 关联文档：`docs/technical/RUNTIME_INPUT_DEVICE_ABSTRACTION_2026-05-10.md`、`docs/technical/PUZZLE_RUNTIME_FRONTEND_LOGIC_REHOME_2026-05-02.md`。

+## 2026-05-10 儿童动作热身关直接消费 mocap 手势流
+
+- 背景：儿童动作 Demo 的挥手、左右手挥动和跳跃阶段不能只依赖键鼠调试输入，否则真实硬件接入后会出现“能看到画面但动作不推进”的卡点。
+- 决策：热身关在 gesture 阶段直接接入 `useMocapInput`，通过本地 mocap WebSocket `/stream` 消费 `actions/action/gesture/gestures/event/name/type` 动作名，以及 `hands[]`、`leftHand/rightHand`、`left_hand/right_hand` 手部坐标；`wave_greeting`、`wave_left_hand`、`wave_right_hand` 和 `jump_once` 都可以由 mocap 包推进，同时保留键鼠作为本地调试兜底。
+- 影响范围：`src/services/useMocapInput.ts`、`src/components/child-motion-demo/ChildMotionWarmupDemo.tsx`、对应单测与热身关技术文档。
+- 验证方式：执行 `npx vitest run src/components/child-motion-demo/ChildMotionWarmupDemo.test.tsx src/components/child-motion-demo/childMotionWarmupModel.test.ts src/services/child-motion-demo/childMotionDebugInput.test.ts src/routing/appRoutes.test.ts`、`npx eslint ...`、`npm run typecheck`、`npm run check:encoding`，并确认 `http://127.0.0.1:3000/child-motion-demo` 与 `http://127.0.0.1:3100/healthz` 可访问。
+- 关联文档：`docs/technical/CHILD_MOTION_DEMO_WARMUP_IMPLEMENTATION_SPEC_2026-05-09.md`。
+
 ## 2026-05-09 GPT-image-2 图片生成统一迁移到 VectorEngine

 - 背景：仓库内 RPG、拼图、方洞和本地模板脚本的 GPT-image-2 生图此前依赖 APIMart 图片网关；团队要求参考 VectorEngine Apifox `api-448710071`，后续不再使用 APIMart 执行 GPT-image-2 图片生成。
--- a/.hermes/shared-memory/pitfalls.md
+++ b/.hermes/shared-memory/pitfalls.md
@@ -43,6 +43,14 @@
 - 验证：提交前检查 `git diff -- .hermes`，确认没有密钥、会话记录或个人路径敏感信息。
 - 关联：`.hermes/README.md`。

+## 儿童动作 Demo 挥手阶段不推进先查 mocap 消费链路
+
+- 现象：`/child-motion-demo` 能打开摄像头画面，但到“打个招呼”或左右手挥动阶段时，真实硬件动作无法检测通过，只能用鼠标拖拽或键盘调试继续。
+- 原因：摄像头视频流只是舞台背景；如果热身关没有消费 `useMocapInput` 的动作名和手部坐标，就不会把硬件动作转换成热身状态机完成事件。
+- 处理：确认 `src/components/child-motion-demo/ChildMotionWarmupDemo.tsx` 在 `step.kind === 'gesture'` 时启用 `useMocapInput`；确认 `src/services/useMocapInput.ts` 能解析 `/stream` 包里的 `actions/action/gesture/gestures/event/name/type`、`hands[]`、`leftHand/rightHand`、`left_hand/right_hand`、左右手标记和 `open_palm/grab` 状态。热身关应由 mocap 推进，键鼠只作为本地调试兜底。
+- 验证：运行 `npx vitest run src\components\child-motion-demo\ChildMotionWarmupDemo.test.tsx`，并在本地硬件服务启动后进入 `/child-motion-demo` 实测招手、左右手挥动和跳跃阶段。
+- 关联：`src/services/useMocapInput.ts`、`src/components/child-motion-demo/ChildMotionWarmupDemo.tsx`、`docs/technical/CHILD_MOTION_DEMO_WARMUP_IMPLEMENTATION_SPEC_2026-05-09.md`。
+
 ## GPT-image-2 不再读 APIMart 图片配置

 - 现象：配置了 `APIMART_BASE_URL` / `APIMART_API_KEY` 后，RPG、拼图或方洞的 GPT-image-2 生图仍返回缺配置，或请求体里还出现 `official_fallback` / `image_urls`。
--- a/docs/technical/CHILD_MOTION_DEMO_WARMUP_IMPLEMENTATION_SPEC_2026-05-09.md
+++ b/docs/technical/CHILD_MOTION_DEMO_WARMUP_IMPLEMENTATION_SPEC_2026-05-09.md
@@ -21,7 +21,7 @@
 9. 后续关卡使用热身记录的边界进行安全提醒和暂停恢复。
 10. 热身结束后进入关卡选择。

-当前阶段先落浏览器本地 Demo。浏览器摄像头视频流已接入舞台背景；摄像头硬件动作识别 SDK、正式动作识别接口和正式语音播报接口继续预留适配层，不阻塞前端热身流程、调试输入和页面表现骨架落地。
+当前阶段先落浏览器本地 Demo。浏览器摄像头视频流已接入舞台背景；热身动作阶段已接入本地 mocap 动作检测接口，通过 `useMocapInput` 连接 `http://127.0.0.1:8876/stream` 对应的 WebSocket 流，消费手势、左右手坐标和跳跃事件推进招手、左右手挥动与原地跳跃步骤。正式语音播报接口继续预留适配层，不阻塞前端热身流程、调试输入和页面表现骨架落地。

 ## 2. 非目标范围

@@ -654,12 +654,21 @@
 4. 鼠标右键按下并拖动映射右手轨迹。
 5. 空格键映射原地跳跃。

+当前硬件和动作检测接口接入：
+
+1. 浏览器摄像头视频流已接入舞台背景。
+2. 热身关手势阶段已通过 `src/services/useMocapInput.ts` 接入本地 mocap WebSocket `/stream`。
+3. mocap 包支持从 `actions/action/gesture/gestures/event/name/type` 读取动作名，并支持 `hands[]`、`leftHand/rightHand`、`left_hand/right_hand` 读取左右手坐标。
+4. `wave_greeting` 可由 `wave/wave_greeting/hand_wave/open_palm` 等动作或 open palm 手势完成。
+5. `wave_left_hand` 和 `wave_right_hand` 优先消费对应左右手动作名；当硬件只持续输出手部坐标时，也可以根据连续手部横向轨迹完成挥手检测。
+6. `jump_once` 消费 `jump/jump_once/hop` 等跳跃动作事件完成。
+7. 键盘 `A/D/Space` 与鼠标左右键拖拽仍保留为本地 Demo 调试兜底，不代表正式硬件口径。
+
 当前未接入但已保留边界：

-1. 浏览器摄像头视频流已接入；硬件动作识别 SDK 和正式动作识别接口暂不接入，后续通过动作输入适配层替换或并行接入调试输入。
-2. 正式语音播报接口暂不接入，当前先展示热身文案。
-3. 正式 gpt-image-2 视觉资源暂不接入，当前使用 CSS 占位表达相同位置和状态。
-4. 后续关卡安全边界暂停逻辑暂未落地，当前只完成热身记录和下一关按钮占位。
+1. 正式语音播报接口暂不接入，当前先展示热身文案。
+2. 正式 gpt-image-2 视觉资源暂不接入，当前使用 CSS 占位表达相同位置和状态。
+3. 后续关卡安全边界暂停逻辑暂未落地，当前只完成热身记录和下一关按钮占位。

 已执行的定向验证命令：

--- a/src/components/child-motion-demo/ChildMotionWarmupDemo.test.tsx
+++ b/src/components/child-motion-demo/ChildMotionWarmupDemo.test.tsx
@@ -1,6 +1,6 @@
 /* @vitest-environment jsdom */

-import { fireEvent, render, screen } from '@testing-library/react';
+import { act, fireEvent, render, screen } from '@testing-library/react';
 import { afterEach, beforeEach, expect, test, vi } from 'vitest';

 import { ChildMotionWarmupDemo } from './ChildMotionWarmupDemo';
@@ -9,9 +9,40 @@ import {
  resetChildMotionWarmupRuntimeSession,
 } from './childMotionWarmupModel';

+const mocapMock = vi.hoisted(() => ({
+  status: 'connected' as 'idle' | 'connecting' | 'connected' | 'error',
+  command: null as null | {
+    actions: string[];
+    hands?: Array<{ x: number; y: number; state: string; side: string }>;
+    primaryHand?: { x: number; y: number; state: string; side: string } | null;
+    leftHand?: { x: number; y: number; state: string; side: string } | null;
+    rightHand?: { x: number; y: number; state: string; side: string } | null;
+    bodyCenter?: { x: number; y: number } | null;
+  },
+  receivedAtMs: 1,
+}));
+
+vi.mock('../../services/useMocapInput', () => ({
+  useMocapInput: ({ enabled }: { enabled: boolean }) => ({
+    status: enabled ? mocapMock.status : 'idle',
+    latestCommand: enabled ? mocapMock.command : null,
+    rawPacketPreview:
+      enabled && mocapMock.command
+        ? {
+            text: JSON.stringify(mocapMock.command),
+            receivedAtMs: mocapMock.receivedAtMs,
+          }
+        : null,
+    error: null,
+  }),
+}));
+
 beforeEach(() => {
  resetChildMotionWarmupRuntimeSession();
  vi.restoreAllMocks();
+  mocapMock.status = 'connected';
+  mocapMock.command = null;
+  mocapMock.receivedAtMs = 1;
  Object.defineProperty(navigator, 'mediaDevices', {
    configurable: true,
    value: undefined,
@@ -19,6 +50,7 @@ beforeEach(() => {
 });

 afterEach(() => {
+  vi.useRealTimers();
  vi.restoreAllMocks();
 });

@@ -57,6 +89,241 @@ test('developer keyboard input moves the avatar and triggers jump state', () =>
  expect(avatar.className).toContain('child-motion-avatar--jumping');
 });

+test('mocap body center keeps the warmup flow on the motion data source', async () => {
+  vi.useFakeTimers();
+  mocapMock.command = {
+    actions: [],
+    bodyCenter: { x: 0.5, y: 0.6 },
+    hands: [],
+    primaryHand: null,
+    leftHand: null,
+    rightHand: null,
+  };
+  const { rerender, unmount } = render(<ChildMotionWarmupDemo />);
+
+  expect(screen.queryByText('摄像头暂不可用，已切换到本地演示')).toBeNull();
+  expect(screen.queryByText('动作数据已连接，等待识别')).toBeNull();
+  expect(screen.getByTestId('child-motion-avatar').getAttribute('style')).toContain(
+    'left: 50%',
+  );
+
+  await act(async () => {
+    vi.advanceTimersByTime(2100);
+    await vi.runOnlyPendingTimersAsync();
+  });
+  await vi.waitFor(() => {
+    expect(screen.getByText('打个招呼')).toBeTruthy();
+  });
+
+  mocapMock.command = {
+    actions: ['open_palm'],
+    bodyCenter: { x: 0.5, y: 0.6 },
+    hands: [{ x: 0.48, y: 0.34, state: 'open_palm', side: 'left' }],
+    primaryHand: { x: 0.48, y: 0.34, state: 'open_palm', side: 'left' },
+    leftHand: { x: 0.48, y: 0.34, state: 'open_palm', side: 'left' },
+    rightHand: null,
+  };
+  mocapMock.receivedAtMs += 1;
+  await act(async () => {
+    rerender(<ChildMotionWarmupDemo />);
+    vi.advanceTimersByTime(1000);
+    await vi.runOnlyPendingTimersAsync();
+  });
+
+  await vi.waitFor(() => {
+    expect(screen.getByText('准备热身')).toBeTruthy();
+  });
+
+  await act(async () => {
+    vi.advanceTimersByTime(1000);
+    await vi.runOnlyPendingTimersAsync();
+  });
+
+  await vi.waitFor(() => {
+    expect(screen.getByRole('heading', { name: '向左一步' })).toBeTruthy();
+  });
+
+  mocapMock.command = {
+    actions: [],
+    bodyCenter: { x: 0.34, y: 0.6 },
+    hands: [],
+    primaryHand: null,
+    leftHand: null,
+    rightHand: null,
+  };
+  mocapMock.receivedAtMs += 1;
+  await act(async () => {
+    rerender(<ChildMotionWarmupDemo />);
+  });
+  await vi.waitFor(() => {
+    expect(screen.getByTestId('child-motion-avatar').getAttribute('style')).toContain(
+      'left: 34%',
+    );
+  });
+  await act(async () => {
+    vi.advanceTimersByTime(2100);
+    await vi.runOnlyPendingTimersAsync();
+  });
+
+  await vi.waitFor(() => {
+    expect(screen.getByRole('heading', { name: '回到中间来' })).toBeTruthy();
+  });
+
+  await act(async () => {
+    unmount();
+  });
+  vi.useRealTimers();
+});
+
+test('mocap open palm completes the greeting wave step', async () => {
+  vi.useFakeTimers();
+  const { rerender, unmount } = render(<ChildMotionWarmupDemo />);
+
+  await act(async () => {
+    vi.advanceTimersByTime(2100);
+    await vi.runOnlyPendingTimersAsync();
+  });
+  await vi.waitFor(() => {
+    expect(screen.getByText('打个招呼')).toBeTruthy();
+  });
+
+  mocapMock.command = {
+    actions: ['open_palm'],
+    hands: [{ x: 0.46, y: 0.34, state: 'open_palm', side: 'left' }],
+    primaryHand: { x: 0.46, y: 0.34, state: 'open_palm', side: 'left' },
+    leftHand: { x: 0.46, y: 0.34, state: 'open_palm', side: 'left' },
+    rightHand: null,
+  };
+  mocapMock.receivedAtMs += 1;
+  await act(async () => {
+    rerender(<ChildMotionWarmupDemo />);
+  });
+
+  await vi.waitFor(() => {
+    expect(screen.getByText('准备热身')).toBeTruthy();
+  });
+  await act(async () => {
+    unmount();
+  });
+  vi.useRealTimers();
+});
+
+test('mocap hand tracks complete left and right wave steps only after movement is visible', async () => {
+  vi.useFakeTimers();
+  const { rerender, unmount } = render(<ChildMotionWarmupDemo />);
+
+  const advancePositionStep = async (key: string, code: string) => {
+    await act(async () => {
+      fireEvent.keyDown(window, { key, code });
+    });
+    await act(async () => {
+      vi.advanceTimersByTime(2100);
+      await vi.runOnlyPendingTimersAsync();
+    });
+    await act(async () => {
+      fireEvent.keyUp(window, { key, code });
+    });
+  };
+
+  await act(async () => {
+    vi.advanceTimersByTime(2100);
+    await vi.runOnlyPendingTimersAsync();
+  });
+  await vi.waitFor(() => {
+    expect(screen.getByText('打个招呼')).toBeTruthy();
+  });
+
+  mocapMock.command = {
+    actions: ['open_palm'],
+    hands: [{ x: 0.48, y: 0.34, state: 'open_palm', side: 'left' }],
+    primaryHand: { x: 0.48, y: 0.34, state: 'open_palm', side: 'left' },
+    leftHand: { x: 0.48, y: 0.34, state: 'open_palm', side: 'left' },
+    rightHand: null,
+  };
+  mocapMock.receivedAtMs += 1;
+  await act(async () => {
+    rerender(<ChildMotionWarmupDemo />);
+  });
+
+  await act(async () => {
+    vi.advanceTimersByTime(1000);
+    await vi.runOnlyPendingTimersAsync();
+  });
+  await advancePositionStep('a', 'KeyA');
+  await act(async () => {
+    vi.advanceTimersByTime(120);
+    await vi.runOnlyPendingTimersAsync();
+    vi.advanceTimersByTime(2100);
+    await vi.runOnlyPendingTimersAsync();
+  });
+  await advancePositionStep('d', 'KeyD');
+  await act(async () => {
+    vi.advanceTimersByTime(120);
+    await vi.runOnlyPendingTimersAsync();
+    vi.advanceTimersByTime(2100);
+    await vi.runOnlyPendingTimersAsync();
+  });
+  await vi.waitFor(() => {
+    expect(screen.getByRole('heading', { name: '挥动左手' })).toBeTruthy();
+  });
+
+  mocapMock.command = {
+    actions: [],
+    leftHand: { x: 0.3, y: 0.38, state: 'unknown', side: 'left' },
+    primaryHand: { x: 0.3, y: 0.38, state: 'unknown', side: 'left' },
+    rightHand: null,
+  };
+  mocapMock.receivedAtMs += 1;
+  await act(async () => {
+    rerender(<ChildMotionWarmupDemo />);
+  });
+  mocapMock.command = {
+    actions: [],
+    leftHand: { x: 0.39, y: 0.36, state: 'unknown', side: 'left' },
+    primaryHand: { x: 0.39, y: 0.36, state: 'unknown', side: 'left' },
+    rightHand: null,
+  };
+  mocapMock.receivedAtMs += 1;
+  await act(async () => {
+    rerender(<ChildMotionWarmupDemo />);
+  });
+  mocapMock.command = {
+    actions: [],
+    leftHand: { x: 0.31, y: 0.34, state: 'unknown', side: 'left' },
+    primaryHand: { x: 0.31, y: 0.34, state: 'unknown', side: 'left' },
+    rightHand: null,
+  };
+  mocapMock.receivedAtMs += 1;
+  await act(async () => {
+    rerender(<ChildMotionWarmupDemo />);
+  });
+
+  await vi.waitFor(() => {
+    expect(screen.getByRole('heading', { name: '挥动右手' })).toBeTruthy();
+  });
+
+  mocapMock.command = {
+    actions: ['right_hand_wave'],
+    leftHand: null,
+    primaryHand: { x: 0.64, y: 0.35, state: 'unknown', side: 'right' },
+    rightHand: { x: 0.64, y: 0.35, state: 'unknown', side: 'right' },
+  };
+  mocapMock.receivedAtMs += 1;
+  await act(async () => {
+    rerender(<ChildMotionWarmupDemo />);
+  });
+
+  await vi.waitFor(() => {
+    expect(screen.getByRole('heading', { name: '原地跳一下' })).toBeTruthy();
+  });
+  await act(async () => {
+    vi.advanceTimersByTime(720);
+    await vi.runOnlyPendingTimersAsync();
+    unmount();
+  });
+  vi.useRealTimers();
+});
+
 test('connects camera stream and releases it on unmount', async () => {
  const stopTrack = vi.fn();
  const stream = {
@@ -79,7 +346,8 @@ test('connects camera stream and releases it on unmount', async () => {

  const { unmount } = render(<ChildMotionWarmupDemo />);

-  expect(await screen.findByText('正在连接摄像头')).toBeTruthy();
+  expect(screen.queryByText('摄像头暂不可用，已切换到本地演示')).toBeNull();
+  expect(screen.getByText('动作数据已连接，等待识别')).toBeTruthy();
  await vi.waitFor(() => {
    expect(getUserMedia).toHaveBeenCalledWith({
      audio: false,
--- a/src/components/child-motion-demo/ChildMotionWarmupDemo.tsx
+++ b/src/components/child-motion-demo/ChildMotionWarmupDemo.tsx
@@ -4,6 +4,12 @@ import type {
 } from 'react';
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react';

+import type {
+  MocapConnectionStatus,
+  MocapHandInput,
+  MocapInputCommand,
+} from '../../services/useMocapInput';
+import { useMocapInput } from '../../services/useMocapInput';
 import {
  applyChildMotionWarmupCompletion,
  CHILD_MOTION_CENTER_X,
@@ -24,6 +30,11 @@ import {

 type DragHand = 'left' | 'right';
 type CameraAccessState = 'idle' | 'requesting' | 'ready' | 'blocked';
+type MotionSourceState = 'connecting' | 'ready' | 'waiting' | 'offline';
+type WarmupMocapGestureIntent = 'greeting' | 'left-hand' | 'right-hand' | 'jump';
+
+const WARMUP_MOCAP_WAVE_MIN_POINTS = 3;
+const WARMUP_MOCAP_WAVE_MIN_X_RANGE = 0.055;

 function clampMotionUnit(value: number) {
  return Math.max(0, Math.min(1, value));
@@ -50,6 +61,163 @@ function formatPercent(value: number | null) {
  return `${Math.round(value * 100)}%`;
 }

+function mocapHandToChildMotionPoint(
+  hand: MocapHandInput | null | undefined,
+): ChildMotionPoint | null {
+  if (!hand) {
+    return null;
+  }
+
+  return {
+    x: clampMotionUnit(hand.x),
+    y: clampMotionUnit(hand.y),
+  };
+}
+
+function appendWarmupMocapPoint(
+  points: ChildMotionPoint[],
+  point: ChildMotionPoint,
+) {
+  return [...points, point].slice(-16);
+}
+
+function getMotionSourceState(
+  mocapStatus: MocapConnectionStatus,
+  latestCommand: MocapInputCommand | null,
+): MotionSourceState {
+  if (mocapStatus === 'connecting' || mocapStatus === 'idle') {
+    return 'connecting';
+  }
+
+  if (mocapStatus === 'connected') {
+    return latestCommand &&
+      (Boolean(latestCommand.bodyCenter) ||
+        Boolean(latestCommand.hands?.length) ||
+        latestCommand.actions.length > 0)
+      ? 'ready'
+      : 'waiting';
+  }
+
+  return 'offline';
+}
+
+function getMotionSourceText(state: MotionSourceState) {
+  if (state === 'ready') {
+    return '动作数据已连接';
+  }
+
+  if (state === 'waiting') {
+    return '动作数据已连接，等待识别';
+  }
+
+  if (state === 'offline') {
+    return '动作数据暂不可用，已保留本地调试';
+  }
+
+  return '正在连接动作数据';
+}
+
+function hasWarmupMocapAction(
+  command: MocapInputCommand,
+  expectedActions: string[],
+) {
+  return command.actions.some((action) => expectedActions.includes(action));
+}
+
+function hasWarmupMocapWavePath(points: ChildMotionPoint[]) {
+  if (points.length < WARMUP_MOCAP_WAVE_MIN_POINTS) {
+    return false;
+  }
+
+  const xValues = points.map((point) => point.x);
+  return (
+    Math.max(...xValues) - Math.min(...xValues) >=
+    WARMUP_MOCAP_WAVE_MIN_X_RANGE
+  );
+}
+
+function resolveAvatarXFromMocap(command: MocapInputCommand) {
+  return command.bodyCenter?.x ?? null;
+}
+
+function resolveWarmupMocapGestureIntent(
+  stepId: ChildMotionWarmupStepId,
+  command: MocapInputCommand,
+  paths: {
+    leftHandPath: ChildMotionPoint[];
+    rightHandPath: ChildMotionPoint[];
+    primaryHandPath: ChildMotionPoint[];
+  },
+): WarmupMocapGestureIntent | null {
+  if (stepId === 'wave_greeting') {
+    if (
+      hasWarmupMocapAction(command, [
+        'wave',
+        'wave_greeting',
+        'hand_wave',
+        'hello',
+        'greeting',
+        'open_palm',
+        'handwave',
+        'wavehand',
+        '招手',
+        '挥手',
+      ]) ||
+      command.hands?.some((hand) => hand.state === 'open_palm') ||
+      hasWarmupMocapWavePath(paths.leftHandPath) ||
+      hasWarmupMocapWavePath(paths.rightHandPath) ||
+      hasWarmupMocapWavePath(paths.primaryHandPath)
+    ) {
+      return 'greeting';
+    }
+  }
+
+  if (
+    stepId === 'wave_left_hand' &&
+    (hasWarmupMocapAction(command, [
+      'left_wave',
+      'wave_left',
+      'left_hand_wave',
+      'wave_left_hand',
+      'left_handwave',
+      'lefthand_wave',
+      'lefthandwave',
+      '左手挥手',
+      '挥动左手',
+    ]) ||
+      hasWarmupMocapWavePath(paths.leftHandPath))
+  ) {
+    return 'left-hand';
+  }
+
+  if (
+    stepId === 'wave_right_hand' &&
+    (hasWarmupMocapAction(command, [
+      'right_wave',
+      'wave_right',
+      'right_hand_wave',
+      'wave_right_hand',
+      'right_handwave',
+      'righthand_wave',
+      'righthandwave',
+      '右手挥手',
+      '挥动右手',
+    ]) ||
+      hasWarmupMocapWavePath(paths.rightHandPath))
+  ) {
+    return 'right-hand';
+  }
+
+  if (
+    stepId === 'jump_once' &&
+    hasWarmupMocapAction(command, ['jump', 'jump_once', 'hop', '跳跃', '原地跳'])
+  ) {
+    return 'jump';
+  }
+
+  return null;
+}
+
 function getHoldProgress(
  stepId: ChildMotionWarmupStepId,
  avatarX: number,
@@ -230,12 +398,25 @@ export function ChildMotionWarmupDemo() {
  const holdCompletionRef = useRef(false);
  const cameraVideoRef = useRef<HTMLVideoElement | null>(null);
  const cameraStreamRef = useRef<MediaStream | null>(null);
+  const handledMocapPacketKeyRef = useRef<string | null>(null);

  const step = getChildMotionWarmupStep(stepId);
+  const mocapInput = useMocapInput({
+    enabled:
+      step.kind === 'position' ||
+      step.kind === 'gesture' ||
+      step.kind === 'narration' ||
+      step.kind === 'finish',
+  });
  const stepIndex = getStepIndex(stepId);
  const progressPercent = Math.round((stepIndex / 12) * 100);
  const holdProgress = getHoldProgress(stepId, avatarX, holdStartedAt, nowMs);
  const targetX = step.target ? getChildMotionTargetX(step.target) : null;
+  const motionSourceState = getMotionSourceState(
+    mocapInput.status,
+    mocapInput.latestCommand,
+  );
+  const motionSourceText = getMotionSourceText(motionSourceState);

  const completeStep = useCallback(
    (completion: Parameters<typeof applyChildMotionWarmupCompletion>[2]) => {
@@ -382,6 +563,115 @@ export function ChildMotionWarmupDemo() {
    return () => window.clearTimeout(timeout);
  }, [completeStep, step.kind]);

+  useEffect(() => {
+    if (step.kind !== 'gesture' || !mocapInput.latestCommand) {
+      return;
+    }
+
+    const command = mocapInput.latestCommand;
+    const packetKey =
+      mocapInput.rawPacketPreview?.receivedAtMs !== undefined
+        ? `${mocapInput.rawPacketPreview.receivedAtMs}:${mocapInput.rawPacketPreview.text}`
+        : JSON.stringify(command);
+    if (handledMocapPacketKeyRef.current === packetKey) {
+      return;
+    }
+
+    const primaryPoint = mocapHandToChildMotionPoint(command.primaryHand);
+    const primaryHandSide = command.primaryHand?.side ?? 'unknown';
+    const fallbackPrimaryToLeft =
+      Boolean(primaryPoint) &&
+      !command.leftHand &&
+      (primaryHandSide === 'left' ||
+        primaryHandSide === 'unknown' ||
+        stepId === 'wave_left_hand' ||
+        stepId === 'wave_greeting');
+    const fallbackPrimaryToRight =
+      Boolean(primaryPoint) &&
+      !command.rightHand &&
+      (primaryHandSide === 'right' ||
+        stepId === 'wave_right_hand');
+    const leftPoint =
+      mocapHandToChildMotionPoint(command.leftHand) ??
+      (fallbackPrimaryToLeft ? primaryPoint : null);
+    const rightPoint =
+      mocapHandToChildMotionPoint(command.rightHand) ??
+      (fallbackPrimaryToRight ? primaryPoint : null);
+    const nextLeftHandPath = leftPoint
+      ? appendWarmupMocapPoint(leftHandPath, leftPoint)
+      : leftHandPath;
+    const nextRightHandPath = rightPoint
+      ? appendWarmupMocapPoint(rightHandPath, rightPoint)
+      : rightHandPath;
+    const nextPrimaryHandPath = primaryPoint
+      ? command.primaryHand?.side === 'right'
+        ? nextRightHandPath
+        : nextLeftHandPath
+      : [];
+    handledMocapPacketKeyRef.current = packetKey;
+    if (leftPoint) {
+      setLeftHandPath(nextLeftHandPath);
+    }
+    if (rightPoint) {
+      setRightHandPath(nextRightHandPath);
+    }
+
+    const intent = resolveWarmupMocapGestureIntent(stepId, command, {
+      leftHandPath: nextLeftHandPath,
+      rightHandPath: nextRightHandPath,
+      primaryHandPath: nextPrimaryHandPath,
+    });
+    if (!intent) {
+      return;
+    }
+
+    if (intent === 'jump') {
+      setIsJumping(true);
+      window.setTimeout(() => setIsJumping(false), 360);
+      completeStep({ type: 'jump', jumpSpace: 0.14 });
+      return;
+    }
+
+    if (intent === 'right-hand') {
+      const path = [...nextRightHandPath, rightPoint ?? primaryPoint].filter(
+        (point): point is ChildMotionPoint => Boolean(point),
+      );
+      completeStep({ type: 'right-hand', path: path.slice(-16) });
+      return;
+    }
+
+    const path = [...nextLeftHandPath, leftPoint ?? primaryPoint].filter(
+      (point): point is ChildMotionPoint => Boolean(point),
+    );
+    completeStep({ type: 'left-hand', path: path.slice(-16) });
+  }, [
+    completeStep,
+    leftHandPath,
+    mocapInput.latestCommand,
+    mocapInput.rawPacketPreview?.receivedAtMs,
+    mocapInput.rawPacketPreview?.text,
+    rightHandPath,
+    step.kind,
+    stepId,
+  ]);
+
+  useEffect(() => {
+    if (!mocapInput.latestCommand) {
+      return;
+    }
+
+    const nextAvatarX = resolveAvatarXFromMocap(mocapInput.latestCommand);
+    if (nextAvatarX === null) {
+      return;
+    }
+
+    setAvatarX(nextAvatarX);
+  }, [
+    mocapInput.latestCommand,
+    mocapInput.rawPacketPreview?.receivedAtMs,
+    mocapInput.rawPacketPreview?.text,
+  ]);
+
  useEffect(() => {
    const handleKeyDown = (event: KeyboardEvent) => {
      if (event.repeat) {
@@ -521,14 +811,12 @@ export function ChildMotionWarmupDemo() {
          muted
          playsInline
        />
-        {cameraAccessState === 'requesting' ? (
-          <div className="child-motion-camera-state" aria-live="polite">
-            正在连接摄像头
-          </div>
-        ) : null}
-        {cameraAccessState === 'blocked' ? (
-          <div className="child-motion-camera-state" aria-live="polite">
-            摄像头暂不可用，已切换到本地演示
+        {motionSourceState !== 'ready' ? (
+          <div
+            className={`child-motion-camera-state child-motion-camera-state--${motionSourceState}`}
+            aria-live="polite"
+          >
+            {motionSourceText}
          </div>
        ) : null}
        <div className="child-motion-floor" aria-hidden="true" />
--- a/src/services/useMocapInput.ts
+++ b/src/services/useMocapInput.ts
@@ -3,15 +3,29 @@ import {useEffect, useMemo, useRef, useState} from 'react';
 export type MocapConnectionStatus = 'idle' | 'connecting' | 'connected' | 'error';

 export type MocapHandState = 'open_palm' | 'grab' | 'unknown';
+export type MocapHandSide = 'left' | 'right' | 'unknown';
+export type MocapHandSource = 'palm_center' | 'direct' | 'landmark';
+
+export type MocapHandInput = {
+  x: number;
+  y: number;
+  state: MocapHandState;
+  side: MocapHandSide;
+  source?: MocapHandSource;
+};
+
+export type MocapBodyCenterInput = {
+  x: number;
+  y: number;
+};

 export type MocapInputCommand = {
  actions: string[];
-  primaryHand?: {
-    x: number;
-    y: number;
-    state: MocapHandState;
-    source?: 'palm_center' | 'direct' | 'landmark';
-  } | null;
+  hands?: MocapHandInput[];
+  primaryHand?: MocapHandInput | null;
+  leftHand?: MocapHandInput | null;
+  rightHand?: MocapHandInput | null;
+  bodyCenter?: MocapBodyCenterInput | null;
  parseWarnings?: string[];
 };

@@ -33,13 +47,6 @@ export type UseMocapInputOptions = {
  reconnectDelayMs?: number;
 };

-type MocapParsedHand = {
-  x: number;
-  y: number;
-  state: MocapHandState;
-  source: 'palm_center' | 'direct' | 'landmark';
-};
-
 type MocapLandmarkRecord = Record<string, unknown>;

 const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876';
@@ -75,18 +82,42 @@ function normalizeCoordinate(value: unknown) {
  return Math.min(1, Math.max(0, numericValue));
 }

+function resolveNormalizedPoint(value: unknown) {
+  if (Array.isArray(value)) {
+    const x = normalizeCoordinate(value[0]);
+    const y = normalizeCoordinate(value[1]);
+    if (x === null || y === null) {
+      return null;
+    }
+    return {x, y};
+  }
+
+  if (!value || typeof value !== 'object') {
+    return null;
+  }
+
+  const pointRecord = value as {x?: unknown; y?: unknown};
+  const x = normalizeCoordinate(pointRecord.x);
+  const y = normalizeCoordinate(pointRecord.y);
+  if (x === null || y === null) {
+    return null;
+  }
+
+  return {x, y};
+}
+
 function resolveLandmarkCoordinate(landmark: MocapLandmarkRecord | undefined) {
  const x = normalizeCoordinate(landmark?.x);
  const y = normalizeCoordinate(landmark?.y);
  if (x === null || y === null) {
    return null;
  }
-  return { x, y };
+  return {x, y};
 }

 export function resolveMocapPalmCenter(
  landmarks: Array<MocapLandmarkRecord>,
-): { x: number; y: number } | null {
+): {x: number; y: number} | null {
  const landmarksByName = new Map(
    landmarks
      .filter((landmark) => typeof landmark?.name === 'string')
@@ -94,8 +125,10 @@ export function resolveMocapPalmCenter(
  );
  const weightedPoints = PALM_CENTER_WEIGHTS.map(([name, weight]) => {
    const point = resolveLandmarkCoordinate(landmarksByName.get(name));
-    return point ? { ...point, weight: Number(weight) } : null;
-  }).filter((point): point is { x: number; y: number; weight: number } => Boolean(point));
+    return point ? {...point, weight: Number(weight)} : null;
+  }).filter((point): point is {x: number; y: number; weight: number} =>
+    Boolean(point),
+  );

  if (weightedPoints.length < MIN_PALM_CENTER_POINT_COUNT) {
    return null;
@@ -112,56 +145,213 @@ export function resolveMocapPalmCenter(
  };
 }

-function resolvePrimaryHand(hands: unknown): MocapParsedHand | null {
-  if (!Array.isArray(hands)) {
+function normaliseHandSide(side: unknown): MocapHandSide {
+  if (typeof side !== 'string') {
+    return 'unknown';
+  }
+
+  const normalized = side.trim().toLocaleLowerCase('en-US');
+  if (
+    normalized === 'left' ||
+    normalized === 'l' ||
+    normalized === 'left hand' ||
+    normalized === 'left_hand' ||
+    normalized === 'left-hand' ||
+    normalized === '左' ||
+    normalized === '左手'
+  ) {
+    return 'left';
+  }
+
+  if (
+    normalized === 'right' ||
+    normalized === 'r' ||
+    normalized === 'right hand' ||
+    normalized === 'right_hand' ||
+    normalized === 'right-hand' ||
+    normalized === '右' ||
+    normalized === '右手'
+  ) {
+    return 'right';
+  }
+
+  return 'unknown';
+}
+
+function normalizeMocapAction(value: unknown) {
+  if (typeof value !== 'string') {
    return null;
  }

-  for (const hand of hands) {
-    const parsedHand = resolveHandLike(hand);
-    if (parsedHand) {
-      return parsedHand;
-    }
-  }
+  const normalized = value
+    .trim()
+    .toLocaleLowerCase('en-US')
+    .replace(/\s+/gu, '_')
+    .replace(/-/gu, '_');

-  return null;
+  return normalized || null;
 }

-function resolveHandLike(record: unknown): MocapParsedHand | null {
+function addMocapActions(actions: Set<string>, value: unknown) {
+  if (Array.isArray(value)) {
+    value.forEach((item) => addMocapActions(actions, item));
+    return;
+  }
+
+  if (value && typeof value === 'object') {
+    const actionRecord = value as Record<string, unknown>;
+    addMocapActions(actions, actionRecord.action);
+    addMocapActions(actions, actionRecord.actions);
+    addMocapActions(actions, actionRecord.gesture);
+    addMocapActions(actions, actionRecord.gestures);
+    addMocapActions(actions, actionRecord.event);
+    addMocapActions(actions, actionRecord.name);
+    addMocapActions(actions, actionRecord.type);
+    return;
+  }
+
+  const normalized = normalizeMocapAction(value);
+  if (normalized) {
+    actions.add(normalized);
+  }
+}
+
+function normaliseHandState(state: unknown): MocapHandState {
+  if (typeof state !== 'string') {
+    return 'unknown';
+  }
+
+  const normalized = state
+    .trim()
+    .toLocaleLowerCase('en-US')
+    .replace(/\s+/gu, '_')
+    .replace(/-/gu, '_');
+
+  if (
+    normalized === 'grab' ||
+    normalized === 'grabbing' ||
+    normalized === 'close' ||
+    normalized === 'fist' ||
+    normalized === 'closed_fist' ||
+    normalized === 'closed'
+  ) {
+    return 'grab';
+  }
+
+  if (
+    normalized === 'open_palm' ||
+    normalized === 'open_palm_up' ||
+    normalized === 'open' ||
+    normalized === 'palm' ||
+    normalized === 'hand_open'
+  ) {
+    return 'open_palm';
+  }
+
+  return 'unknown';
+}
+
+function resolveMocapHand(record: unknown, fallbackSide: MocapHandSide) {
  if (!record || typeof record !== 'object') {
    return null;
  }

-  const handRecord = record as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
+  const handRecord = record as {
+    state?: unknown;
+    landmarks?: unknown;
+    x?: unknown;
+    y?: unknown;
+    side?: unknown;
+    handedness?: unknown;
+    label?: unknown;
+    hand?: unknown;
+  };
  const state = normaliseHandState(handRecord.state);
+  const detectedSide = normaliseHandSide(
+    handRecord.side ??
+      handRecord.handedness ??
+      handRecord.label ??
+      handRecord.hand,
+  );
+  const side = detectedSide === 'unknown' ? fallbackSide : detectedSide;
+
  if (Array.isArray(handRecord.landmarks)) {
    const landmarks = handRecord.landmarks as Array<MocapLandmarkRecord>;
    const palmCenter = resolveMocapPalmCenter(landmarks);
    if (palmCenter) {
-      return { ...palmCenter, state, source: 'palm_center' };
+      return {...palmCenter, state, side, source: 'palm_center' as const};
    }

-    const landmark = landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
+    const landmark =
+      landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
    const fallbackPoint = resolveLandmarkCoordinate(landmark);
    if (fallbackPoint) {
-      return { ...fallbackPoint, state, source: 'landmark' };
+      return {...fallbackPoint, state, side, source: 'landmark' as const};
    }
  }

  const directX = normalizeCoordinate(handRecord.x);
  const directY = normalizeCoordinate(handRecord.y);
  if (directX !== null && directY !== null) {
-    return {x: directX, y: directY, state, source: 'direct'};
+    return {x: directX, y: directY, state, side, source: 'direct' as const};
  }

  return null;
 }

-function normaliseHandState(state: unknown): MocapHandState {
-  if (state === 'grab' || state === 'open_palm') {
-    return state;
+function resolveHands(packetRecord: Record<string, unknown>) {
+  const resolvedHands: MocapHandInput[] = [];
+  const packetHands = packetRecord.hands;
+  if (!Array.isArray(packetHands)) {
+    const leftHand = resolveMocapHand(
+      packetRecord.leftHand ?? packetRecord.left_hand,
+      'left',
+    );
+    const rightHand = resolveMocapHand(
+      packetRecord.rightHand ?? packetRecord.right_hand,
+      'right',
+    );
+    if (leftHand) {
+      resolvedHands.push(leftHand);
+    }
+    if (rightHand) {
+      resolvedHands.push(rightHand);
+    }
+    return resolvedHands;
  }
-  return 'unknown';
+
+  for (const hand of packetHands) {
+    const resolvedHand = resolveMocapHand(hand, 'unknown');
+    if (resolvedHand) {
+      resolvedHands.push(resolvedHand);
+    }
+  }
+
+  return resolvedHands;
+}
+
+function resolveBodyCenter(packetRecord: Record<string, unknown>) {
+  const generalRecord =
+    packetRecord.general && typeof packetRecord.general === 'object'
+      ? (packetRecord.general as Record<string, unknown>)
+      : null;
+  const bodyCandidates = [generalRecord?.body, packetRecord.body];
+
+  for (const bodyCandidate of bodyCandidates) {
+    if (!bodyCandidate || typeof bodyCandidate !== 'object') {
+      continue;
+    }
+
+    const bodyRecord = bodyCandidate as Record<string, unknown>;
+    const center = resolveNormalizedPoint(
+      bodyRecord.center_norm ?? bodyRecord.centerNorm ?? bodyRecord.center,
+    );
+    if (center) {
+      return center;
+    }
+  }
+
+  return null;
 }

 export function parseMocapPacket(packet: unknown): MocapInputCommand {
@@ -169,13 +359,24 @@ export function parseMocapPacket(packet: unknown): MocapInputCommand {
    return {actions: [], parseWarnings: ['packet 不是对象']};
  }

-  const packetRecord = packet as {hands?: unknown};
-  const primaryHand = resolvePrimaryHand(packetRecord.hands);
+  const packetRecord = packet as Record<string, unknown>;
+  const hands = resolveHands(packetRecord);
+  const primaryHand = hands[0] ?? null;
+  const leftHand = hands.find((hand) => hand.side === 'left') ?? null;
+  const rightHand = hands.find((hand) => hand.side === 'right') ?? null;
+  const bodyCenter = resolveBodyCenter(packetRecord);
  const actions = new Set<string>();
  const parseWarnings: string[] = [];
-  if (!Array.isArray(packetRecord.hands)) {
+  addMocapActions(actions, packetRecord.actions);
+  addMocapActions(actions, packetRecord.action);
+  addMocapActions(actions, packetRecord.gesture);
+  addMocapActions(actions, packetRecord.gestures);
+  addMocapActions(actions, packetRecord.event);
+  addMocapActions(actions, packetRecord.name);
+  addMocapActions(actions, packetRecord.type);
+  if (!Array.isArray(packetRecord.hands) && hands.length === 0 && !bodyCenter) {
    parseWarnings.push('缺少 hands 数组');
-  } else if (!primaryHand) {
+  } else if (!primaryHand && !bodyCenter) {
    parseWarnings.push('hands 中没有可用坐标');
  }
  if (primaryHand?.state === 'grab') {
@@ -184,13 +385,22 @@ export function parseMocapPacket(packet: unknown): MocapInputCommand {
  if (primaryHand?.state === 'open_palm') {
    actions.add('open_palm');
  }
+  for (const hand of hands) {
+    if (hand.state !== 'unknown') {
+      actions.add(hand.state);
+    }
+  }
  if (primaryHand && primaryHand.state === 'unknown') {
    parseWarnings.push('手势 state 未识别');
  }

  return {
    actions: Array.from(actions),
+    hands,
    primaryHand,
+    leftHand,
+    rightHand,
+    bodyCenter,
    parseWarnings,
  };
 }