feat: smooth mocap palm cursor

2026-05-10 18:51:43 +08:00
parent 46a254f142
commit 75bca28191
5 changed files with 256 additions and 52 deletions
--- a/src/components/puzzle-runtime/PuzzleRuntimeShell.test.tsx
+++ b/src/components/puzzle-runtime/PuzzleRuntimeShell.test.tsx
@@ -36,7 +36,7 @@ vi.mock('../../services/useMocapInput', () => ({
    status: 'connected',
    latestCommand: {
      actions: [mocapMock.state],
-      primaryHand: {x: mocapMock.x, y: mocapMock.y, state: mocapMock.state},
+      primaryHand: {x: mocapMock.x, y: mocapMock.y, state: mocapMock.state, source: 'palm_center'},
      parseWarnings: [],
    },
    rawPacketPreview: {text: '{"hands":[{"state":"grab"}]}', receivedAtMs: 1},
--- a/src/components/puzzle-runtime/PuzzleRuntimeShell.tsx
+++ b/src/components/puzzle-runtime/PuzzleRuntimeShell.tsx
@@ -221,6 +221,7 @@ const PUZZLE_PIECE_PRESS_HAPTIC_PATTERN_MS = 12;
 const PUZZLE_EXIT_REMODEL_PROMPT_STORAGE_PREFIX =
  'genarrative.puzzle-runtime.exit-remodel-prompt.v1';
 const PUZZLE_MOCAP_DRAG_INPUT_ID = 'mocap:primary-hand';
+const PUZZLE_MOCAP_CURSOR_FRAME_MS = 1000 / 60;

 const shownExitRemodelPromptProfileIds = new Set<string>();

@@ -300,6 +301,10 @@ type PuzzleMocapCursorState = {
  state: string;
 };

+type PuzzleMocapCursorSample = PuzzleMocapCursorState & {
+  receivedAtMs: number;
+};
+
 type PuzzleRuntimeDragTargetState = {
  pieceId: string;
  groupId: string | null;
@@ -394,6 +399,14 @@ export function PuzzleRuntimeShell({
  const [mocapCursor, setMocapCursor] = useState<PuzzleMocapCursorState | null>(
    null,
  );
+  const mocapCursorPreviousSampleRef = useRef<PuzzleMocapCursorSample | null>(
+    null,
+  );
+  const mocapCursorTargetSampleRef = useRef<PuzzleMocapCursorSample | null>(null);
+  const mocapCursorIntervalRef = useRef<number | null>(null);
+  const updateMocapCursorSampleRef = useRef<(
+    nextSample: PuzzleMocapCursorSample,
+  ) => void>(() => {});
  const runtimeDragInputControllerRef = useRef(
    createRuntimeDragInputController<string>(),
  );
@@ -930,6 +943,21 @@ export function PuzzleRuntimeShell({
      readRuntimeInputElementBounds(boardRef.current),
    );

+  const resetMocapCursorInterpolation = () => {
+    mocapCursorPreviousSampleRef.current = null;
+    mocapCursorTargetSampleRef.current = null;
+    setMocapCursor(null);
+  };
+
+  updateMocapCursorSampleRef.current = (nextSample: PuzzleMocapCursorSample) => {
+    const previousTarget = mocapCursorTargetSampleRef.current;
+    mocapCursorPreviousSampleRef.current = previousTarget ?? nextSample;
+    mocapCursorTargetSampleRef.current = nextSample;
+    if (!previousTarget) {
+      setMocapCursor(nextSample);
+    }
+  };
+
  const syncRuntimeDragFromController = (
    session: RuntimeDragInputSession<string> | null,
  ) => {
@@ -994,7 +1022,7 @@ export function PuzzleRuntimeShell({
    const activeSession = runtimeDragInputControllerRef.current.getSession();
    if (!board || runtimeStatus !== 'playing' || isInteractionLocked) {
      runtimeDragInputControllerRef.current.cancel();
-      setMocapCursor(null);
+      resetMocapCursorInterpolation();
      return;
    }
    if (
@@ -1003,19 +1031,18 @@ export function PuzzleRuntimeShell({
      typeof primaryMocapHandY !== 'number'
    ) {
      runtimeDragInputControllerRef.current.cancel(PUZZLE_MOCAP_DRAG_INPUT_ID);
-      setMocapCursor(null);
+      resetMocapCursorInterpolation();
      return;
    }

-    setMocapCursor({
+    const nextSample = {
      x: primaryMocapHandX,
      y: primaryMocapHandY,
      state: primaryMocapHandState,
-    });
-    const handPoint = resolveBoardInputPointFromNormalized(
-      primaryMocapHandX,
-      primaryMocapHandY,
-    );
+      receivedAtMs: performance.now(),
+    };
+    updateMocapCursorSampleRef.current(nextSample);
+    const handPoint = resolveBoardInputPointFromNormalized(nextSample.x, nextSample.y);
    if (primaryMocapHandState === 'grab') {
      if (activeSession?.inputId !== PUZZLE_MOCAP_DRAG_INPUT_ID) {
        const sourceCell = resolveRuntimeInputGridCell(handPoint, board);
@@ -1063,6 +1090,64 @@ export function PuzzleRuntimeShell({
    runtimeStatus,
  ]);

+  useEffect(() => {
+    if (!board || runtimeStatus !== 'playing') {
+      if (mocapCursorIntervalRef.current !== null) {
+        window.clearInterval(mocapCursorIntervalRef.current);
+        mocapCursorIntervalRef.current = null;
+      }
+      return;
+    }
+
+    const tickMocapCursor = () => {
+      const targetSample = mocapCursorTargetSampleRef.current;
+      if (!targetSample) {
+        return;
+      }
+      const previousSample = mocapCursorPreviousSampleRef.current ?? targetSample;
+      const durationMs = Math.max(
+        PUZZLE_MOCAP_CURSOR_FRAME_MS,
+        targetSample.receivedAtMs - previousSample.receivedAtMs,
+      );
+      const progress = targetSample.receivedAtMs === previousSample.receivedAtMs
+        ? 1
+        : Math.min(
+            1,
+            Math.max(0, (performance.now() - targetSample.receivedAtMs) / durationMs),
+          );
+      const nextCursor = {
+        x: previousSample.x + (targetSample.x - previousSample.x) * progress,
+        y: previousSample.y + (targetSample.y - previousSample.y) * progress,
+        state: targetSample.state,
+      };
+      const nextPoint = resolveBoardInputPointFromNormalized(
+        nextCursor.x,
+        nextCursor.y,
+      );
+      setMocapCursor(nextCursor);
+      const activeSession = runtimeDragInputControllerRef.current.getSession();
+      if (activeSession?.inputId === PUZZLE_MOCAP_DRAG_INPUT_ID) {
+        runtimeDragInputControllerRef.current.move({
+          inputId: PUZZLE_MOCAP_DRAG_INPUT_ID,
+          point: nextPoint,
+          forceDragging: true,
+        });
+      }
+    };
+
+    tickMocapCursor();
+    mocapCursorIntervalRef.current = window.setInterval(
+      tickMocapCursor,
+      PUZZLE_MOCAP_CURSOR_FRAME_MS,
+    );
+    return () => {
+      if (mocapCursorIntervalRef.current !== null) {
+        window.clearInterval(mocapCursorIntervalRef.current);
+        mocapCursorIntervalRef.current = null;
+      }
+    };
+  }, [board, runtimeStatus]);
+
  if (!run || !currentLevel || !board) {
    return (
      <div
--- a/src/services/useMocapInput.test.ts
+++ b/src/services/useMocapInput.test.ts
@@ -0,0 +1,81 @@
+import { describe, expect, test } from 'vitest';
+
+import { parseMocapPacket, resolveMocapPalmCenter } from './useMocapInput';
+
+describe('resolveMocapPalmCenter', () => {
+  test('优先用手腕和四个 MCP 点加权计算掌心派生点', () => {
+    const center = resolveMocapPalmCenter([
+      { name: 'wrist', x: 0.1, y: 0.2 },
+      { name: 'index_mcp', x: 0.3, y: 0.4 },
+      { name: 'middle_mcp', x: 0.5, y: 0.6 },
+      { name: 'ring_mcp', x: 0.7, y: 0.8 },
+      { name: 'pinky_mcp', x: 0.9, y: 1 },
+      { name: 'index_finger_tip', x: 1, y: 1 },
+    ]);
+
+    expect(center?.x).toBeCloseTo(0.44);
+    expect(center?.y).toBeCloseTo(0.54);
+  });
+
+  test('可用掌心点少于三个时不返回掌心坐标', () => {
+    expect(
+      resolveMocapPalmCenter([
+        { name: 'wrist', x: 0.1, y: 0.2 },
+        { name: 'index_mcp', x: 0.3, y: 0.4 },
+      ]),
+    ).toBeNull();
+  });
+});
+
+describe('parseMocapPacket', () => {
+  test('解析手部数据时优先把 primaryHand 定位到掌心而不是腕部或指尖', () => {
+    const command = parseMocapPacket({
+      hands: [
+        {
+          state: 'open_palm',
+          x: 0.01,
+          y: 0.02,
+          landmarks: [
+            { name: 'wrist', x: 0.1, y: 0.2 },
+            { name: 'index_mcp', x: 0.3, y: 0.4 },
+            { name: 'middle_mcp', x: 0.5, y: 0.6 },
+            { name: 'ring_mcp', x: 0.7, y: 0.8 },
+            { name: 'pinky_mcp', x: 0.9, y: 1 },
+          ],
+        },
+      ],
+    });
+
+    expect(command.primaryHand?.x).toBeCloseTo(0.44);
+    expect(command.primaryHand?.y).toBeCloseTo(0.54);
+    expect(command.primaryHand).toEqual(
+      expect.objectContaining({
+        state: 'open_palm',
+        source: 'palm_center',
+      }),
+    );
+  });
+
+  test('缺少足够掌心关键点时退回 wrist landmark，再退回 hand 直出坐标', () => {
+    const landmarkFallback = parseMocapPacket({
+      hands: [
+        {
+          state: 'grab',
+          x: 0.9,
+          y: 0.8,
+          landmarks: [{ name: 'wrist', x: 0.25, y: 0.75 }],
+        },
+      ],
+    });
+    expect(landmarkFallback.primaryHand).toEqual(
+      expect.objectContaining({x: 0.25, y: 0.75, source: 'landmark'}),
+    );
+
+    const directFallback = parseMocapPacket({
+      hands: [{ state: 'grab', x: 0.9, y: 0.8 }],
+    });
+    expect(directFallback.primaryHand).toEqual(
+      expect.objectContaining({x: 0.9, y: 0.8, source: 'direct'}),
+    );
+  });
+});
--- a/src/services/useMocapInput.ts
+++ b/src/services/useMocapInput.ts
@@ -10,6 +10,7 @@ export type MocapInputCommand = {
    x: number;
    y: number;
    state: MocapHandState;
+    source?: 'palm_center' | 'direct' | 'landmark';
  } | null;
  parseWarnings?: string[];
 };
@@ -32,9 +33,26 @@ export type UseMocapInputOptions = {
  reconnectDelayMs?: number;
 };

+type MocapParsedHand = {
+  x: number;
+  y: number;
+  state: MocapHandState;
+  source: 'palm_center' | 'direct' | 'landmark';
+};
+
+type MocapLandmarkRecord = Record<string, unknown>;
+
 const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876';
 const DEFAULT_RECONNECT_DELAY_MS = 1200;
 const MAX_RAW_PACKET_PREVIEW_LENGTH = 360;
+const PALM_CENTER_WEIGHTS = [
+  ['wrist', 0.25],
+  ['index_mcp', 0.2],
+  ['middle_mcp', 0.25],
+  ['ring_mcp', 0.2],
+  ['pinky_mcp', 0.1],
+] as const;
+const MIN_PALM_CENTER_POINT_COUNT = 3;

 function buildRawPacketPreview(rawData: unknown): string {
  const rawText = typeof rawData === 'string' ? rawData : JSON.stringify(rawData);
@@ -57,68 +75,86 @@ function normalizeCoordinate(value: unknown) {
  return Math.min(1, Math.max(0, numericValue));
 }

-function resolvePrimaryHand(hands: unknown) {
+function resolveLandmarkCoordinate(landmark: MocapLandmarkRecord | undefined) {
+  const x = normalizeCoordinate(landmark?.x);
+  const y = normalizeCoordinate(landmark?.y);
+  if (x === null || y === null) {
+    return null;
+  }
+  return { x, y };
+}
+
+export function resolveMocapPalmCenter(
+  landmarks: Array<MocapLandmarkRecord>,
+): { x: number; y: number } | null {
+  const landmarksByName = new Map(
+    landmarks
+      .filter((landmark) => typeof landmark?.name === 'string')
+      .map((landmark) => [String(landmark.name), landmark]),
+  );
+  const weightedPoints = PALM_CENTER_WEIGHTS.map(([name, weight]) => {
+    const point = resolveLandmarkCoordinate(landmarksByName.get(name));
+    return point ? { ...point, weight: Number(weight) } : null;
+  }).filter((point): point is { x: number; y: number; weight: number } => Boolean(point));
+
+  if (weightedPoints.length < MIN_PALM_CENTER_POINT_COUNT) {
+    return null;
+  }
+
+  const weightTotal = weightedPoints.reduce((sum, point) => sum + point.weight, 0);
+  if (weightTotal <= 0) {
+    return null;
+  }
+
+  return {
+    x: weightedPoints.reduce((sum, point) => sum + point.x * point.weight, 0) / weightTotal,
+    y: weightedPoints.reduce((sum, point) => sum + point.y * point.weight, 0) / weightTotal,
+  };
+}
+
+function resolvePrimaryHand(hands: unknown): MocapParsedHand | null {
  if (!Array.isArray(hands)) {
    return null;
  }

  for (const hand of hands) {
-    if (!hand || typeof hand !== 'object') {
-      continue;
+    const parsedHand = resolveHandLike(hand);
+    if (parsedHand) {
+      return parsedHand;
    }
-
-    const handRecord = hand as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
-    const state = normaliseHandState(handRecord.state);
-    const directX = normalizeCoordinate(handRecord.x);
-    const directY = normalizeCoordinate(handRecord.y);
-    if (directX !== null && directY !== null) {
-      return {x: directX, y: directY, state};
-    }
-    if (!Array.isArray(handRecord.landmarks)) {
-      continue;
-    }
-
-    const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
-    const landmark =
-      landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
-    const x = normalizeCoordinate(landmark?.x);
-    const y = normalizeCoordinate(landmark?.y);
-    if (x === null || y === null) {
-      continue;
-    }
-
-    return {x, y, state};
  }

  return null;
 }

-function resolveHandLike(record: unknown) {
+function resolveHandLike(record: unknown): MocapParsedHand | null {
  if (!record || typeof record !== 'object') {
    return null;
  }

  const handRecord = record as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
  const state = normaliseHandState(handRecord.state);
+  if (Array.isArray(handRecord.landmarks)) {
+    const landmarks = handRecord.landmarks as Array<MocapLandmarkRecord>;
+    const palmCenter = resolveMocapPalmCenter(landmarks);
+    if (palmCenter) {
+      return { ...palmCenter, state, source: 'palm_center' };
+    }
+
+    const landmark = landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
+    const fallbackPoint = resolveLandmarkCoordinate(landmark);
+    if (fallbackPoint) {
+      return { ...fallbackPoint, state, source: 'landmark' };
+    }
+  }
+
  const directX = normalizeCoordinate(handRecord.x);
  const directY = normalizeCoordinate(handRecord.y);
  if (directX !== null && directY !== null) {
-    return {x: directX, y: directY, state};
-  }
-  if (!Array.isArray(handRecord.landmarks)) {
-    return null;
+    return {x: directX, y: directY, state, source: 'direct'};
  }

-  const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
-  const landmark =
-    landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
-  const x = normalizeCoordinate(landmark?.x);
-  const y = normalizeCoordinate(landmark?.y);
-  if (x === null || y === null) {
-    return null;
-  }
-
-  return {x, y, state};
+  return null;
 }

 function normaliseHandState(state: unknown): MocapHandState {
@@ -128,7 +164,7 @@ function normaliseHandState(state: unknown): MocapHandState {
  return 'unknown';
 }

-function parseMocapPacket(packet: unknown): MocapInputCommand {
+export function parseMocapPacket(packet: unknown): MocapInputCommand {
  if (!packet || typeof packet !== 'object') {
    return {actions: [], parseWarnings: ['packet 不是对象']};
  }