feat: add mocap puzzle debug and drag support

src/components/puzzle-runtime/PuzzleRuntimeShell.test.tsx

@@ -25,6 +25,25 @@ vi.mock('../ResolvedAssetImage', () => ({
   ResolvedAssetImage: () => null,
 }));
 
+const mocapMock = vi.hoisted(() => ({
+  state: 'grab',
+  x: 0.42,
+  y: 0.58,
+}));
+
+vi.mock('../../services/useMocapInput', () => ({
+  useMocapInput: () => ({
+    status: 'connected',
+    latestCommand: {
+      actions: [mocapMock.state],
+      primaryHand: {x: mocapMock.x, y: mocapMock.y, state: mocapMock.state},
+      parseWarnings: [],
+    },
+    rawPacketPreview: {text: '{"hands":[{"state":"grab"}]}', receivedAtMs: 1},
+    error: null,
+  }),
+}));
+
 function createAuthValue() {
   return {
     user: null,
@@ -138,6 +157,150 @@ const clearedRun: PuzzleRunSnapshot = {
   },
 };
 
+test('拼图界面显示 mocap 连接状态和最近动作调试信息', () => {
+  renderPuzzleRuntime(
+    <PuzzleRuntimeShell
+      run={{
+        ...clearedRun,
+        currentLevel: {
+          ...clearedRun.currentLevel!,
+          status: 'playing',
+        },
+      }}
+      onBack={vi.fn()}
+      onSwapPieces={vi.fn()}
+      onDragPiece={vi.fn()}
+      onAdvanceNextLevel={vi.fn()}
+    />,
+  );
+
+  const debugPanel = screen.getByTestId('puzzle-mocap-debug');
+  expect(within(debugPanel).getByText('mocap: connected')).toBeTruthy();
+  expect(within(debugPanel).getByText('动作: grab')).toBeTruthy();
+  expect(within(debugPanel).getByText('手势: grab @ 0.42, 0.58')).toBeTruthy();
+  expect(within(debugPanel).getByText('解析: 无')).toBeTruthy();
+  expect(within(debugPanel).getByText(/原始:/)).toBeTruthy();
+});
+
+test('拼图界面在 mocap open_palm 时显示体感光标', () => {
+  mocapMock.state = 'open_palm';
+  mocapMock.x = 0.42;
+  mocapMock.y = 0.58;
+  renderPuzzleRuntime(
+    <PuzzleRuntimeShell
+      run={{
+        ...clearedRun,
+        currentLevel: {
+          ...clearedRun.currentLevel!,
+          status: 'playing',
+          startedAtMs: Date.now(),
+          remainingMs: 300_000,
+          timeLimitMs: 300_000,
+        },
+      }}
+      onBack={vi.fn()}
+      onSwapPieces={vi.fn()}
+      onDragPiece={vi.fn()}
+      onAdvanceNextLevel={vi.fn()}
+    />,
+  );
+
+  const cursor = screen.getByTestId('puzzle-mocap-cursor');
+  expect(cursor).toBeTruthy();
+  expect(cursor).toHaveStyle({left: '42%', top: '58%'});
+  mocapMock.state = 'grab';
+});
+
+test('抓握时会触发拖拽提交并在松开时落子', () => {
+  mocapMock.state = 'grab';
+  mocapMock.x = 0.34;
+  mocapMock.y = 0.34;
+  const onDragPiece = vi.fn();
+  const playingRun: PuzzleRunSnapshot = {
+    ...clearedRun,
+    currentLevel: {
+      ...clearedRun.currentLevel!,
+      status: 'playing',
+      startedAtMs: Date.now(),
+      remainingMs: 300_000,
+      timeLimitMs: 300_000,
+      board: {
+        ...clearedRun.currentLevel!.board,
+        allTilesResolved: false,
+        pieces: clearedRun.currentLevel!.board.pieces.map((piece) =>
+          piece.pieceId === 'piece-0'
+            ? {...piece, currentRow: 0, currentCol: 0}
+            : piece,
+        ),
+      },
+    },
+  };
+
+  const { container } = renderPuzzleRuntime(
+    <PuzzleRuntimeShell
+      run={playingRun}
+      onBack={vi.fn()}
+      onSwapPieces={vi.fn()}
+      onDragPiece={onDragPiece}
+      onAdvanceNextLevel={vi.fn()}
+    />,
+  );
+
+  const piece = container.querySelector('[data-piece-id="piece-0"]') as HTMLElement | null;
+  if (!piece) {
+    throw new Error('缺少测试拼图片');
+  }
+  const board = container.querySelector('[data-testid="puzzle-board"]') as HTMLElement | null;
+  if (!board) {
+    throw new Error('缺少测试棋盘');
+  }
+  board.getBoundingClientRect = () => ({
+    x: 0,
+    y: 0,
+    left: 0,
+    top: 0,
+    right: 300,
+    bottom: 300,
+    width: 300,
+    height: 300,
+    toJSON: () => ({}),
+  } as DOMRect);
+
+  act(() => {
+    dispatchPointerEvent(piece, 'pointerdown', {
+      pointerId: 11,
+      clientX: 40,
+      clientY: 40,
+    });
+  });
+  act(() => {
+    dispatchPointerEvent(piece, 'pointermove', {
+      pointerId: 11,
+      clientX: 70,
+      clientY: 70,
+    });
+  });
+  act(() => {
+    dispatchPointerEvent(piece, 'pointermove', {
+      pointerId: 11,
+      clientX: 140,
+      clientY: 140,
+    });
+  });
+  act(() => {
+    dispatchPointerEvent(piece, 'pointerup', {
+      pointerId: 11,
+      clientX: 140,
+      clientY: 140,
+    });
+  });
+
+  expect(onDragPiece).toHaveBeenCalledTimes(1);
+  expect(onDragPiece).toHaveBeenCalledWith(
+    expect.objectContaining({pieceId: 'piece-0'}),
+  );
+});
+
 test('通关后显示结算弹窗、排行榜和下一关按钮', () => {
   vi.useFakeTimers();
   const onAdvanceNextLevel = vi.fn();

src/components/puzzle-runtime/PuzzleRuntimeShell.tsx

@@ -23,6 +23,7 @@ import type {
   SwapPuzzlePiecesRequest,
 } from '../../../packages/shared/src/contracts/puzzleRuntimeSession';
 import { useResolvedAssetReadUrl } from '../../hooks/useResolvedAssetReadUrl';
+import { useMocapInput } from '../../services/useMocapInput';
 import { CHROME_ICONS, getNineSliceStyle, UI_CHROME } from '../../uiAssets';
 import { useAuthUi } from '../auth/AuthUiContext';
 import { PixelIcon } from '../PixelIcon';
@@ -283,6 +284,12 @@ type PuzzleHintDemoState = {
   offsetYPercent: number;
 };
 
+type PuzzleMocapCursorState = {
+  x: number;
+  y: number;
+  state: string;
+};
+
 function triggerPuzzlePiecePressHapticFeedback() {
   if (typeof navigator === 'undefined') {
     return;
@@ -367,6 +374,10 @@ export function PuzzleRuntimeShell({
     pieceId: string;
     groupId: string | null;
   } | null>(null);
+  const [mocapCursor, setMocapCursor] = useState<PuzzleMocapCursorState | null>(
+    null,
+  );
+  const mocapDragRef = useRef<{pieceId: string} | null>(null);
   const [dismissedClearKey, setDismissedClearKey] = useState<string | null>(
     null,
   );
@@ -397,6 +408,18 @@ export function PuzzleRuntimeShell({
   const { resolvedUrl: resolvedCoverImage } = useResolvedAssetReadUrl(
     currentLevel?.coverImageSrc ?? null,
   );
+  const mocapInput = useMocapInput({enabled: runtimeStatus === 'playing'});
+  const mocapActionsLabel =
+    mocapInput.latestCommand?.actions.length
+      ? mocapInput.latestCommand.actions.join(', ')
+      : '无';
+  const mocapHandLabel = mocapInput.latestCommand?.primaryHand
+    ? `${mocapInput.latestCommand.primaryHand.state} @ ${mocapInput.latestCommand.primaryHand.x.toFixed(2)}, ${mocapInput.latestCommand.primaryHand.y.toFixed(2)}`
+    : '无';
+  const mocapParseWarningLabel = mocapInput.latestCommand?.parseWarnings?.length
+    ? mocapInput.latestCommand.parseWarnings.join('；')
+    : '无';
+  const mocapRawPacketLabel = mocapInput.rawPacketPreview?.text ?? '未收到';
 
   useEffect(() => {
     currentLevelRef.current = currentLevel;
@@ -850,6 +873,49 @@ export function PuzzleRuntimeShell({
     return { row, col };
   };
 
+  const resolveMocapTargetCell = (x: number, y: number) => ({
+    row: Math.min(board.rows - 1, Math.max(0, Math.floor(y * board.rows))),
+    col: Math.min(board.cols - 1, Math.max(0, Math.floor(x * board.cols))),
+  });
+
+  const handleMocapInputCommand = () => {
+    const hand = mocapInput.latestCommand?.primaryHand;
+    if (runtimeStatus !== 'playing' || isInteractionLocked || !hand) {
+      mocapDragRef.current = null;
+      setMocapCursor(null);
+      return;
+    }
+
+    setMocapCursor({x: hand.x, y: hand.y, state: hand.state});
+    if (hand.state === 'grab') {
+      if (mocapDragRef.current) {
+        return;
+      }
+      const sourceCell = resolveMocapTargetCell(hand.x, hand.y);
+      const sourcePiece = pieceByCell.get(`${sourceCell.row}:${sourceCell.col}`) ?? null;
+      if (!sourcePiece || sourcePiece.mergedGroupId) {
+        return;
+      }
+      mocapDragRef.current = {pieceId: sourcePiece.pieceId};
+      setSelectedPieceId(sourcePiece.pieceId);
+      triggerPuzzlePiecePressHapticFeedback();
+      return;
+    }
+
+    const draggingPiece = mocapDragRef.current;
+    if (!draggingPiece) {
+      return;
+    }
+    const targetCell = resolveMocapTargetCell(hand.x, hand.y);
+    mocapDragRef.current = null;
+    setSelectedPieceId(null);
+    onDragPiece({
+      pieceId: draggingPiece.pieceId,
+      targetRow: targetCell.row,
+      targetCol: targetCell.col,
+    });
+  };
+
   const handlePiecePointerUp = (
     pieceId: string,
     event: React.PointerEvent<HTMLDivElement>,
@@ -973,7 +1039,6 @@ export function PuzzleRuntimeShell({
     isClearResultReady;
   const isInteractionLocked =
     isBusy || runtimeStatus !== 'playing' || Boolean(propDialog);
-
   const handleBackRequest = () => {
     if (hideExitControls) {
       return;
@@ -1085,6 +1150,10 @@ export function PuzzleRuntimeShell({
     }
   };
 
+  useEffect(() => {
+    handleMocapInputCommand();
+  }, [mocapInput.latestCommand?.primaryHand]);
+
   return (
     <div
       className={`puzzle-runtime-shell ${embedded ? 'relative h-full min-h-0 w-full' : 'fixed inset-0 z-[100]'} flex justify-center`}
@@ -1445,6 +1514,21 @@ export function PuzzleRuntimeShell({
                 />
               </div>
             ) : null}
+            {mocapCursor ? (
+              <div
+                data-testid="puzzle-mocap-cursor"
+                className={`pointer-events-none absolute z-[70] flex h-8 w-8 -translate-x-1/2 -translate-y-1/2 items-center justify-center rounded-full border-2 ${
+                  mocapCursor.state === 'grab'
+                    ? 'border-amber-200 bg-amber-400/90 text-amber-950'
+                    : 'border-cyan-200 bg-cyan-300/90 text-cyan-950'
+                } shadow-[0_10px_24px_rgba(15,23,42,0.25)]`}
+                style={{left: `${mocapCursor.x * 100}%`, top: `${mocapCursor.y * 100}%`}}
+              >
+                <span className="text-[10px] font-black leading-none">
+                  {mocapCursor.state === 'grab' ? '抓' : '手'}
+                </span>
+              </div>
+            ) : null}
             {mergeFlash ? (
               <div
                 key={mergeFlash.key}
@@ -1472,6 +1556,19 @@ export function PuzzleRuntimeShell({
               已选择
             </div>
           ) : null}
+          <div
+            data-testid="puzzle-mocap-debug"
+            className="w-[min(92vw,34rem)] rounded-[0.9rem] border border-white/20 bg-slate-950/70 px-3 py-2 font-mono text-[10px] leading-4 text-white shadow-[0_12px_32px_rgba(15,23,42,0.25)] backdrop-blur"
+          >
+            <div>mocap: {mocapInput.status}</div>
+            <div>动作: {mocapActionsLabel}</div>
+            <div>手势: {mocapHandLabel}</div>
+            <div>解析: {mocapParseWarningLabel}</div>
+            <div className="max-h-20 overflow-auto break-all text-white/75">
+              原始: {mocapRawPacketLabel}
+            </div>
+            {mocapInput.error ? <div>错误: {mocapInput.error}</div> : null}
+          </div>
           {canShowNextAction ? (
             <button
               type="button"

src/services/useMocapInput.ts

@@ -0,0 +1,244 @@
+import {useEffect, useMemo, useRef, useState} from 'react';
+
+export type MocapConnectionStatus = 'idle' | 'connecting' | 'connected' | 'error';
+
+export type MocapHandState = 'open_palm' | 'grab' | 'unknown';
+
+export type MocapInputCommand = {
+  actions: string[];
+  primaryHand?: {
+    x: number;
+    y: number;
+    state: MocapHandState;
+  } | null;
+  parseWarnings?: string[];
+};
+
+export type MocapRawPacketPreview = {
+  text: string;
+  receivedAtMs: number;
+};
+
+export type UseMocapInputResult = {
+  status: MocapConnectionStatus;
+  latestCommand: MocapInputCommand | null;
+  rawPacketPreview: MocapRawPacketPreview | null;
+  error: string | null;
+};
+
+export type UseMocapInputOptions = {
+  enabled: boolean;
+  serviceUrl?: string;
+  reconnectDelayMs?: number;
+};
+
+const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876';
+const DEFAULT_RECONNECT_DELAY_MS = 1200;
+const MAX_RAW_PACKET_PREVIEW_LENGTH = 360;
+
+function buildRawPacketPreview(rawData: unknown): string {
+  const rawText = typeof rawData === 'string' ? rawData : JSON.stringify(rawData);
+  return rawText.length > MAX_RAW_PACKET_PREVIEW_LENGTH
+    ? `${rawText.slice(0, MAX_RAW_PACKET_PREVIEW_LENGTH)}…`
+    : rawText;
+}
+
+function buildMocapStreamUrl(serviceUrl: string) {
+  const url = new URL('/stream', serviceUrl);
+  url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
+  return url.toString();
+}
+
+function normalizeCoordinate(value: unknown) {
+  const numericValue = Number(value);
+  if (!Number.isFinite(numericValue)) {
+    return null;
+  }
+  return Math.min(1, Math.max(0, numericValue));
+}
+
+function resolvePrimaryHand(hands: unknown) {
+  if (!Array.isArray(hands)) {
+    return null;
+  }
+
+  for (const hand of hands) {
+    if (!hand || typeof hand !== 'object') {
+      continue;
+    }
+
+    const handRecord = hand as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
+    const state = normaliseHandState(handRecord.state);
+    const directX = normalizeCoordinate(handRecord.x);
+    const directY = normalizeCoordinate(handRecord.y);
+    if (directX !== null && directY !== null) {
+      return {x: directX, y: directY, state};
+    }
+    if (!Array.isArray(handRecord.landmarks)) {
+      continue;
+    }
+
+    const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
+    const landmark =
+      landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
+    const x = normalizeCoordinate(landmark?.x);
+    const y = normalizeCoordinate(landmark?.y);
+    if (x === null || y === null) {
+      continue;
+    }
+
+    return {x, y, state};
+  }
+
+  return null;
+}
+
+function resolveHandLike(record: unknown) {
+  if (!record || typeof record !== 'object') {
+    return null;
+  }
+
+  const handRecord = record as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
+  const state = normaliseHandState(handRecord.state);
+  const directX = normalizeCoordinate(handRecord.x);
+  const directY = normalizeCoordinate(handRecord.y);
+  if (directX !== null && directY !== null) {
+    return {x: directX, y: directY, state};
+  }
+  if (!Array.isArray(handRecord.landmarks)) {
+    return null;
+  }
+
+  const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
+  const landmark =
+    landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
+  const x = normalizeCoordinate(landmark?.x);
+  const y = normalizeCoordinate(landmark?.y);
+  if (x === null || y === null) {
+    return null;
+  }
+
+  return {x, y, state};
+}
+
+function normaliseHandState(state: unknown): MocapHandState {
+  if (state === 'grab' || state === 'open_palm') {
+    return state;
+  }
+  return 'unknown';
+}
+
+function parseMocapPacket(packet: unknown): MocapInputCommand {
+  if (!packet || typeof packet !== 'object') {
+    return {actions: [], parseWarnings: ['packet 不是对象']};
+  }
+
+  const packetRecord = packet as {hands?: unknown};
+  const primaryHand = resolvePrimaryHand(packetRecord.hands);
+  const actions = new Set<string>();
+  const parseWarnings: string[] = [];
+  if (!Array.isArray(packetRecord.hands)) {
+    parseWarnings.push('缺少 hands 数组');
+  } else if (!primaryHand) {
+    parseWarnings.push('hands 中没有可用坐标');
+  }
+  if (primaryHand?.state === 'grab') {
+    actions.add('grab');
+  }
+  if (primaryHand?.state === 'open_palm') {
+    actions.add('open_palm');
+  }
+  if (primaryHand && primaryHand.state === 'unknown') {
+    parseWarnings.push('手势 state 未识别');
+  }
+
+  return {
+    actions: Array.from(actions),
+    primaryHand,
+    parseWarnings,
+  };
+}
+
+export function useMocapInput({
+  enabled,
+  serviceUrl = DEFAULT_MOCAP_SERVICE_URL,
+  reconnectDelayMs = DEFAULT_RECONNECT_DELAY_MS,
+}: UseMocapInputOptions): UseMocapInputResult {
+  const [status, setStatus] = useState<MocapConnectionStatus>('idle');
+  const [latestCommand, setLatestCommand] = useState<MocapInputCommand | null>(
+    null,
+  );
+  const [rawPacketPreview, setRawPacketPreview] = useState<MocapRawPacketPreview | null>(
+    null,
+  );
+  const [error, setError] = useState<string | null>(null);
+  const reconnectTimerRef = useRef<number | null>(null);
+  const websocketRef = useRef<WebSocket | null>(null);
+
+  const streamUrl = useMemo(() => buildMocapStreamUrl(serviceUrl), [serviceUrl]);
+
+  useEffect(() => {
+    if (!enabled || typeof WebSocket === 'undefined') {
+      setStatus('idle');
+      return;
+    }
+
+    let cancelled = false;
+
+    const connect = () => {
+      if (cancelled) {
+        return;
+      }
+      setStatus('connecting');
+      setError(null);
+
+      const websocket = new WebSocket(streamUrl);
+      websocketRef.current = websocket;
+      websocket.onopen = () => {
+        if (!cancelled) {
+          setStatus('connected');
+        }
+      };
+      websocket.onmessage = (event) => {
+        if (cancelled) {
+          return;
+        }
+        try {
+          const rawText = String(event.data);
+          setRawPacketPreview({
+            text: buildRawPacketPreview(rawText),
+            receivedAtMs: Date.now(),
+          });
+          setLatestCommand(parseMocapPacket(JSON.parse(rawText)));
+        } catch (parseError) {
+          setError(parseError instanceof Error ? parseError.message : 'mocap 数据解析失败');
+        }
+      };
+      websocket.onerror = () => {
+        if (!cancelled) {
+          setStatus('error');
+          setError('mocap 连接异常');
+        }
+      };
+      websocket.onclose = () => {
+        if (cancelled) {
+          return;
+        }
+        setStatus('error');
+        reconnectTimerRef.current = window.setTimeout(connect, reconnectDelayMs);
+      };
+    };
+
+    connect();
+
+    return () => {
+      cancelled = true;
+      if (reconnectTimerRef.current !== null) {
+        window.clearTimeout(reconnectTimerRef.current);
+      }
+      websocketRef.current?.close();
+    };
+  }, [enabled, reconnectDelayMs, streamUrl]);
+
+  return {status, latestCommand, rawPacketPreview, error};
+}