feat: add mocap puzzle debug and drag support

2026-05-10 12:34:18 +08:00
parent 9b39a52049
commit 6ed6859855
6 changed files with 651 additions and 37 deletions
--- a/src/services/useMocapInput.ts
+++ b/src/services/useMocapInput.ts
@@ -0,0 +1,244 @@
+import {useEffect, useMemo, useRef, useState} from 'react';
+
+export type MocapConnectionStatus = 'idle' | 'connecting' | 'connected' | 'error';
+
+export type MocapHandState = 'open_palm' | 'grab' | 'unknown';
+
+export type MocapInputCommand = {
+  actions: string[];
+  primaryHand?: {
+    x: number;
+    y: number;
+    state: MocapHandState;
+  } | null;
+  parseWarnings?: string[];
+};
+
+export type MocapRawPacketPreview = {
+  text: string;
+  receivedAtMs: number;
+};
+
+export type UseMocapInputResult = {
+  status: MocapConnectionStatus;
+  latestCommand: MocapInputCommand | null;
+  rawPacketPreview: MocapRawPacketPreview | null;
+  error: string | null;
+};
+
+export type UseMocapInputOptions = {
+  enabled: boolean;
+  serviceUrl?: string;
+  reconnectDelayMs?: number;
+};
+
+const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876';
+const DEFAULT_RECONNECT_DELAY_MS = 1200;
+const MAX_RAW_PACKET_PREVIEW_LENGTH = 360;
+
+function buildRawPacketPreview(rawData: unknown): string {
+  const rawText = typeof rawData === 'string' ? rawData : JSON.stringify(rawData);
+  return rawText.length > MAX_RAW_PACKET_PREVIEW_LENGTH
+    ? `${rawText.slice(0, MAX_RAW_PACKET_PREVIEW_LENGTH)}…`
+    : rawText;
+}
+
+function buildMocapStreamUrl(serviceUrl: string) {
+  const url = new URL('/stream', serviceUrl);
+  url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
+  return url.toString();
+}
+
+function normalizeCoordinate(value: unknown) {
+  const numericValue = Number(value);
+  if (!Number.isFinite(numericValue)) {
+    return null;
+  }
+  return Math.min(1, Math.max(0, numericValue));
+}
+
+function resolvePrimaryHand(hands: unknown) {
+  if (!Array.isArray(hands)) {
+    return null;
+  }
+
+  for (const hand of hands) {
+    if (!hand || typeof hand !== 'object') {
+      continue;
+    }
+
+    const handRecord = hand as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
+    const state = normaliseHandState(handRecord.state);
+    const directX = normalizeCoordinate(handRecord.x);
+    const directY = normalizeCoordinate(handRecord.y);
+    if (directX !== null && directY !== null) {
+      return {x: directX, y: directY, state};
+    }
+    if (!Array.isArray(handRecord.landmarks)) {
+      continue;
+    }
+
+    const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
+    const landmark =
+      landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
+    const x = normalizeCoordinate(landmark?.x);
+    const y = normalizeCoordinate(landmark?.y);
+    if (x === null || y === null) {
+      continue;
+    }
+
+    return {x, y, state};
+  }
+
+  return null;
+}
+
+function resolveHandLike(record: unknown) {
+  if (!record || typeof record !== 'object') {
+    return null;
+  }
+
+  const handRecord = record as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
+  const state = normaliseHandState(handRecord.state);
+  const directX = normalizeCoordinate(handRecord.x);
+  const directY = normalizeCoordinate(handRecord.y);
+  if (directX !== null && directY !== null) {
+    return {x: directX, y: directY, state};
+  }
+  if (!Array.isArray(handRecord.landmarks)) {
+    return null;
+  }
+
+  const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
+  const landmark =
+    landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
+  const x = normalizeCoordinate(landmark?.x);
+  const y = normalizeCoordinate(landmark?.y);
+  if (x === null || y === null) {
+    return null;
+  }
+
+  return {x, y, state};
+}
+
+function normaliseHandState(state: unknown): MocapHandState {
+  if (state === 'grab' || state === 'open_palm') {
+    return state;
+  }
+  return 'unknown';
+}
+
+function parseMocapPacket(packet: unknown): MocapInputCommand {
+  if (!packet || typeof packet !== 'object') {
+    return {actions: [], parseWarnings: ['packet 不是对象']};
+  }
+
+  const packetRecord = packet as {hands?: unknown};
+  const primaryHand = resolvePrimaryHand(packetRecord.hands);
+  const actions = new Set<string>();
+  const parseWarnings: string[] = [];
+  if (!Array.isArray(packetRecord.hands)) {
+    parseWarnings.push('缺少 hands 数组');
+  } else if (!primaryHand) {
+    parseWarnings.push('hands 中没有可用坐标');
+  }
+  if (primaryHand?.state === 'grab') {
+    actions.add('grab');
+  }
+  if (primaryHand?.state === 'open_palm') {
+    actions.add('open_palm');
+  }
+  if (primaryHand && primaryHand.state === 'unknown') {
+    parseWarnings.push('手势 state 未识别');
+  }
+
+  return {
+    actions: Array.from(actions),
+    primaryHand,
+    parseWarnings,
+  };
+}
+
+export function useMocapInput({
+  enabled,
+  serviceUrl = DEFAULT_MOCAP_SERVICE_URL,
+  reconnectDelayMs = DEFAULT_RECONNECT_DELAY_MS,
+}: UseMocapInputOptions): UseMocapInputResult {
+  const [status, setStatus] = useState<MocapConnectionStatus>('idle');
+  const [latestCommand, setLatestCommand] = useState<MocapInputCommand | null>(
+    null,
+  );
+  const [rawPacketPreview, setRawPacketPreview] = useState<MocapRawPacketPreview | null>(
+    null,
+  );
+  const [error, setError] = useState<string | null>(null);
+  const reconnectTimerRef = useRef<number | null>(null);
+  const websocketRef = useRef<WebSocket | null>(null);
+
+  const streamUrl = useMemo(() => buildMocapStreamUrl(serviceUrl), [serviceUrl]);
+
+  useEffect(() => {
+    if (!enabled || typeof WebSocket === 'undefined') {
+      setStatus('idle');
+      return;
+    }
+
+    let cancelled = false;
+
+    const connect = () => {
+      if (cancelled) {
+        return;
+      }
+      setStatus('connecting');
+      setError(null);
+
+      const websocket = new WebSocket(streamUrl);
+      websocketRef.current = websocket;
+      websocket.onopen = () => {
+        if (!cancelled) {
+          setStatus('connected');
+        }
+      };
+      websocket.onmessage = (event) => {
+        if (cancelled) {
+          return;
+        }
+        try {
+          const rawText = String(event.data);
+          setRawPacketPreview({
+            text: buildRawPacketPreview(rawText),
+            receivedAtMs: Date.now(),
+          });
+          setLatestCommand(parseMocapPacket(JSON.parse(rawText)));
+        } catch (parseError) {
+          setError(parseError instanceof Error ? parseError.message : 'mocap 数据解析失败');
+        }
+      };
+      websocket.onerror = () => {
+        if (!cancelled) {
+          setStatus('error');
+          setError('mocap 连接异常');
+        }
+      };
+      websocket.onclose = () => {
+        if (cancelled) {
+          return;
+        }
+        setStatus('error');
+        reconnectTimerRef.current = window.setTimeout(connect, reconnectDelayMs);
+      };
+    };
+
+    connect();
+
+    return () => {
+      cancelled = true;
+      if (reconnectTimerRef.current !== null) {
+        window.clearTimeout(reconnectTimerRef.current);
+      }
+      websocketRef.current?.close();
+    };
+  }, [enabled, reconnectDelayMs, streamUrl]);
+
+  return {status, latestCommand, rawPacketPreview, error};
+}