import {useEffect, useMemo, useRef, useState} from 'react'; export type MocapConnectionStatus = 'idle' | 'connecting' | 'connected' | 'error'; export type MocapHandState = 'open_palm' | 'grab' | 'unknown'; export type MocapHandSide = 'left' | 'right' | 'unknown'; export type MocapHandSource = 'palm_center' | 'direct' | 'landmark'; export type MocapHandInput = { x: number; y: number; state: MocapHandState; side: MocapHandSide; source?: MocapHandSource; }; export type MocapBodyCenterInput = { x: number; y: number; }; export type MocapInputCommand = { actions: string[]; hands?: MocapHandInput[]; primaryHand?: MocapHandInput | null; leftHand?: MocapHandInput | null; rightHand?: MocapHandInput | null; bodyCenter?: MocapBodyCenterInput | null; parseWarnings?: string[]; }; export type MocapRawPacketPreview = { text: string; receivedAtMs: number; }; export type UseMocapInputResult = { status: MocapConnectionStatus; latestCommand: MocapInputCommand | null; rawPacketPreview: MocapRawPacketPreview | null; error: string | null; }; export type UseMocapInputOptions = { enabled: boolean; serviceUrl?: string; reconnectDelayMs?: number; }; type MocapLandmarkRecord = Record; const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876'; const DEFAULT_RECONNECT_DELAY_MS = 1200; const MAX_RAW_PACKET_PREVIEW_LENGTH = 360; const PALM_CENTER_WEIGHTS = [ ['wrist', 0.25], ['index_mcp', 0.2], ['middle_mcp', 0.25], ['ring_mcp', 0.2], ['pinky_mcp', 0.1], ] as const; const MIN_PALM_CENTER_POINT_COUNT = 3; function buildRawPacketPreview(rawData: unknown): string { const rawText = typeof rawData === 'string' ? rawData : JSON.stringify(rawData); return rawText.length > MAX_RAW_PACKET_PREVIEW_LENGTH ? `${rawText.slice(0, MAX_RAW_PACKET_PREVIEW_LENGTH)}…` : rawText; } function buildMocapStreamUrl(serviceUrl: string) { const url = new URL('/stream', serviceUrl); url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:'; return url.toString(); } function normalizeCoordinate(value: unknown) { const numericValue = Number(value); if (!Number.isFinite(numericValue)) { return null; } return Math.min(1, Math.max(0, numericValue)); } function resolveNormalizedPoint(value: unknown) { if (Array.isArray(value)) { const x = normalizeCoordinate(value[0]); const y = normalizeCoordinate(value[1]); if (x === null || y === null) { return null; } return {x, y}; } if (!value || typeof value !== 'object') { return null; } const pointRecord = value as {x?: unknown; y?: unknown}; const x = normalizeCoordinate(pointRecord.x); const y = normalizeCoordinate(pointRecord.y); if (x === null || y === null) { return null; } return {x, y}; } function resolveLandmarkCoordinate(landmark: MocapLandmarkRecord | undefined) { const x = normalizeCoordinate(landmark?.x); const y = normalizeCoordinate(landmark?.y); if (x === null || y === null) { return null; } return {x, y}; } export function resolveMocapPalmCenter( landmarks: Array, ): {x: number; y: number} | null { const landmarksByName = new Map( landmarks .filter((landmark) => typeof landmark?.name === 'string') .map((landmark) => [String(landmark.name), landmark]), ); const weightedPoints = PALM_CENTER_WEIGHTS.map(([name, weight]) => { const point = resolveLandmarkCoordinate(landmarksByName.get(name)); return point ? {...point, weight: Number(weight)} : null; }).filter((point): point is {x: number; y: number; weight: number} => Boolean(point), ); if (weightedPoints.length < MIN_PALM_CENTER_POINT_COUNT) { return null; } const weightTotal = weightedPoints.reduce((sum, point) => sum + point.weight, 0); if (weightTotal <= 0) { return null; } return { x: weightedPoints.reduce((sum, point) => sum + point.x * point.weight, 0) / weightTotal, y: weightedPoints.reduce((sum, point) => sum + point.y * point.weight, 0) / weightTotal, }; } function normaliseHandSide(side: unknown): MocapHandSide { if (typeof side !== 'string') { return 'unknown'; } const normalized = side.trim().toLocaleLowerCase('en-US'); if ( normalized === 'left' || normalized === 'l' || normalized === 'left hand' || normalized === 'left_hand' || normalized === 'left-hand' || normalized === '左' || normalized === '左手' ) { return 'left'; } if ( normalized === 'right' || normalized === 'r' || normalized === 'right hand' || normalized === 'right_hand' || normalized === 'right-hand' || normalized === '右' || normalized === '右手' ) { return 'right'; } return 'unknown'; } function normalizeMocapAction(value: unknown) { if (typeof value !== 'string') { return null; } const normalized = value .trim() .toLocaleLowerCase('en-US') .replace(/\s+/gu, '_') .replace(/-/gu, '_'); return normalized || null; } function addMocapActions(actions: Set, value: unknown) { if (Array.isArray(value)) { value.forEach((item) => addMocapActions(actions, item)); return; } if (value && typeof value === 'object') { const actionRecord = value as Record; addMocapActions(actions, actionRecord.action); addMocapActions(actions, actionRecord.actions); addMocapActions(actions, actionRecord.gesture); addMocapActions(actions, actionRecord.gestures); addMocapActions(actions, actionRecord.event); addMocapActions(actions, actionRecord.name); addMocapActions(actions, actionRecord.type); return; } const normalized = normalizeMocapAction(value); if (normalized) { actions.add(normalized); } } function normaliseHandState(state: unknown): MocapHandState { if (typeof state !== 'string') { return 'unknown'; } const normalized = state .trim() .toLocaleLowerCase('en-US') .replace(/\s+/gu, '_') .replace(/-/gu, '_'); if ( normalized === 'grab' || normalized === 'grabbing' || normalized === 'close' || normalized === 'fist' || normalized === 'closed_fist' || normalized === 'closed' ) { return 'grab'; } if ( normalized === 'open_palm' || normalized === 'open_palm_up' || normalized === 'open' || normalized === 'palm' || normalized === 'hand_open' ) { return 'open_palm'; } return 'unknown'; } function resolveMocapHand(record: unknown, fallbackSide: MocapHandSide) { if (!record || typeof record !== 'object') { return null; } const handRecord = record as { state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown; side?: unknown; handedness?: unknown; label?: unknown; hand?: unknown; }; const state = normaliseHandState(handRecord.state); const detectedSide = normaliseHandSide( handRecord.side ?? handRecord.handedness ?? handRecord.label ?? handRecord.hand, ); const side = detectedSide === 'unknown' ? fallbackSide : detectedSide; if (Array.isArray(handRecord.landmarks)) { const landmarks = handRecord.landmarks as Array; const palmCenter = resolveMocapPalmCenter(landmarks); if (palmCenter) { return {...palmCenter, state, side, source: 'palm_center' as const}; } const landmark = landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0]; const fallbackPoint = resolveLandmarkCoordinate(landmark); if (fallbackPoint) { return {...fallbackPoint, state, side, source: 'landmark' as const}; } } const directX = normalizeCoordinate(handRecord.x); const directY = normalizeCoordinate(handRecord.y); if (directX !== null && directY !== null) { return {x: directX, y: directY, state, side, source: 'direct' as const}; } return null; } function resolveHands(packetRecord: Record) { const resolvedHands: MocapHandInput[] = []; const packetHands = packetRecord.hands; if (!Array.isArray(packetHands)) { const leftHand = resolveMocapHand( packetRecord.leftHand ?? packetRecord.left_hand, 'left', ); const rightHand = resolveMocapHand( packetRecord.rightHand ?? packetRecord.right_hand, 'right', ); if (leftHand) { resolvedHands.push(leftHand); } if (rightHand) { resolvedHands.push(rightHand); } return resolvedHands; } for (const hand of packetHands) { const resolvedHand = resolveMocapHand(hand, 'unknown'); if (resolvedHand) { resolvedHands.push(resolvedHand); } } return resolvedHands; } function resolveBodyCenter(packetRecord: Record) { const generalRecord = packetRecord.general && typeof packetRecord.general === 'object' ? (packetRecord.general as Record) : null; const bodyCandidates = [generalRecord?.body, packetRecord.body]; for (const bodyCandidate of bodyCandidates) { if (!bodyCandidate || typeof bodyCandidate !== 'object') { continue; } const bodyRecord = bodyCandidate as Record; const center = resolveNormalizedPoint( bodyRecord.center_norm ?? bodyRecord.centerNorm ?? bodyRecord.center, ); if (center) { return center; } } return null; } export function parseMocapPacket(packet: unknown): MocapInputCommand { if (!packet || typeof packet !== 'object') { return {actions: [], parseWarnings: ['packet 不是对象']}; } const packetRecord = packet as Record; const hands = resolveHands(packetRecord); const primaryHand = hands[0] ?? null; const leftHand = hands.find((hand) => hand.side === 'left') ?? null; const rightHand = hands.find((hand) => hand.side === 'right') ?? null; const bodyCenter = resolveBodyCenter(packetRecord); const actions = new Set(); const parseWarnings: string[] = []; addMocapActions(actions, packetRecord.actions); addMocapActions(actions, packetRecord.action); addMocapActions(actions, packetRecord.gesture); addMocapActions(actions, packetRecord.gestures); addMocapActions(actions, packetRecord.event); addMocapActions(actions, packetRecord.name); addMocapActions(actions, packetRecord.type); if (!Array.isArray(packetRecord.hands) && hands.length === 0 && !bodyCenter) { parseWarnings.push('缺少 hands 数组'); } else if (!primaryHand && !bodyCenter) { parseWarnings.push('hands 中没有可用坐标'); } if (primaryHand?.state === 'grab') { actions.add('grab'); } if (primaryHand?.state === 'open_palm') { actions.add('open_palm'); } for (const hand of hands) { if (hand.state !== 'unknown') { actions.add(hand.state); } } if (primaryHand && primaryHand.state === 'unknown') { parseWarnings.push('手势 state 未识别'); } return { actions: Array.from(actions), hands, primaryHand, leftHand, rightHand, bodyCenter, parseWarnings, }; } export function useMocapInput({ enabled, serviceUrl = DEFAULT_MOCAP_SERVICE_URL, reconnectDelayMs = DEFAULT_RECONNECT_DELAY_MS, }: UseMocapInputOptions): UseMocapInputResult { const [status, setStatus] = useState('idle'); const [latestCommand, setLatestCommand] = useState( null, ); const [rawPacketPreview, setRawPacketPreview] = useState( null, ); const [error, setError] = useState(null); const reconnectTimerRef = useRef(null); const websocketRef = useRef(null); const streamUrl = useMemo(() => buildMocapStreamUrl(serviceUrl), [serviceUrl]); useEffect(() => { if (!enabled || typeof WebSocket === 'undefined') { setStatus('idle'); return; } let cancelled = false; const connect = () => { if (cancelled) { return; } setStatus('connecting'); setError(null); const websocket = new WebSocket(streamUrl); websocketRef.current = websocket; websocket.onopen = () => { if (!cancelled) { setStatus('connected'); } }; websocket.onmessage = (event) => { if (cancelled) { return; } try { const rawText = String(event.data); setRawPacketPreview({ text: buildRawPacketPreview(rawText), receivedAtMs: Date.now(), }); setLatestCommand(parseMocapPacket(JSON.parse(rawText))); } catch (parseError) { setError(parseError instanceof Error ? parseError.message : 'mocap 数据解析失败'); } }; websocket.onerror = () => { if (!cancelled) { setStatus('error'); setError('mocap 连接异常'); } }; websocket.onclose = () => { if (cancelled) { return; } setStatus('error'); reconnectTimerRef.current = window.setTimeout(connect, reconnectDelayMs); }; }; connect(); return () => { cancelled = true; if (reconnectTimerRef.current !== null) { window.clearTimeout(reconnectTimerRef.current); } websocketRef.current?.close(); }; }, [enabled, reconnectDelayMs, streamUrl]); return {status, latestCommand, rawPacketPreview, error}; }