feat: connect child motion warmup to mocap input

src/services/useMocapInput.ts

@@ -3,15 +3,29 @@ import {useEffect, useMemo, useRef, useState} from 'react';
 export type MocapConnectionStatus = 'idle' | 'connecting' | 'connected' | 'error';
 
 export type MocapHandState = 'open_palm' | 'grab' | 'unknown';
+export type MocapHandSide = 'left' | 'right' | 'unknown';
+export type MocapHandSource = 'palm_center' | 'direct' | 'landmark';
+
+export type MocapHandInput = {
+  x: number;
+  y: number;
+  state: MocapHandState;
+  side: MocapHandSide;
+  source?: MocapHandSource;
+};
+
+export type MocapBodyCenterInput = {
+  x: number;
+  y: number;
+};
 
 export type MocapInputCommand = {
   actions: string[];
-  primaryHand?: {
-    x: number;
-    y: number;
-    state: MocapHandState;
-    source?: 'palm_center' | 'direct' | 'landmark';
-  } | null;
+  hands?: MocapHandInput[];
+  primaryHand?: MocapHandInput | null;
+  leftHand?: MocapHandInput | null;
+  rightHand?: MocapHandInput | null;
+  bodyCenter?: MocapBodyCenterInput | null;
   parseWarnings?: string[];
 };
 
@@ -33,13 +47,6 @@ export type UseMocapInputOptions = {
   reconnectDelayMs?: number;
 };
 
-type MocapParsedHand = {
-  x: number;
-  y: number;
-  state: MocapHandState;
-  source: 'palm_center' | 'direct' | 'landmark';
-};
-
 type MocapLandmarkRecord = Record<string, unknown>;
 
 const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876';
@@ -75,18 +82,42 @@ function normalizeCoordinate(value: unknown) {
   return Math.min(1, Math.max(0, numericValue));
 }
 
+function resolveNormalizedPoint(value: unknown) {
+  if (Array.isArray(value)) {
+    const x = normalizeCoordinate(value[0]);
+    const y = normalizeCoordinate(value[1]);
+    if (x === null || y === null) {
+      return null;
+    }
+    return {x, y};
+  }
+
+  if (!value || typeof value !== 'object') {
+    return null;
+  }
+
+  const pointRecord = value as {x?: unknown; y?: unknown};
+  const x = normalizeCoordinate(pointRecord.x);
+  const y = normalizeCoordinate(pointRecord.y);
+  if (x === null || y === null) {
+    return null;
+  }
+
+  return {x, y};
+}
+
 function resolveLandmarkCoordinate(landmark: MocapLandmarkRecord | undefined) {
   const x = normalizeCoordinate(landmark?.x);
   const y = normalizeCoordinate(landmark?.y);
   if (x === null || y === null) {
     return null;
   }
-  return { x, y };
+  return {x, y};
 }
 
 export function resolveMocapPalmCenter(
   landmarks: Array<MocapLandmarkRecord>,
-): { x: number; y: number } | null {
+): {x: number; y: number} | null {
   const landmarksByName = new Map(
     landmarks
       .filter((landmark) => typeof landmark?.name === 'string')
@@ -94,8 +125,10 @@ export function resolveMocapPalmCenter(
   );
   const weightedPoints = PALM_CENTER_WEIGHTS.map(([name, weight]) => {
     const point = resolveLandmarkCoordinate(landmarksByName.get(name));
-    return point ? { ...point, weight: Number(weight) } : null;
-  }).filter((point): point is { x: number; y: number; weight: number } => Boolean(point));
+    return point ? {...point, weight: Number(weight)} : null;
+  }).filter((point): point is {x: number; y: number; weight: number} =>
+    Boolean(point),
+  );
 
   if (weightedPoints.length < MIN_PALM_CENTER_POINT_COUNT) {
     return null;
@@ -112,56 +145,213 @@ export function resolveMocapPalmCenter(
   };
 }
 
-function resolvePrimaryHand(hands: unknown): MocapParsedHand | null {
-  if (!Array.isArray(hands)) {
+function normaliseHandSide(side: unknown): MocapHandSide {
+  if (typeof side !== 'string') {
+    return 'unknown';
+  }
+
+  const normalized = side.trim().toLocaleLowerCase('en-US');
+  if (
+    normalized === 'left' ||
+    normalized === 'l' ||
+    normalized === 'left hand' ||
+    normalized === 'left_hand' ||
+    normalized === 'left-hand' ||
+    normalized === '左' ||
+    normalized === '左手'
+  ) {
+    return 'left';
+  }
+
+  if (
+    normalized === 'right' ||
+    normalized === 'r' ||
+    normalized === 'right hand' ||
+    normalized === 'right_hand' ||
+    normalized === 'right-hand' ||
+    normalized === '右' ||
+    normalized === '右手'
+  ) {
+    return 'right';
+  }
+
+  return 'unknown';
+}
+
+function normalizeMocapAction(value: unknown) {
+  if (typeof value !== 'string') {
     return null;
   }
 
-  for (const hand of hands) {
-    const parsedHand = resolveHandLike(hand);
-    if (parsedHand) {
-      return parsedHand;
-    }
-  }
+  const normalized = value
+    .trim()
+    .toLocaleLowerCase('en-US')
+    .replace(/\s+/gu, '_')
+    .replace(/-/gu, '_');
 
-  return null;
+  return normalized || null;
 }
 
-function resolveHandLike(record: unknown): MocapParsedHand | null {
+function addMocapActions(actions: Set<string>, value: unknown) {
+  if (Array.isArray(value)) {
+    value.forEach((item) => addMocapActions(actions, item));
+    return;
+  }
+
+  if (value && typeof value === 'object') {
+    const actionRecord = value as Record<string, unknown>;
+    addMocapActions(actions, actionRecord.action);
+    addMocapActions(actions, actionRecord.actions);
+    addMocapActions(actions, actionRecord.gesture);
+    addMocapActions(actions, actionRecord.gestures);
+    addMocapActions(actions, actionRecord.event);
+    addMocapActions(actions, actionRecord.name);
+    addMocapActions(actions, actionRecord.type);
+    return;
+  }
+
+  const normalized = normalizeMocapAction(value);
+  if (normalized) {
+    actions.add(normalized);
+  }
+}
+
+function normaliseHandState(state: unknown): MocapHandState {
+  if (typeof state !== 'string') {
+    return 'unknown';
+  }
+
+  const normalized = state
+    .trim()
+    .toLocaleLowerCase('en-US')
+    .replace(/\s+/gu, '_')
+    .replace(/-/gu, '_');
+
+  if (
+    normalized === 'grab' ||
+    normalized === 'grabbing' ||
+    normalized === 'close' ||
+    normalized === 'fist' ||
+    normalized === 'closed_fist' ||
+    normalized === 'closed'
+  ) {
+    return 'grab';
+  }
+
+  if (
+    normalized === 'open_palm' ||
+    normalized === 'open_palm_up' ||
+    normalized === 'open' ||
+    normalized === 'palm' ||
+    normalized === 'hand_open'
+  ) {
+    return 'open_palm';
+  }
+
+  return 'unknown';
+}
+
+function resolveMocapHand(record: unknown, fallbackSide: MocapHandSide) {
   if (!record || typeof record !== 'object') {
     return null;
   }
 
-  const handRecord = record as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
+  const handRecord = record as {
+    state?: unknown;
+    landmarks?: unknown;
+    x?: unknown;
+    y?: unknown;
+    side?: unknown;
+    handedness?: unknown;
+    label?: unknown;
+    hand?: unknown;
+  };
   const state = normaliseHandState(handRecord.state);
+  const detectedSide = normaliseHandSide(
+    handRecord.side ??
+      handRecord.handedness ??
+      handRecord.label ??
+      handRecord.hand,
+  );
+  const side = detectedSide === 'unknown' ? fallbackSide : detectedSide;
+
   if (Array.isArray(handRecord.landmarks)) {
     const landmarks = handRecord.landmarks as Array<MocapLandmarkRecord>;
     const palmCenter = resolveMocapPalmCenter(landmarks);
     if (palmCenter) {
-      return { ...palmCenter, state, source: 'palm_center' };
+      return {...palmCenter, state, side, source: 'palm_center' as const};
     }
 
-    const landmark = landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
+    const landmark =
+      landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
     const fallbackPoint = resolveLandmarkCoordinate(landmark);
     if (fallbackPoint) {
-      return { ...fallbackPoint, state, source: 'landmark' };
+      return {...fallbackPoint, state, side, source: 'landmark' as const};
     }
   }
 
   const directX = normalizeCoordinate(handRecord.x);
   const directY = normalizeCoordinate(handRecord.y);
   if (directX !== null && directY !== null) {
-    return {x: directX, y: directY, state, source: 'direct'};
+    return {x: directX, y: directY, state, side, source: 'direct' as const};
   }
 
   return null;
 }
 
-function normaliseHandState(state: unknown): MocapHandState {
-  if (state === 'grab' || state === 'open_palm') {
-    return state;
+function resolveHands(packetRecord: Record<string, unknown>) {
+  const resolvedHands: MocapHandInput[] = [];
+  const packetHands = packetRecord.hands;
+  if (!Array.isArray(packetHands)) {
+    const leftHand = resolveMocapHand(
+      packetRecord.leftHand ?? packetRecord.left_hand,
+      'left',
+    );
+    const rightHand = resolveMocapHand(
+      packetRecord.rightHand ?? packetRecord.right_hand,
+      'right',
+    );
+    if (leftHand) {
+      resolvedHands.push(leftHand);
+    }
+    if (rightHand) {
+      resolvedHands.push(rightHand);
+    }
+    return resolvedHands;
   }
-  return 'unknown';
+
+  for (const hand of packetHands) {
+    const resolvedHand = resolveMocapHand(hand, 'unknown');
+    if (resolvedHand) {
+      resolvedHands.push(resolvedHand);
+    }
+  }
+
+  return resolvedHands;
+}
+
+function resolveBodyCenter(packetRecord: Record<string, unknown>) {
+  const generalRecord =
+    packetRecord.general && typeof packetRecord.general === 'object'
+      ? (packetRecord.general as Record<string, unknown>)
+      : null;
+  const bodyCandidates = [generalRecord?.body, packetRecord.body];
+
+  for (const bodyCandidate of bodyCandidates) {
+    if (!bodyCandidate || typeof bodyCandidate !== 'object') {
+      continue;
+    }
+
+    const bodyRecord = bodyCandidate as Record<string, unknown>;
+    const center = resolveNormalizedPoint(
+      bodyRecord.center_norm ?? bodyRecord.centerNorm ?? bodyRecord.center,
+    );
+    if (center) {
+      return center;
+    }
+  }
+
+  return null;
 }
 
 export function parseMocapPacket(packet: unknown): MocapInputCommand {
@@ -169,13 +359,24 @@ export function parseMocapPacket(packet: unknown): MocapInputCommand {
     return {actions: [], parseWarnings: ['packet 不是对象']};
   }
 
-  const packetRecord = packet as {hands?: unknown};
-  const primaryHand = resolvePrimaryHand(packetRecord.hands);
+  const packetRecord = packet as Record<string, unknown>;
+  const hands = resolveHands(packetRecord);
+  const primaryHand = hands[0] ?? null;
+  const leftHand = hands.find((hand) => hand.side === 'left') ?? null;
+  const rightHand = hands.find((hand) => hand.side === 'right') ?? null;
+  const bodyCenter = resolveBodyCenter(packetRecord);
   const actions = new Set<string>();
   const parseWarnings: string[] = [];
-  if (!Array.isArray(packetRecord.hands)) {
+  addMocapActions(actions, packetRecord.actions);
+  addMocapActions(actions, packetRecord.action);
+  addMocapActions(actions, packetRecord.gesture);
+  addMocapActions(actions, packetRecord.gestures);
+  addMocapActions(actions, packetRecord.event);
+  addMocapActions(actions, packetRecord.name);
+  addMocapActions(actions, packetRecord.type);
+  if (!Array.isArray(packetRecord.hands) && hands.length === 0 && !bodyCenter) {
     parseWarnings.push('缺少 hands 数组');
-  } else if (!primaryHand) {
+  } else if (!primaryHand && !bodyCenter) {
     parseWarnings.push('hands 中没有可用坐标');
   }
   if (primaryHand?.state === 'grab') {
@@ -184,13 +385,22 @@ export function parseMocapPacket(packet: unknown): MocapInputCommand {
   if (primaryHand?.state === 'open_palm') {
     actions.add('open_palm');
   }
+  for (const hand of hands) {
+    if (hand.state !== 'unknown') {
+      actions.add(hand.state);
+    }
+  }
   if (primaryHand && primaryHand.state === 'unknown') {
     parseWarnings.push('手势 state 未识别');
   }
 
   return {
     actions: Array.from(actions),
+    hands,
     primaryHand,
+    leftHand,
+    rightHand,
+    bodyCenter,
     parseWarnings,
   };
 }