feat: smooth mocap palm cursor

src/services/useMocapInput.test.ts

@@ -0,0 +1,81 @@
+import { describe, expect, test } from 'vitest';
+
+import { parseMocapPacket, resolveMocapPalmCenter } from './useMocapInput';
+
+describe('resolveMocapPalmCenter', () => {
+  test('优先用手腕和四个 MCP 点加权计算掌心派生点', () => {
+    const center = resolveMocapPalmCenter([
+      { name: 'wrist', x: 0.1, y: 0.2 },
+      { name: 'index_mcp', x: 0.3, y: 0.4 },
+      { name: 'middle_mcp', x: 0.5, y: 0.6 },
+      { name: 'ring_mcp', x: 0.7, y: 0.8 },
+      { name: 'pinky_mcp', x: 0.9, y: 1 },
+      { name: 'index_finger_tip', x: 1, y: 1 },
+    ]);
+
+    expect(center?.x).toBeCloseTo(0.44);
+    expect(center?.y).toBeCloseTo(0.54);
+  });
+
+  test('可用掌心点少于三个时不返回掌心坐标', () => {
+    expect(
+      resolveMocapPalmCenter([
+        { name: 'wrist', x: 0.1, y: 0.2 },
+        { name: 'index_mcp', x: 0.3, y: 0.4 },
+      ]),
+    ).toBeNull();
+  });
+});
+
+describe('parseMocapPacket', () => {
+  test('解析手部数据时优先把 primaryHand 定位到掌心而不是腕部或指尖', () => {
+    const command = parseMocapPacket({
+      hands: [
+        {
+          state: 'open_palm',
+          x: 0.01,
+          y: 0.02,
+          landmarks: [
+            { name: 'wrist', x: 0.1, y: 0.2 },
+            { name: 'index_mcp', x: 0.3, y: 0.4 },
+            { name: 'middle_mcp', x: 0.5, y: 0.6 },
+            { name: 'ring_mcp', x: 0.7, y: 0.8 },
+            { name: 'pinky_mcp', x: 0.9, y: 1 },
+          ],
+        },
+      ],
+    });
+
+    expect(command.primaryHand?.x).toBeCloseTo(0.44);
+    expect(command.primaryHand?.y).toBeCloseTo(0.54);
+    expect(command.primaryHand).toEqual(
+      expect.objectContaining({
+        state: 'open_palm',
+        source: 'palm_center',
+      }),
+    );
+  });
+
+  test('缺少足够掌心关键点时退回 wrist landmark，再退回 hand 直出坐标', () => {
+    const landmarkFallback = parseMocapPacket({
+      hands: [
+        {
+          state: 'grab',
+          x: 0.9,
+          y: 0.8,
+          landmarks: [{ name: 'wrist', x: 0.25, y: 0.75 }],
+        },
+      ],
+    });
+    expect(landmarkFallback.primaryHand).toEqual(
+      expect.objectContaining({x: 0.25, y: 0.75, source: 'landmark'}),
+    );
+
+    const directFallback = parseMocapPacket({
+      hands: [{ state: 'grab', x: 0.9, y: 0.8 }],
+    });
+    expect(directFallback.primaryHand).toEqual(
+      expect.objectContaining({x: 0.9, y: 0.8, source: 'direct'}),
+    );
+  });
+});

src/services/useMocapInput.ts

@@ -10,6 +10,7 @@ export type MocapInputCommand = {
     x: number;
     y: number;
     state: MocapHandState;
+    source?: 'palm_center' | 'direct' | 'landmark';
   } | null;
   parseWarnings?: string[];
 };
@@ -32,9 +33,26 @@ export type UseMocapInputOptions = {
   reconnectDelayMs?: number;
 };
 
+type MocapParsedHand = {
+  x: number;
+  y: number;
+  state: MocapHandState;
+  source: 'palm_center' | 'direct' | 'landmark';
+};
+
+type MocapLandmarkRecord = Record<string, unknown>;
+
 const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876';
 const DEFAULT_RECONNECT_DELAY_MS = 1200;
 const MAX_RAW_PACKET_PREVIEW_LENGTH = 360;
+const PALM_CENTER_WEIGHTS = [
+  ['wrist', 0.25],
+  ['index_mcp', 0.2],
+  ['middle_mcp', 0.25],
+  ['ring_mcp', 0.2],
+  ['pinky_mcp', 0.1],
+] as const;
+const MIN_PALM_CENTER_POINT_COUNT = 3;
 
 function buildRawPacketPreview(rawData: unknown): string {
   const rawText = typeof rawData === 'string' ? rawData : JSON.stringify(rawData);
@@ -57,68 +75,86 @@ function normalizeCoordinate(value: unknown) {
   return Math.min(1, Math.max(0, numericValue));
 }
 
-function resolvePrimaryHand(hands: unknown) {
+function resolveLandmarkCoordinate(landmark: MocapLandmarkRecord | undefined) {
+  const x = normalizeCoordinate(landmark?.x);
+  const y = normalizeCoordinate(landmark?.y);
+  if (x === null || y === null) {
+    return null;
+  }
+  return { x, y };
+}
+
+export function resolveMocapPalmCenter(
+  landmarks: Array<MocapLandmarkRecord>,
+): { x: number; y: number } | null {
+  const landmarksByName = new Map(
+    landmarks
+      .filter((landmark) => typeof landmark?.name === 'string')
+      .map((landmark) => [String(landmark.name), landmark]),
+  );
+  const weightedPoints = PALM_CENTER_WEIGHTS.map(([name, weight]) => {
+    const point = resolveLandmarkCoordinate(landmarksByName.get(name));
+    return point ? { ...point, weight: Number(weight) } : null;
+  }).filter((point): point is { x: number; y: number; weight: number } => Boolean(point));
+
+  if (weightedPoints.length < MIN_PALM_CENTER_POINT_COUNT) {
+    return null;
+  }
+
+  const weightTotal = weightedPoints.reduce((sum, point) => sum + point.weight, 0);
+  if (weightTotal <= 0) {
+    return null;
+  }
+
+  return {
+    x: weightedPoints.reduce((sum, point) => sum + point.x * point.weight, 0) / weightTotal,
+    y: weightedPoints.reduce((sum, point) => sum + point.y * point.weight, 0) / weightTotal,
+  };
+}
+
+function resolvePrimaryHand(hands: unknown): MocapParsedHand | null {
   if (!Array.isArray(hands)) {
     return null;
   }
 
   for (const hand of hands) {
-    if (!hand || typeof hand !== 'object') {
-      continue;
+    const parsedHand = resolveHandLike(hand);
+    if (parsedHand) {
+      return parsedHand;
     }
-
-    const handRecord = hand as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
-    const state = normaliseHandState(handRecord.state);
-    const directX = normalizeCoordinate(handRecord.x);
-    const directY = normalizeCoordinate(handRecord.y);
-    if (directX !== null && directY !== null) {
-      return {x: directX, y: directY, state};
-    }
-    if (!Array.isArray(handRecord.landmarks)) {
-      continue;
-    }
-
-    const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
-    const landmark =
-      landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
-    const x = normalizeCoordinate(landmark?.x);
-    const y = normalizeCoordinate(landmark?.y);
-    if (x === null || y === null) {
-      continue;
-    }
-
-    return {x, y, state};
   }
 
   return null;
 }
 
-function resolveHandLike(record: unknown) {
+function resolveHandLike(record: unknown): MocapParsedHand | null {
   if (!record || typeof record !== 'object') {
     return null;
   }
 
   const handRecord = record as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
   const state = normaliseHandState(handRecord.state);
+  if (Array.isArray(handRecord.landmarks)) {
+    const landmarks = handRecord.landmarks as Array<MocapLandmarkRecord>;
+    const palmCenter = resolveMocapPalmCenter(landmarks);
+    if (palmCenter) {
+      return { ...palmCenter, state, source: 'palm_center' };
+    }
+
+    const landmark = landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
+    const fallbackPoint = resolveLandmarkCoordinate(landmark);
+    if (fallbackPoint) {
+      return { ...fallbackPoint, state, source: 'landmark' };
+    }
+  }
+
   const directX = normalizeCoordinate(handRecord.x);
   const directY = normalizeCoordinate(handRecord.y);
   if (directX !== null && directY !== null) {
-    return {x: directX, y: directY, state};
-  }
-  if (!Array.isArray(handRecord.landmarks)) {
-    return null;
+    return {x: directX, y: directY, state, source: 'direct'};
   }
 
-  const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
-  const landmark =
-    landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
-  const x = normalizeCoordinate(landmark?.x);
-  const y = normalizeCoordinate(landmark?.y);
-  if (x === null || y === null) {
-    return null;
-  }
-
-  return {x, y, state};
+  return null;
 }
 
 function normaliseHandState(state: unknown): MocapHandState {
@@ -128,7 +164,7 @@ function normaliseHandState(state: unknown): MocapHandState {
   return 'unknown';
 }
 
-function parseMocapPacket(packet: unknown): MocapInputCommand {
+export function parseMocapPacket(packet: unknown): MocapInputCommand {
   if (!packet || typeof packet !== 'object') {
     return {actions: [], parseWarnings: ['packet 不是对象']};
   }