feat: add edutainment drawing and visual package flows

src/services/useMocapInput.ts

@@ -6,17 +6,27 @@ export type MocapHandState = 'open_palm' | 'grab' | 'unknown';
 export type MocapHandSide = 'left' | 'right' | 'unknown';
 export type MocapHandSource = 'palm_center' | 'direct' | 'landmark';
 
+export type MocapPointInput = {
+  x: number;
+  y: number;
+};
+
 export type MocapHandInput = {
   x: number;
   y: number;
   state: MocapHandState;
   side: MocapHandSide;
   source?: MocapHandSource;
+  wrist?: MocapPointInput | null;
 };
 
-export type MocapBodyCenterInput = {
-  x: number;
-  y: number;
+export type MocapBodyCenterInput = MocapPointInput;
+
+export type MocapBodyJointsInput = {
+  leftShoulder?: MocapPointInput | null;
+  rightShoulder?: MocapPointInput | null;
+  leftElbow?: MocapPointInput | null;
+  rightElbow?: MocapPointInput | null;
 };
 
 export type MocapInputCommand = {
@@ -26,6 +36,7 @@ export type MocapInputCommand = {
   leftHand?: MocapHandInput | null;
   rightHand?: MocapHandInput | null;
   bodyCenter?: MocapBodyCenterInput | null;
+  bodyJoints?: MocapBodyJointsInput;
   parseWarnings?: string[];
 };
 
@@ -251,6 +262,36 @@ function normaliseHandState(state: unknown): MocapHandState {
   return 'unknown';
 }
 
+function normalizeBodyJointName(name: unknown) {
+  if (typeof name !== 'string') {
+    return null;
+  }
+
+  const normalized = name
+    .trim()
+    .toLocaleLowerCase('en-US')
+    .replace(/\s+/gu, '_')
+    .replace(/-/gu, '_');
+
+  if (normalized === 'left_shoulder' || normalized === 'leftshoulder') {
+    return 'leftShoulder' as const;
+  }
+
+  if (normalized === 'right_shoulder' || normalized === 'rightshoulder') {
+    return 'rightShoulder' as const;
+  }
+
+  if (normalized === 'left_elbow' || normalized === 'leftelbow') {
+    return 'leftElbow' as const;
+  }
+
+  if (normalized === 'right_elbow' || normalized === 'rightelbow') {
+    return 'rightElbow' as const;
+  }
+
+  return null;
+}
+
 function resolveMocapHand(record: unknown, fallbackSide: MocapHandSide) {
   if (!record || typeof record !== 'object') {
     return null;
@@ -277,23 +318,45 @@ function resolveMocapHand(record: unknown, fallbackSide: MocapHandSide) {
 
   if (Array.isArray(handRecord.landmarks)) {
     const landmarks = handRecord.landmarks as Array<MocapLandmarkRecord>;
+    const wristPoint = resolveLandmarkCoordinate(
+      landmarks.find((item) => item?.name === 'wrist'),
+    );
     const palmCenter = resolveMocapPalmCenter(landmarks);
     if (palmCenter) {
-      return {...palmCenter, state, side, source: 'palm_center' as const};
+      return {
+        ...palmCenter,
+        state,
+        side,
+        source: 'palm_center' as const,
+        wrist: wristPoint ?? palmCenter,
+      };
     }
 
     const landmark =
       landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
     const fallbackPoint = resolveLandmarkCoordinate(landmark);
     if (fallbackPoint) {
-      return {...fallbackPoint, state, side, source: 'landmark' as const};
+      return {
+        ...fallbackPoint,
+        state,
+        side,
+        source: 'landmark' as const,
+        wrist: wristPoint ?? fallbackPoint,
+      };
     }
   }
 
   const directX = normalizeCoordinate(handRecord.x);
   const directY = normalizeCoordinate(handRecord.y);
   if (directX !== null && directY !== null) {
-    return {x: directX, y: directY, state, side, source: 'direct' as const};
+    return {
+      x: directX,
+      y: directY,
+      state,
+      side,
+      source: 'direct' as const,
+      wrist: {x: directX, y: directY},
+    };
   }
 
   return null;
@@ -354,6 +417,53 @@ function resolveBodyCenter(packetRecord: Record<string, unknown>) {
   return null;
 }
 
+function resolveBodyJoints(packetRecord: Record<string, unknown>) {
+  const joints: MocapBodyJointsInput = {};
+  const generalRecord =
+    packetRecord.general && typeof packetRecord.general === 'object'
+      ? (packetRecord.general as Record<string, unknown>)
+      : null;
+  const bodyRecord =
+    generalRecord?.body && typeof generalRecord.body === 'object'
+      ? (generalRecord.body as Record<string, unknown>)
+      : null;
+  const limbCandidates = [
+    generalRecord?.limb_nodes,
+    generalRecord?.limbNodes,
+    bodyRecord?.limb_nodes,
+    bodyRecord?.limbNodes,
+    packetRecord.limb_nodes,
+    packetRecord.limbNodes,
+  ];
+
+  for (const candidate of limbCandidates) {
+    if (!Array.isArray(candidate)) {
+      continue;
+    }
+
+    for (const node of candidate) {
+      if (!node || typeof node !== 'object') {
+        continue;
+      }
+
+      const nodeRecord = node as Record<string, unknown>;
+      const jointName = normalizeBodyJointName(
+        nodeRecord.name ?? nodeRecord.label ?? nodeRecord.type,
+      );
+      if (!jointName || joints[jointName]) {
+        continue;
+      }
+
+      const point = resolveNormalizedPoint(nodeRecord);
+      if (point) {
+        joints[jointName] = point;
+      }
+    }
+  }
+
+  return Object.keys(joints).length > 0 ? joints : undefined;
+}
+
 export function parseMocapPacket(packet: unknown): MocapInputCommand {
   if (!packet || typeof packet !== 'object') {
     return {actions: [], parseWarnings: ['packet 不是对象']};
@@ -365,6 +475,7 @@ export function parseMocapPacket(packet: unknown): MocapInputCommand {
   const leftHand = hands.find((hand) => hand.side === 'left') ?? null;
   const rightHand = hands.find((hand) => hand.side === 'right') ?? null;
   const bodyCenter = resolveBodyCenter(packetRecord);
+  const bodyJoints = resolveBodyJoints(packetRecord);
   const actions = new Set<string>();
   const parseWarnings: string[] = [];
   addMocapActions(actions, packetRecord.actions);
@@ -401,6 +512,7 @@ export function parseMocapPacket(packet: unknown): MocapInputCommand {
     leftHand,
     rightHand,
     bodyCenter,
+    bodyJoints,
     parseWarnings,
   };
 }