- 摄像头暂不可用，已切换到本地演示
+ {motionSourceState !== 'ready' ? (
+
+ {motionSourceText}
) : null}
diff --git a/src/services/useMocapInput.ts b/src/services/useMocapInput.ts
index 27f7cecd..910fe1d2 100644
--- a/src/services/useMocapInput.ts
+++ b/src/services/useMocapInput.ts
@@ -3,15 +3,29 @@ import {useEffect, useMemo, useRef, useState} from 'react';
export type MocapConnectionStatus = 'idle' | 'connecting' | 'connected' | 'error';
export type MocapHandState = 'open_palm' | 'grab' | 'unknown';
+export type MocapHandSide = 'left' | 'right' | 'unknown';
+export type MocapHandSource = 'palm_center' | 'direct' | 'landmark';
+
+export type MocapHandInput = {
+ x: number;
+ y: number;
+ state: MocapHandState;
+ side: MocapHandSide;
+ source?: MocapHandSource;
+};
+
+export type MocapBodyCenterInput = {
+ x: number;
+ y: number;
+};
export type MocapInputCommand = {
actions: string[];
- primaryHand?: {
- x: number;
- y: number;
- state: MocapHandState;
- source?: 'palm_center' | 'direct' | 'landmark';
- } | null;
+ hands?: MocapHandInput[];
+ primaryHand?: MocapHandInput | null;
+ leftHand?: MocapHandInput | null;
+ rightHand?: MocapHandInput | null;
+ bodyCenter?: MocapBodyCenterInput | null;
parseWarnings?: string[];
};
@@ -33,13 +47,6 @@ export type UseMocapInputOptions = {
reconnectDelayMs?: number;
};
-type MocapParsedHand = {
- x: number;
- y: number;
- state: MocapHandState;
- source: 'palm_center' | 'direct' | 'landmark';
-};
-
type MocapLandmarkRecord = Record
;
const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876';
@@ -75,18 +82,42 @@ function normalizeCoordinate(value: unknown) {
return Math.min(1, Math.max(0, numericValue));
}
+function resolveNormalizedPoint(value: unknown) {
+ if (Array.isArray(value)) {
+ const x = normalizeCoordinate(value[0]);
+ const y = normalizeCoordinate(value[1]);
+ if (x === null || y === null) {
+ return null;
+ }
+ return {x, y};
+ }
+
+ if (!value || typeof value !== 'object') {
+ return null;
+ }
+
+ const pointRecord = value as {x?: unknown; y?: unknown};
+ const x = normalizeCoordinate(pointRecord.x);
+ const y = normalizeCoordinate(pointRecord.y);
+ if (x === null || y === null) {
+ return null;
+ }
+
+ return {x, y};
+}
+
function resolveLandmarkCoordinate(landmark: MocapLandmarkRecord | undefined) {
const x = normalizeCoordinate(landmark?.x);
const y = normalizeCoordinate(landmark?.y);
if (x === null || y === null) {
return null;
}
- return { x, y };
+ return {x, y};
}
export function resolveMocapPalmCenter(
landmarks: Array,
-): { x: number; y: number } | null {
+): {x: number; y: number} | null {
const landmarksByName = new Map(
landmarks
.filter((landmark) => typeof landmark?.name === 'string')
@@ -94,8 +125,10 @@ export function resolveMocapPalmCenter(
);
const weightedPoints = PALM_CENTER_WEIGHTS.map(([name, weight]) => {
const point = resolveLandmarkCoordinate(landmarksByName.get(name));
- return point ? { ...point, weight: Number(weight) } : null;
- }).filter((point): point is { x: number; y: number; weight: number } => Boolean(point));
+ return point ? {...point, weight: Number(weight)} : null;
+ }).filter((point): point is {x: number; y: number; weight: number} =>
+ Boolean(point),
+ );
if (weightedPoints.length < MIN_PALM_CENTER_POINT_COUNT) {
return null;
@@ -112,56 +145,213 @@ export function resolveMocapPalmCenter(
};
}
-function resolvePrimaryHand(hands: unknown): MocapParsedHand | null {
- if (!Array.isArray(hands)) {
+function normaliseHandSide(side: unknown): MocapHandSide {
+ if (typeof side !== 'string') {
+ return 'unknown';
+ }
+
+ const normalized = side.trim().toLocaleLowerCase('en-US');
+ if (
+ normalized === 'left' ||
+ normalized === 'l' ||
+ normalized === 'left hand' ||
+ normalized === 'left_hand' ||
+ normalized === 'left-hand' ||
+ normalized === '左' ||
+ normalized === '左手'
+ ) {
+ return 'left';
+ }
+
+ if (
+ normalized === 'right' ||
+ normalized === 'r' ||
+ normalized === 'right hand' ||
+ normalized === 'right_hand' ||
+ normalized === 'right-hand' ||
+ normalized === '右' ||
+ normalized === '右手'
+ ) {
+ return 'right';
+ }
+
+ return 'unknown';
+}
+
+function normalizeMocapAction(value: unknown) {
+ if (typeof value !== 'string') {
return null;
}
- for (const hand of hands) {
- const parsedHand = resolveHandLike(hand);
- if (parsedHand) {
- return parsedHand;
- }
- }
+ const normalized = value
+ .trim()
+ .toLocaleLowerCase('en-US')
+ .replace(/\s+/gu, '_')
+ .replace(/-/gu, '_');
- return null;
+ return normalized || null;
}
-function resolveHandLike(record: unknown): MocapParsedHand | null {
+function addMocapActions(actions: Set, value: unknown) {
+ if (Array.isArray(value)) {
+ value.forEach((item) => addMocapActions(actions, item));
+ return;
+ }
+
+ if (value && typeof value === 'object') {
+ const actionRecord = value as Record;
+ addMocapActions(actions, actionRecord.action);
+ addMocapActions(actions, actionRecord.actions);
+ addMocapActions(actions, actionRecord.gesture);
+ addMocapActions(actions, actionRecord.gestures);
+ addMocapActions(actions, actionRecord.event);
+ addMocapActions(actions, actionRecord.name);
+ addMocapActions(actions, actionRecord.type);
+ return;
+ }
+
+ const normalized = normalizeMocapAction(value);
+ if (normalized) {
+ actions.add(normalized);
+ }
+}
+
+function normaliseHandState(state: unknown): MocapHandState {
+ if (typeof state !== 'string') {
+ return 'unknown';
+ }
+
+ const normalized = state
+ .trim()
+ .toLocaleLowerCase('en-US')
+ .replace(/\s+/gu, '_')
+ .replace(/-/gu, '_');
+
+ if (
+ normalized === 'grab' ||
+ normalized === 'grabbing' ||
+ normalized === 'close' ||
+ normalized === 'fist' ||
+ normalized === 'closed_fist' ||
+ normalized === 'closed'
+ ) {
+ return 'grab';
+ }
+
+ if (
+ normalized === 'open_palm' ||
+ normalized === 'open_palm_up' ||
+ normalized === 'open' ||
+ normalized === 'palm' ||
+ normalized === 'hand_open'
+ ) {
+ return 'open_palm';
+ }
+
+ return 'unknown';
+}
+
+function resolveMocapHand(record: unknown, fallbackSide: MocapHandSide) {
if (!record || typeof record !== 'object') {
return null;
}
- const handRecord = record as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
+ const handRecord = record as {
+ state?: unknown;
+ landmarks?: unknown;
+ x?: unknown;
+ y?: unknown;
+ side?: unknown;
+ handedness?: unknown;
+ label?: unknown;
+ hand?: unknown;
+ };
const state = normaliseHandState(handRecord.state);
+ const detectedSide = normaliseHandSide(
+ handRecord.side ??
+ handRecord.handedness ??
+ handRecord.label ??
+ handRecord.hand,
+ );
+ const side = detectedSide === 'unknown' ? fallbackSide : detectedSide;
+
if (Array.isArray(handRecord.landmarks)) {
const landmarks = handRecord.landmarks as Array;
const palmCenter = resolveMocapPalmCenter(landmarks);
if (palmCenter) {
- return { ...palmCenter, state, source: 'palm_center' };
+ return {...palmCenter, state, side, source: 'palm_center' as const};
}
- const landmark = landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
+ const landmark =
+ landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
const fallbackPoint = resolveLandmarkCoordinate(landmark);
if (fallbackPoint) {
- return { ...fallbackPoint, state, source: 'landmark' };
+ return {...fallbackPoint, state, side, source: 'landmark' as const};
}
}
const directX = normalizeCoordinate(handRecord.x);
const directY = normalizeCoordinate(handRecord.y);
if (directX !== null && directY !== null) {
- return {x: directX, y: directY, state, source: 'direct'};
+ return {x: directX, y: directY, state, side, source: 'direct' as const};
}
return null;
}
-function normaliseHandState(state: unknown): MocapHandState {
- if (state === 'grab' || state === 'open_palm') {
- return state;
+function resolveHands(packetRecord: Record) {
+ const resolvedHands: MocapHandInput[] = [];
+ const packetHands = packetRecord.hands;
+ if (!Array.isArray(packetHands)) {
+ const leftHand = resolveMocapHand(
+ packetRecord.leftHand ?? packetRecord.left_hand,
+ 'left',
+ );
+ const rightHand = resolveMocapHand(
+ packetRecord.rightHand ?? packetRecord.right_hand,
+ 'right',
+ );
+ if (leftHand) {
+ resolvedHands.push(leftHand);
+ }
+ if (rightHand) {
+ resolvedHands.push(rightHand);
+ }
+ return resolvedHands;
}
- return 'unknown';
+
+ for (const hand of packetHands) {
+ const resolvedHand = resolveMocapHand(hand, 'unknown');
+ if (resolvedHand) {
+ resolvedHands.push(resolvedHand);
+ }
+ }
+
+ return resolvedHands;
+}
+
+function resolveBodyCenter(packetRecord: Record) {
+ const generalRecord =
+ packetRecord.general && typeof packetRecord.general === 'object'
+ ? (packetRecord.general as Record)
+ : null;
+ const bodyCandidates = [generalRecord?.body, packetRecord.body];
+
+ for (const bodyCandidate of bodyCandidates) {
+ if (!bodyCandidate || typeof bodyCandidate !== 'object') {
+ continue;
+ }
+
+ const bodyRecord = bodyCandidate as Record;
+ const center = resolveNormalizedPoint(
+ bodyRecord.center_norm ?? bodyRecord.centerNorm ?? bodyRecord.center,
+ );
+ if (center) {
+ return center;
+ }
+ }
+
+ return null;
}
export function parseMocapPacket(packet: unknown): MocapInputCommand {
@@ -169,13 +359,24 @@ export function parseMocapPacket(packet: unknown): MocapInputCommand {
return {actions: [], parseWarnings: ['packet 不是对象']};
}
- const packetRecord = packet as {hands?: unknown};
- const primaryHand = resolvePrimaryHand(packetRecord.hands);
+ const packetRecord = packet as Record;
+ const hands = resolveHands(packetRecord);
+ const primaryHand = hands[0] ?? null;
+ const leftHand = hands.find((hand) => hand.side === 'left') ?? null;
+ const rightHand = hands.find((hand) => hand.side === 'right') ?? null;
+ const bodyCenter = resolveBodyCenter(packetRecord);
const actions = new Set();
const parseWarnings: string[] = [];
- if (!Array.isArray(packetRecord.hands)) {
+ addMocapActions(actions, packetRecord.actions);
+ addMocapActions(actions, packetRecord.action);
+ addMocapActions(actions, packetRecord.gesture);
+ addMocapActions(actions, packetRecord.gestures);
+ addMocapActions(actions, packetRecord.event);
+ addMocapActions(actions, packetRecord.name);
+ addMocapActions(actions, packetRecord.type);
+ if (!Array.isArray(packetRecord.hands) && hands.length === 0 && !bodyCenter) {
parseWarnings.push('缺少 hands 数组');
- } else if (!primaryHand) {
+ } else if (!primaryHand && !bodyCenter) {
parseWarnings.push('hands 中没有可用坐标');
}
if (primaryHand?.state === 'grab') {
@@ -184,13 +385,22 @@ export function parseMocapPacket(packet: unknown): MocapInputCommand {
if (primaryHand?.state === 'open_palm') {
actions.add('open_palm');
}
+ for (const hand of hands) {
+ if (hand.state !== 'unknown') {
+ actions.add(hand.state);
+ }
+ }
if (primaryHand && primaryHand.state === 'unknown') {
parseWarnings.push('手势 state 未识别');
}
return {
actions: Array.from(actions),
+ hands,
primaryHand,
+ leftHand,
+ rightHand,
+ bodyCenter,
parseWarnings,
};
}