491 lines
13 KiB
TypeScript
491 lines
13 KiB
TypeScript
import {useEffect, useMemo, useRef, useState} from 'react';
|
|
|
|
export type MocapConnectionStatus = 'idle' | 'connecting' | 'connected' | 'error';
|
|
|
|
export type MocapHandState = 'open_palm' | 'grab' | 'unknown';
|
|
export type MocapHandSide = 'left' | 'right' | 'unknown';
|
|
export type MocapHandSource = 'palm_center' | 'direct' | 'landmark';
|
|
|
|
export type MocapHandInput = {
|
|
x: number;
|
|
y: number;
|
|
state: MocapHandState;
|
|
side: MocapHandSide;
|
|
source?: MocapHandSource;
|
|
};
|
|
|
|
export type MocapBodyCenterInput = {
|
|
x: number;
|
|
y: number;
|
|
};
|
|
|
|
export type MocapInputCommand = {
|
|
actions: string[];
|
|
hands?: MocapHandInput[];
|
|
primaryHand?: MocapHandInput | null;
|
|
leftHand?: MocapHandInput | null;
|
|
rightHand?: MocapHandInput | null;
|
|
bodyCenter?: MocapBodyCenterInput | null;
|
|
parseWarnings?: string[];
|
|
};
|
|
|
|
export type MocapRawPacketPreview = {
|
|
text: string;
|
|
receivedAtMs: number;
|
|
};
|
|
|
|
export type UseMocapInputResult = {
|
|
status: MocapConnectionStatus;
|
|
latestCommand: MocapInputCommand | null;
|
|
rawPacketPreview: MocapRawPacketPreview | null;
|
|
error: string | null;
|
|
};
|
|
|
|
export type UseMocapInputOptions = {
|
|
enabled: boolean;
|
|
serviceUrl?: string;
|
|
reconnectDelayMs?: number;
|
|
};
|
|
|
|
type MocapLandmarkRecord = Record<string, unknown>;
|
|
|
|
const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876';
|
|
const DEFAULT_RECONNECT_DELAY_MS = 1200;
|
|
const MAX_RAW_PACKET_PREVIEW_LENGTH = 360;
|
|
const PALM_CENTER_WEIGHTS = [
|
|
['wrist', 0.25],
|
|
['index_mcp', 0.2],
|
|
['middle_mcp', 0.25],
|
|
['ring_mcp', 0.2],
|
|
['pinky_mcp', 0.1],
|
|
] as const;
|
|
const MIN_PALM_CENTER_POINT_COUNT = 3;
|
|
|
|
function buildRawPacketPreview(rawData: unknown): string {
|
|
const rawText = typeof rawData === 'string' ? rawData : JSON.stringify(rawData);
|
|
return rawText.length > MAX_RAW_PACKET_PREVIEW_LENGTH
|
|
? `${rawText.slice(0, MAX_RAW_PACKET_PREVIEW_LENGTH)}…`
|
|
: rawText;
|
|
}
|
|
|
|
function buildMocapStreamUrl(serviceUrl: string) {
|
|
const url = new URL('/stream', serviceUrl);
|
|
url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
|
|
return url.toString();
|
|
}
|
|
|
|
function normalizeCoordinate(value: unknown) {
|
|
const numericValue = Number(value);
|
|
if (!Number.isFinite(numericValue)) {
|
|
return null;
|
|
}
|
|
return Math.min(1, Math.max(0, numericValue));
|
|
}
|
|
|
|
function resolveNormalizedPoint(value: unknown) {
|
|
if (Array.isArray(value)) {
|
|
const x = normalizeCoordinate(value[0]);
|
|
const y = normalizeCoordinate(value[1]);
|
|
if (x === null || y === null) {
|
|
return null;
|
|
}
|
|
return {x, y};
|
|
}
|
|
|
|
if (!value || typeof value !== 'object') {
|
|
return null;
|
|
}
|
|
|
|
const pointRecord = value as {x?: unknown; y?: unknown};
|
|
const x = normalizeCoordinate(pointRecord.x);
|
|
const y = normalizeCoordinate(pointRecord.y);
|
|
if (x === null || y === null) {
|
|
return null;
|
|
}
|
|
|
|
return {x, y};
|
|
}
|
|
|
|
function resolveLandmarkCoordinate(landmark: MocapLandmarkRecord | undefined) {
|
|
const x = normalizeCoordinate(landmark?.x);
|
|
const y = normalizeCoordinate(landmark?.y);
|
|
if (x === null || y === null) {
|
|
return null;
|
|
}
|
|
return {x, y};
|
|
}
|
|
|
|
export function resolveMocapPalmCenter(
|
|
landmarks: Array<MocapLandmarkRecord>,
|
|
): {x: number; y: number} | null {
|
|
const landmarksByName = new Map(
|
|
landmarks
|
|
.filter((landmark) => typeof landmark?.name === 'string')
|
|
.map((landmark) => [String(landmark.name), landmark]),
|
|
);
|
|
const weightedPoints = PALM_CENTER_WEIGHTS.map(([name, weight]) => {
|
|
const point = resolveLandmarkCoordinate(landmarksByName.get(name));
|
|
return point ? {...point, weight: Number(weight)} : null;
|
|
}).filter((point): point is {x: number; y: number; weight: number} =>
|
|
Boolean(point),
|
|
);
|
|
|
|
if (weightedPoints.length < MIN_PALM_CENTER_POINT_COUNT) {
|
|
return null;
|
|
}
|
|
|
|
const weightTotal = weightedPoints.reduce((sum, point) => sum + point.weight, 0);
|
|
if (weightTotal <= 0) {
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
x: weightedPoints.reduce((sum, point) => sum + point.x * point.weight, 0) / weightTotal,
|
|
y: weightedPoints.reduce((sum, point) => sum + point.y * point.weight, 0) / weightTotal,
|
|
};
|
|
}
|
|
|
|
function normaliseHandSide(side: unknown): MocapHandSide {
|
|
if (typeof side !== 'string') {
|
|
return 'unknown';
|
|
}
|
|
|
|
const normalized = side.trim().toLocaleLowerCase('en-US');
|
|
if (
|
|
normalized === 'left' ||
|
|
normalized === 'l' ||
|
|
normalized === 'left hand' ||
|
|
normalized === 'left_hand' ||
|
|
normalized === 'left-hand' ||
|
|
normalized === '左' ||
|
|
normalized === '左手'
|
|
) {
|
|
return 'left';
|
|
}
|
|
|
|
if (
|
|
normalized === 'right' ||
|
|
normalized === 'r' ||
|
|
normalized === 'right hand' ||
|
|
normalized === 'right_hand' ||
|
|
normalized === 'right-hand' ||
|
|
normalized === '右' ||
|
|
normalized === '右手'
|
|
) {
|
|
return 'right';
|
|
}
|
|
|
|
return 'unknown';
|
|
}
|
|
|
|
function normalizeMocapAction(value: unknown) {
|
|
if (typeof value !== 'string') {
|
|
return null;
|
|
}
|
|
|
|
const normalized = value
|
|
.trim()
|
|
.toLocaleLowerCase('en-US')
|
|
.replace(/\s+/gu, '_')
|
|
.replace(/-/gu, '_');
|
|
|
|
return normalized || null;
|
|
}
|
|
|
|
function addMocapActions(actions: Set<string>, value: unknown) {
|
|
if (Array.isArray(value)) {
|
|
value.forEach((item) => addMocapActions(actions, item));
|
|
return;
|
|
}
|
|
|
|
if (value && typeof value === 'object') {
|
|
const actionRecord = value as Record<string, unknown>;
|
|
addMocapActions(actions, actionRecord.action);
|
|
addMocapActions(actions, actionRecord.actions);
|
|
addMocapActions(actions, actionRecord.gesture);
|
|
addMocapActions(actions, actionRecord.gestures);
|
|
addMocapActions(actions, actionRecord.event);
|
|
addMocapActions(actions, actionRecord.name);
|
|
addMocapActions(actions, actionRecord.type);
|
|
return;
|
|
}
|
|
|
|
const normalized = normalizeMocapAction(value);
|
|
if (normalized) {
|
|
actions.add(normalized);
|
|
}
|
|
}
|
|
|
|
function normaliseHandState(state: unknown): MocapHandState {
|
|
if (typeof state !== 'string') {
|
|
return 'unknown';
|
|
}
|
|
|
|
const normalized = state
|
|
.trim()
|
|
.toLocaleLowerCase('en-US')
|
|
.replace(/\s+/gu, '_')
|
|
.replace(/-/gu, '_');
|
|
|
|
if (
|
|
normalized === 'grab' ||
|
|
normalized === 'grabbing' ||
|
|
normalized === 'close' ||
|
|
normalized === 'fist' ||
|
|
normalized === 'closed_fist' ||
|
|
normalized === 'closed'
|
|
) {
|
|
return 'grab';
|
|
}
|
|
|
|
if (
|
|
normalized === 'open_palm' ||
|
|
normalized === 'open_palm_up' ||
|
|
normalized === 'open' ||
|
|
normalized === 'palm' ||
|
|
normalized === 'hand_open'
|
|
) {
|
|
return 'open_palm';
|
|
}
|
|
|
|
return 'unknown';
|
|
}
|
|
|
|
function resolveMocapHand(record: unknown, fallbackSide: MocapHandSide) {
|
|
if (!record || typeof record !== 'object') {
|
|
return null;
|
|
}
|
|
|
|
const handRecord = record as {
|
|
state?: unknown;
|
|
landmarks?: unknown;
|
|
x?: unknown;
|
|
y?: unknown;
|
|
side?: unknown;
|
|
handedness?: unknown;
|
|
label?: unknown;
|
|
hand?: unknown;
|
|
};
|
|
const state = normaliseHandState(handRecord.state);
|
|
const detectedSide = normaliseHandSide(
|
|
handRecord.side ??
|
|
handRecord.handedness ??
|
|
handRecord.label ??
|
|
handRecord.hand,
|
|
);
|
|
const side = detectedSide === 'unknown' ? fallbackSide : detectedSide;
|
|
|
|
if (Array.isArray(handRecord.landmarks)) {
|
|
const landmarks = handRecord.landmarks as Array<MocapLandmarkRecord>;
|
|
const palmCenter = resolveMocapPalmCenter(landmarks);
|
|
if (palmCenter) {
|
|
return {...palmCenter, state, side, source: 'palm_center' as const};
|
|
}
|
|
|
|
const landmark =
|
|
landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
|
|
const fallbackPoint = resolveLandmarkCoordinate(landmark);
|
|
if (fallbackPoint) {
|
|
return {...fallbackPoint, state, side, source: 'landmark' as const};
|
|
}
|
|
}
|
|
|
|
const directX = normalizeCoordinate(handRecord.x);
|
|
const directY = normalizeCoordinate(handRecord.y);
|
|
if (directX !== null && directY !== null) {
|
|
return {x: directX, y: directY, state, side, source: 'direct' as const};
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function resolveHands(packetRecord: Record<string, unknown>) {
|
|
const resolvedHands: MocapHandInput[] = [];
|
|
const packetHands = packetRecord.hands;
|
|
if (!Array.isArray(packetHands)) {
|
|
const leftHand = resolveMocapHand(
|
|
packetRecord.leftHand ?? packetRecord.left_hand,
|
|
'left',
|
|
);
|
|
const rightHand = resolveMocapHand(
|
|
packetRecord.rightHand ?? packetRecord.right_hand,
|
|
'right',
|
|
);
|
|
if (leftHand) {
|
|
resolvedHands.push(leftHand);
|
|
}
|
|
if (rightHand) {
|
|
resolvedHands.push(rightHand);
|
|
}
|
|
return resolvedHands;
|
|
}
|
|
|
|
for (const hand of packetHands) {
|
|
const resolvedHand = resolveMocapHand(hand, 'unknown');
|
|
if (resolvedHand) {
|
|
resolvedHands.push(resolvedHand);
|
|
}
|
|
}
|
|
|
|
return resolvedHands;
|
|
}
|
|
|
|
function resolveBodyCenter(packetRecord: Record<string, unknown>) {
|
|
const generalRecord =
|
|
packetRecord.general && typeof packetRecord.general === 'object'
|
|
? (packetRecord.general as Record<string, unknown>)
|
|
: null;
|
|
const bodyCandidates = [generalRecord?.body, packetRecord.body];
|
|
|
|
for (const bodyCandidate of bodyCandidates) {
|
|
if (!bodyCandidate || typeof bodyCandidate !== 'object') {
|
|
continue;
|
|
}
|
|
|
|
const bodyRecord = bodyCandidate as Record<string, unknown>;
|
|
const center = resolveNormalizedPoint(
|
|
bodyRecord.center_norm ?? bodyRecord.centerNorm ?? bodyRecord.center,
|
|
);
|
|
if (center) {
|
|
return center;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
export function parseMocapPacket(packet: unknown): MocapInputCommand {
|
|
if (!packet || typeof packet !== 'object') {
|
|
return {actions: [], parseWarnings: ['packet 不是对象']};
|
|
}
|
|
|
|
const packetRecord = packet as Record<string, unknown>;
|
|
const hands = resolveHands(packetRecord);
|
|
const primaryHand = hands[0] ?? null;
|
|
const leftHand = hands.find((hand) => hand.side === 'left') ?? null;
|
|
const rightHand = hands.find((hand) => hand.side === 'right') ?? null;
|
|
const bodyCenter = resolveBodyCenter(packetRecord);
|
|
const actions = new Set<string>();
|
|
const parseWarnings: string[] = [];
|
|
addMocapActions(actions, packetRecord.actions);
|
|
addMocapActions(actions, packetRecord.action);
|
|
addMocapActions(actions, packetRecord.gesture);
|
|
addMocapActions(actions, packetRecord.gestures);
|
|
addMocapActions(actions, packetRecord.event);
|
|
addMocapActions(actions, packetRecord.name);
|
|
addMocapActions(actions, packetRecord.type);
|
|
if (!Array.isArray(packetRecord.hands) && hands.length === 0 && !bodyCenter) {
|
|
parseWarnings.push('缺少 hands 数组');
|
|
} else if (!primaryHand && !bodyCenter) {
|
|
parseWarnings.push('hands 中没有可用坐标');
|
|
}
|
|
if (primaryHand?.state === 'grab') {
|
|
actions.add('grab');
|
|
}
|
|
if (primaryHand?.state === 'open_palm') {
|
|
actions.add('open_palm');
|
|
}
|
|
for (const hand of hands) {
|
|
if (hand.state !== 'unknown') {
|
|
actions.add(hand.state);
|
|
}
|
|
}
|
|
if (primaryHand && primaryHand.state === 'unknown') {
|
|
parseWarnings.push('手势 state 未识别');
|
|
}
|
|
|
|
return {
|
|
actions: Array.from(actions),
|
|
hands,
|
|
primaryHand,
|
|
leftHand,
|
|
rightHand,
|
|
bodyCenter,
|
|
parseWarnings,
|
|
};
|
|
}
|
|
|
|
export function useMocapInput({
|
|
enabled,
|
|
serviceUrl = DEFAULT_MOCAP_SERVICE_URL,
|
|
reconnectDelayMs = DEFAULT_RECONNECT_DELAY_MS,
|
|
}: UseMocapInputOptions): UseMocapInputResult {
|
|
const [status, setStatus] = useState<MocapConnectionStatus>('idle');
|
|
const [latestCommand, setLatestCommand] = useState<MocapInputCommand | null>(
|
|
null,
|
|
);
|
|
const [rawPacketPreview, setRawPacketPreview] = useState<MocapRawPacketPreview | null>(
|
|
null,
|
|
);
|
|
const [error, setError] = useState<string | null>(null);
|
|
const reconnectTimerRef = useRef<number | null>(null);
|
|
const websocketRef = useRef<WebSocket | null>(null);
|
|
|
|
const streamUrl = useMemo(() => buildMocapStreamUrl(serviceUrl), [serviceUrl]);
|
|
|
|
useEffect(() => {
|
|
if (!enabled || typeof WebSocket === 'undefined') {
|
|
setStatus('idle');
|
|
return;
|
|
}
|
|
|
|
let cancelled = false;
|
|
|
|
const connect = () => {
|
|
if (cancelled) {
|
|
return;
|
|
}
|
|
setStatus('connecting');
|
|
setError(null);
|
|
|
|
const websocket = new WebSocket(streamUrl);
|
|
websocketRef.current = websocket;
|
|
websocket.onopen = () => {
|
|
if (!cancelled) {
|
|
setStatus('connected');
|
|
}
|
|
};
|
|
websocket.onmessage = (event) => {
|
|
if (cancelled) {
|
|
return;
|
|
}
|
|
try {
|
|
const rawText = String(event.data);
|
|
setRawPacketPreview({
|
|
text: buildRawPacketPreview(rawText),
|
|
receivedAtMs: Date.now(),
|
|
});
|
|
setLatestCommand(parseMocapPacket(JSON.parse(rawText)));
|
|
} catch (parseError) {
|
|
setError(parseError instanceof Error ? parseError.message : 'mocap 数据解析失败');
|
|
}
|
|
};
|
|
websocket.onerror = () => {
|
|
if (!cancelled) {
|
|
setStatus('error');
|
|
setError('mocap 连接异常');
|
|
}
|
|
};
|
|
websocket.onclose = () => {
|
|
if (cancelled) {
|
|
return;
|
|
}
|
|
setStatus('error');
|
|
reconnectTimerRef.current = window.setTimeout(connect, reconnectDelayMs);
|
|
};
|
|
};
|
|
|
|
connect();
|
|
|
|
return () => {
|
|
cancelled = true;
|
|
if (reconnectTimerRef.current !== null) {
|
|
window.clearTimeout(reconnectTimerRef.current);
|
|
}
|
|
websocketRef.current?.close();
|
|
};
|
|
}, [enabled, reconnectDelayMs, streamUrl]);
|
|
|
|
return {status, latestCommand, rawPacketPreview, error};
|
|
}
|