GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: smooth mocap palm cursor

Browse Source
This commit is contained in:
kdletters
2026-05-10 18:51:43 +08:00
parent 46a254f142
commit 75bca28191
5 changed files with 256 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
src/services/useMocapInput.ts
View File

@@ -10,6 +10,7 @@ export type MocapInputCommand = {
x: number;
y: number;
state: MocapHandState;
source?: 'palm_center' | 'direct' | 'landmark';
} | null;
parseWarnings?: string[];
};
@@ -32,9 +33,26 @@ export type UseMocapInputOptions = {
reconnectDelayMs?: number;
};
type MocapParsedHand = {
x: number;
y: number;
state: MocapHandState;
source: 'palm_center' | 'direct' | 'landmark';
};
type MocapLandmarkRecord = Record<string, unknown>;
const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876';
const DEFAULT_RECONNECT_DELAY_MS = 1200;
const MAX_RAW_PACKET_PREVIEW_LENGTH = 360;
const PALM_CENTER_WEIGHTS = [
['wrist', 0.25],
['index_mcp', 0.2],
['middle_mcp', 0.25],
['ring_mcp', 0.2],
['pinky_mcp', 0.1],
] as const;
const MIN_PALM_CENTER_POINT_COUNT = 3;
function buildRawPacketPreview(rawData: unknown): string {
const rawText = typeof rawData === 'string' ? rawData : JSON.stringify(rawData);
@@ -57,68 +75,86 @@ function normalizeCoordinate(value: unknown) {
return Math.min(1, Math.max(0, numericValue));
}
function resolvePrimaryHand(hands: unknown) {
function resolveLandmarkCoordinate(landmark: MocapLandmarkRecord | undefined) {
const x = normalizeCoordinate(landmark?.x);
const y = normalizeCoordinate(landmark?.y);
if (x === null || y === null) {
return null;
}
return { x, y };
}
export function resolveMocapPalmCenter(
landmarks: Array<MocapLandmarkRecord>,
): { x: number; y: number } | null {
const landmarksByName = new Map(
landmarks
.filter((landmark) => typeof landmark?.name === 'string')
.map((landmark) => [String(landmark.name), landmark]),
);
const weightedPoints = PALM_CENTER_WEIGHTS.map(([name, weight]) => {
const point = resolveLandmarkCoordinate(landmarksByName.get(name));
return point ? { ...point, weight: Number(weight) } : null;
}).filter((point): point is { x: number; y: number; weight: number } => Boolean(point));
if (weightedPoints.length < MIN_PALM_CENTER_POINT_COUNT) {
return null;
}
const weightTotal = weightedPoints.reduce((sum, point) => sum + point.weight, 0);
if (weightTotal <= 0) {
return null;
}
return {
x: weightedPoints.reduce((sum, point) => sum + point.x * point.weight, 0) / weightTotal,
y: weightedPoints.reduce((sum, point) => sum + point.y * point.weight, 0) / weightTotal,
};
}
function resolvePrimaryHand(hands: unknown): MocapParsedHand | null {
if (!Array.isArray(hands)) {
return null;
}
for (const hand of hands) {
if (!hand || typeof hand !== 'object') {
continue;
const parsedHand = resolveHandLike(hand);
if (parsedHand) {
return parsedHand;
}
const handRecord = hand as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
const state = normaliseHandState(handRecord.state);
const directX = normalizeCoordinate(handRecord.x);
const directY = normalizeCoordinate(handRecord.y);
if (directX !== null && directY !== null) {
return {x: directX, y: directY, state};
}
if (!Array.isArray(handRecord.landmarks)) {
continue;
}
const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
const landmark =
landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
const x = normalizeCoordinate(landmark?.x);
const y = normalizeCoordinate(landmark?.y);
if (x === null || y === null) {
continue;
}
return {x, y, state};
}
return null;
}
function resolveHandLike(record: unknown) {
function resolveHandLike(record: unknown): MocapParsedHand | null {
if (!record || typeof record !== 'object') {
return null;
}
const handRecord = record as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
const state = normaliseHandState(handRecord.state);
if (Array.isArray(handRecord.landmarks)) {
const landmarks = handRecord.landmarks as Array<MocapLandmarkRecord>;
const palmCenter = resolveMocapPalmCenter(landmarks);
if (palmCenter) {
return { ...palmCenter, state, source: 'palm_center' };
}
const landmark = landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
const fallbackPoint = resolveLandmarkCoordinate(landmark);
if (fallbackPoint) {
return { ...fallbackPoint, state, source: 'landmark' };
}
}
const directX = normalizeCoordinate(handRecord.x);
const directY = normalizeCoordinate(handRecord.y);
if (directX !== null && directY !== null) {
return {x: directX, y: directY, state};
}
if (!Array.isArray(handRecord.landmarks)) {
return null;
return {x: directX, y: directY, state, source: 'direct'};
}
const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
const landmark =
landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
const x = normalizeCoordinate(landmark?.x);
const y = normalizeCoordinate(landmark?.y);
if (x === null || y === null) {
return null;
}
return {x, y, state};
return null;
}
function normaliseHandState(state: unknown): MocapHandState {
@@ -128,7 +164,7 @@ function normaliseHandState(state: unknown): MocapHandState {
return 'unknown';
}
function parseMocapPacket(packet: unknown): MocapInputCommand {
export function parseMocapPacket(packet: unknown): MocapInputCommand {
if (!packet || typeof packet !== 'object') {
return {actions: [], parseWarnings: ['packet 不是对象']};
}