diff --git a/docs/technical/RUNTIME_INPUT_DEVICE_ABSTRACTION_2026-05-10.md b/docs/technical/RUNTIME_INPUT_DEVICE_ABSTRACTION_2026-05-10.md index 01dada71..d7e00cfc 100644 --- a/docs/technical/RUNTIME_INPUT_DEVICE_ABSTRACTION_2026-05-10.md +++ b/docs/technical/RUNTIME_INPUT_DEVICE_ABSTRACTION_2026-05-10.md @@ -17,11 +17,13 @@ ## 当前接入 +`useMocapInput` 解析 mocap `hands[].landmarks` 时应优先用 MediaPipe 21 点里的 `wrist / index_mcp / middle_mcp / ring_mcp / pinky_mcp` 加权计算掌心派生点；少于 3 个掌心关键点时才回退到 `wrist` 或直出 `hand.x/y`。这样运行态光标不会直接贴在腕部或指尖。 + 拼图运行态已接入该层： - 鼠标/触控 `pointerdown / pointermove / pointerup` 进入同一个 drag controller。 - mocap `grab` 进入同一个 drag controller，并强制使用持续拖拽语义。 -- mocap 松手时按当前棋盘归一坐标提交 drop。 +- mocap 光标按 60Hz 插值更新 UI 位置，并在拖拽中用插值后的当前点持续驱动输入层，避免输入包帧率低或抖动时出现明显跳变。 - 合并大块由拼图运行态把手部坐标命中到任一成员拼块；本地拼图运行时再按 `mergedGroupId` 执行整组平移。 ## 接入规则 @@ -39,7 +41,7 @@ 基础抽象层验证： ```bash -npm run test -- src\services\input-devices\runtimeDragInputController.test.ts +npm run test -- src\services\input-devices\runtimeDragInputController.test.ts src\services\useMocapInput.test.ts ``` 拼图接入验证： diff --git a/src/components/puzzle-runtime/PuzzleRuntimeShell.test.tsx b/src/components/puzzle-runtime/PuzzleRuntimeShell.test.tsx index 0201dc0c..f136f6f4 100644 --- a/src/components/puzzle-runtime/PuzzleRuntimeShell.test.tsx +++ b/src/components/puzzle-runtime/PuzzleRuntimeShell.test.tsx @@ -36,7 +36,7 @@ vi.mock('../../services/useMocapInput', () => ({ status: 'connected', latestCommand: { actions: [mocapMock.state], - primaryHand: {x: mocapMock.x, y: mocapMock.y, state: mocapMock.state}, + primaryHand: {x: mocapMock.x, y: mocapMock.y, state: mocapMock.state, source: 'palm_center'}, parseWarnings: [], }, rawPacketPreview: {text: '{"hands":[{"state":"grab"}]}', receivedAtMs: 1}, diff --git a/src/components/puzzle-runtime/PuzzleRuntimeShell.tsx b/src/components/puzzle-runtime/PuzzleRuntimeShell.tsx index bf276b2c..0e59b105 100644 --- a/src/components/puzzle-runtime/PuzzleRuntimeShell.tsx +++ b/src/components/puzzle-runtime/PuzzleRuntimeShell.tsx @@ -221,6 +221,7 @@ const PUZZLE_PIECE_PRESS_HAPTIC_PATTERN_MS = 12; const PUZZLE_EXIT_REMODEL_PROMPT_STORAGE_PREFIX = 'genarrative.puzzle-runtime.exit-remodel-prompt.v1'; const PUZZLE_MOCAP_DRAG_INPUT_ID = 'mocap:primary-hand'; +const PUZZLE_MOCAP_CURSOR_FRAME_MS = 1000 / 60; const shownExitRemodelPromptProfileIds = new Set(); @@ -300,6 +301,10 @@ type PuzzleMocapCursorState = { state: string; }; +type PuzzleMocapCursorSample = PuzzleMocapCursorState & { + receivedAtMs: number; +}; + type PuzzleRuntimeDragTargetState = { pieceId: string; groupId: string | null; @@ -394,6 +399,14 @@ export function PuzzleRuntimeShell({ const [mocapCursor, setMocapCursor] = useState( null, ); + const mocapCursorPreviousSampleRef = useRef( + null, + ); + const mocapCursorTargetSampleRef = useRef(null); + const mocapCursorIntervalRef = useRef(null); + const updateMocapCursorSampleRef = useRef<( + nextSample: PuzzleMocapCursorSample, + ) => void>(() => {}); const runtimeDragInputControllerRef = useRef( createRuntimeDragInputController(), ); @@ -930,6 +943,21 @@ export function PuzzleRuntimeShell({ readRuntimeInputElementBounds(boardRef.current), ); + const resetMocapCursorInterpolation = () => { + mocapCursorPreviousSampleRef.current = null; + mocapCursorTargetSampleRef.current = null; + setMocapCursor(null); + }; + + updateMocapCursorSampleRef.current = (nextSample: PuzzleMocapCursorSample) => { + const previousTarget = mocapCursorTargetSampleRef.current; + mocapCursorPreviousSampleRef.current = previousTarget ?? nextSample; + mocapCursorTargetSampleRef.current = nextSample; + if (!previousTarget) { + setMocapCursor(nextSample); + } + }; + const syncRuntimeDragFromController = ( session: RuntimeDragInputSession | null, ) => { @@ -994,7 +1022,7 @@ export function PuzzleRuntimeShell({ const activeSession = runtimeDragInputControllerRef.current.getSession(); if (!board || runtimeStatus !== 'playing' || isInteractionLocked) { runtimeDragInputControllerRef.current.cancel(); - setMocapCursor(null); + resetMocapCursorInterpolation(); return; } if ( @@ -1003,19 +1031,18 @@ export function PuzzleRuntimeShell({ typeof primaryMocapHandY !== 'number' ) { runtimeDragInputControllerRef.current.cancel(PUZZLE_MOCAP_DRAG_INPUT_ID); - setMocapCursor(null); + resetMocapCursorInterpolation(); return; } - setMocapCursor({ + const nextSample = { x: primaryMocapHandX, y: primaryMocapHandY, state: primaryMocapHandState, - }); - const handPoint = resolveBoardInputPointFromNormalized( - primaryMocapHandX, - primaryMocapHandY, - ); + receivedAtMs: performance.now(), + }; + updateMocapCursorSampleRef.current(nextSample); + const handPoint = resolveBoardInputPointFromNormalized(nextSample.x, nextSample.y); if (primaryMocapHandState === 'grab') { if (activeSession?.inputId !== PUZZLE_MOCAP_DRAG_INPUT_ID) { const sourceCell = resolveRuntimeInputGridCell(handPoint, board); @@ -1063,6 +1090,64 @@ export function PuzzleRuntimeShell({ runtimeStatus, ]); + useEffect(() => { + if (!board || runtimeStatus !== 'playing') { + if (mocapCursorIntervalRef.current !== null) { + window.clearInterval(mocapCursorIntervalRef.current); + mocapCursorIntervalRef.current = null; + } + return; + } + + const tickMocapCursor = () => { + const targetSample = mocapCursorTargetSampleRef.current; + if (!targetSample) { + return; + } + const previousSample = mocapCursorPreviousSampleRef.current ?? targetSample; + const durationMs = Math.max( + PUZZLE_MOCAP_CURSOR_FRAME_MS, + targetSample.receivedAtMs - previousSample.receivedAtMs, + ); + const progress = targetSample.receivedAtMs === previousSample.receivedAtMs + ? 1 + : Math.min( + 1, + Math.max(0, (performance.now() - targetSample.receivedAtMs) / durationMs), + ); + const nextCursor = { + x: previousSample.x + (targetSample.x - previousSample.x) * progress, + y: previousSample.y + (targetSample.y - previousSample.y) * progress, + state: targetSample.state, + }; + const nextPoint = resolveBoardInputPointFromNormalized( + nextCursor.x, + nextCursor.y, + ); + setMocapCursor(nextCursor); + const activeSession = runtimeDragInputControllerRef.current.getSession(); + if (activeSession?.inputId === PUZZLE_MOCAP_DRAG_INPUT_ID) { + runtimeDragInputControllerRef.current.move({ + inputId: PUZZLE_MOCAP_DRAG_INPUT_ID, + point: nextPoint, + forceDragging: true, + }); + } + }; + + tickMocapCursor(); + mocapCursorIntervalRef.current = window.setInterval( + tickMocapCursor, + PUZZLE_MOCAP_CURSOR_FRAME_MS, + ); + return () => { + if (mocapCursorIntervalRef.current !== null) { + window.clearInterval(mocapCursorIntervalRef.current); + mocapCursorIntervalRef.current = null; + } + }; + }, [board, runtimeStatus]); + if (!run || !currentLevel || !board) { return (
{ + test('优先用手腕和四个 MCP 点加权计算掌心派生点', () => { + const center = resolveMocapPalmCenter([ + { name: 'wrist', x: 0.1, y: 0.2 }, + { name: 'index_mcp', x: 0.3, y: 0.4 }, + { name: 'middle_mcp', x: 0.5, y: 0.6 }, + { name: 'ring_mcp', x: 0.7, y: 0.8 }, + { name: 'pinky_mcp', x: 0.9, y: 1 }, + { name: 'index_finger_tip', x: 1, y: 1 }, + ]); + + expect(center?.x).toBeCloseTo(0.44); + expect(center?.y).toBeCloseTo(0.54); + }); + + test('可用掌心点少于三个时不返回掌心坐标', () => { + expect( + resolveMocapPalmCenter([ + { name: 'wrist', x: 0.1, y: 0.2 }, + { name: 'index_mcp', x: 0.3, y: 0.4 }, + ]), + ).toBeNull(); + }); +}); + +describe('parseMocapPacket', () => { + test('解析手部数据时优先把 primaryHand 定位到掌心而不是腕部或指尖', () => { + const command = parseMocapPacket({ + hands: [ + { + state: 'open_palm', + x: 0.01, + y: 0.02, + landmarks: [ + { name: 'wrist', x: 0.1, y: 0.2 }, + { name: 'index_mcp', x: 0.3, y: 0.4 }, + { name: 'middle_mcp', x: 0.5, y: 0.6 }, + { name: 'ring_mcp', x: 0.7, y: 0.8 }, + { name: 'pinky_mcp', x: 0.9, y: 1 }, + ], + }, + ], + }); + + expect(command.primaryHand?.x).toBeCloseTo(0.44); + expect(command.primaryHand?.y).toBeCloseTo(0.54); + expect(command.primaryHand).toEqual( + expect.objectContaining({ + state: 'open_palm', + source: 'palm_center', + }), + ); + }); + + test('缺少足够掌心关键点时退回 wrist landmark，再退回 hand 直出坐标', () => { + const landmarkFallback = parseMocapPacket({ + hands: [ + { + state: 'grab', + x: 0.9, + y: 0.8, + landmarks: [{ name: 'wrist', x: 0.25, y: 0.75 }], + }, + ], + }); + expect(landmarkFallback.primaryHand).toEqual( + expect.objectContaining({x: 0.25, y: 0.75, source: 'landmark'}), + ); + + const directFallback = parseMocapPacket({ + hands: [{ state: 'grab', x: 0.9, y: 0.8 }], + }); + expect(directFallback.primaryHand).toEqual( + expect.objectContaining({x: 0.9, y: 0.8, source: 'direct'}), + ); + }); +}); diff --git a/src/services/useMocapInput.ts b/src/services/useMocapInput.ts index c20144b5..27f7cecd 100644 --- a/src/services/useMocapInput.ts +++ b/src/services/useMocapInput.ts @@ -10,6 +10,7 @@ export type MocapInputCommand = { x: number; y: number; state: MocapHandState; + source?: 'palm_center' | 'direct' | 'landmark'; } | null; parseWarnings?: string[]; }; @@ -32,9 +33,26 @@ export type UseMocapInputOptions = { reconnectDelayMs?: number; }; +type MocapParsedHand = { + x: number; + y: number; + state: MocapHandState; + source: 'palm_center' | 'direct' | 'landmark'; +}; + +type MocapLandmarkRecord = Record; + const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876'; const DEFAULT_RECONNECT_DELAY_MS = 1200; const MAX_RAW_PACKET_PREVIEW_LENGTH = 360; +const PALM_CENTER_WEIGHTS = [ + ['wrist', 0.25], + ['index_mcp', 0.2], + ['middle_mcp', 0.25], + ['ring_mcp', 0.2], + ['pinky_mcp', 0.1], +] as const; +const MIN_PALM_CENTER_POINT_COUNT = 3; function buildRawPacketPreview(rawData: unknown): string { const rawText = typeof rawData === 'string' ? rawData : JSON.stringify(rawData); @@ -57,68 +75,86 @@ function normalizeCoordinate(value: unknown) { return Math.min(1, Math.max(0, numericValue)); } -function resolvePrimaryHand(hands: unknown) { +function resolveLandmarkCoordinate(landmark: MocapLandmarkRecord | undefined) { + const x = normalizeCoordinate(landmark?.x); + const y = normalizeCoordinate(landmark?.y); + if (x === null || y === null) { + return null; + } + return { x, y }; +} + +export function resolveMocapPalmCenter( + landmarks: Array, +): { x: number; y: number } | null { + const landmarksByName = new Map( + landmarks + .filter((landmark) => typeof landmark?.name === 'string') + .map((landmark) => [String(landmark.name), landmark]), + ); + const weightedPoints = PALM_CENTER_WEIGHTS.map(([name, weight]) => { + const point = resolveLandmarkCoordinate(landmarksByName.get(name)); + return point ? { ...point, weight: Number(weight) } : null; + }).filter((point): point is { x: number; y: number; weight: number } => Boolean(point)); + + if (weightedPoints.length < MIN_PALM_CENTER_POINT_COUNT) { + return null; + } + + const weightTotal = weightedPoints.reduce((sum, point) => sum + point.weight, 0); + if (weightTotal <= 0) { + return null; + } + + return { + x: weightedPoints.reduce((sum, point) => sum + point.x * point.weight, 0) / weightTotal, + y: weightedPoints.reduce((sum, point) => sum + point.y * point.weight, 0) / weightTotal, + }; +} + +function resolvePrimaryHand(hands: unknown): MocapParsedHand | null { if (!Array.isArray(hands)) { return null; } for (const hand of hands) { - if (!hand || typeof hand !== 'object') { - continue; + const parsedHand = resolveHandLike(hand); + if (parsedHand) { + return parsedHand; } - - const handRecord = hand as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown}; - const state = normaliseHandState(handRecord.state); - const directX = normalizeCoordinate(handRecord.x); - const directY = normalizeCoordinate(handRecord.y); - if (directX !== null && directY !== null) { - return {x: directX, y: directY, state}; - } - if (!Array.isArray(handRecord.landmarks)) { - continue; - } - - const landmarks = handRecord.landmarks as Array>; - const landmark = - landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0]; - const x = normalizeCoordinate(landmark?.x); - const y = normalizeCoordinate(landmark?.y); - if (x === null || y === null) { - continue; - } - - return {x, y, state}; } return null; } -function resolveHandLike(record: unknown) { +function resolveHandLike(record: unknown): MocapParsedHand | null { if (!record || typeof record !== 'object') { return null; } const handRecord = record as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown}; const state = normaliseHandState(handRecord.state); + if (Array.isArray(handRecord.landmarks)) { + const landmarks = handRecord.landmarks as Array; + const palmCenter = resolveMocapPalmCenter(landmarks); + if (palmCenter) { + return { ...palmCenter, state, source: 'palm_center' }; + } + + const landmark = landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0]; + const fallbackPoint = resolveLandmarkCoordinate(landmark); + if (fallbackPoint) { + return { ...fallbackPoint, state, source: 'landmark' }; + } + } + const directX = normalizeCoordinate(handRecord.x); const directY = normalizeCoordinate(handRecord.y); if (directX !== null && directY !== null) { - return {x: directX, y: directY, state}; - } - if (!Array.isArray(handRecord.landmarks)) { - return null; + return {x: directX, y: directY, state, source: 'direct'}; } - const landmarks = handRecord.landmarks as Array>; - const landmark = - landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0]; - const x = normalizeCoordinate(landmark?.x); - const y = normalizeCoordinate(landmark?.y); - if (x === null || y === null) { - return null; - } - - return {x, y, state}; + return null; } function normaliseHandState(state: unknown): MocapHandState { @@ -128,7 +164,7 @@ function normaliseHandState(state: unknown): MocapHandState { return 'unknown'; } -function parseMocapPacket(packet: unknown): MocapInputCommand { +export function parseMocapPacket(packet: unknown): MocapInputCommand { if (!packet || typeof packet !== 'object') { return {actions: [], parseWarnings: ['packet 不是对象']}; }