Merge branch 'master' of http://82.157.175.59:3000/GenarrativeAI/Genarrative

src/services/child-motion-demo/childMotionDebugInput.test.ts

@@ -0,0 +1,263 @@
+// @vitest-environment jsdom
+
+import { afterEach, describe, expect, test, vi } from 'vitest';
+
+import {
+  type ChildMotionDebugAction,
+  createChildMotionDebugInputController,
+  resolveKeyboardDebugAction,
+} from './childMotionDebugInput';
+
+let mountedTargets: HTMLElement[] = [];
+
+afterEach(() => {
+  mountedTargets.forEach((target) => target.remove());
+  mountedTargets = [];
+});
+
+function createTarget() {
+  const target = document.createElement('div');
+  document.body.appendChild(target);
+  mountedTargets.push(target);
+  return target;
+}
+
+function dispatchKeyboard(
+  target: HTMLElement,
+  options: { key: string; code?: string; repeat?: boolean },
+) {
+  const event = new KeyboardEvent('keydown', {
+    bubbles: true,
+    cancelable: true,
+    key: options.key,
+    code: options.code ?? '',
+    repeat: options.repeat ?? false,
+  });
+  target.dispatchEvent(event);
+  return event;
+}
+
+function dispatchPointer(
+  target: HTMLElement,
+  type: string,
+  options: {
+    button?: number;
+    clientX: number;
+    clientY: number;
+    pointerId?: number;
+  },
+) {
+  const event = new MouseEvent(type, {
+    bubbles: true,
+    cancelable: true,
+    button: options.button ?? 0,
+    clientX: options.clientX,
+    clientY: options.clientY,
+  });
+  Object.assign(event, { pointerId: options.pointerId ?? 1 });
+  target.dispatchEvent(event);
+  return event;
+}
+
+describe('childMotionDebugInput', () => {
+  test('maps A, D and Space keys to movement and jump actions', () => {
+    const target = createTarget();
+    const actions: ChildMotionDebugAction[] = [];
+    const controller = createChildMotionDebugInputController({
+      target,
+      onAction: (action) => actions.push(action),
+      now: () => 120,
+    });
+
+    const leftEvent = dispatchKeyboard(target, { key: 'a', code: 'KeyA' });
+    dispatchKeyboard(target, { key: 'D', code: 'KeyD' });
+    dispatchKeyboard(target, { key: ' ', code: 'Space' });
+
+    expect(leftEvent.defaultPrevented).toBe(true);
+    expect(actions).toEqual([
+      {
+        kind: 'move',
+        direction: 'left',
+        source: 'keyboard',
+        occurredAtMs: 120,
+      },
+      {
+        kind: 'move',
+        direction: 'right',
+        source: 'keyboard',
+        occurredAtMs: 120,
+      },
+      {
+        kind: 'jump',
+        source: 'keyboard',
+        occurredAtMs: 120,
+      },
+    ]);
+
+    controller.dispose();
+  });
+
+  test('ignores repeated or unrelated keyboard events', () => {
+    const unrelatedEvent = new KeyboardEvent('keydown', {
+      key: 'x',
+      code: 'KeyX',
+    });
+    const repeatEvent = new KeyboardEvent('keydown', {
+      key: 'a',
+      code: 'KeyA',
+      repeat: true,
+    });
+
+    expect(resolveKeyboardDebugAction(unrelatedEvent)).toBeNull();
+    expect(resolveKeyboardDebugAction(repeatEvent)).toBeNull();
+  });
+
+  test('maps left mouse drag to a left hand trajectory', () => {
+    const target = createTarget();
+    const actions: ChildMotionDebugAction[] = [];
+    const controller = createChildMotionDebugInputController({
+      target,
+      onAction: (action) => actions.push(action),
+      now: () => 240,
+    });
+
+    dispatchPointer(target, 'pointerdown', {
+      button: 0,
+      clientX: 10,
+      clientY: 20,
+      pointerId: 7,
+    });
+    dispatchPointer(target, 'pointermove', {
+      clientX: 18,
+      clientY: 24,
+      pointerId: 7,
+    });
+    dispatchPointer(target, 'pointerup', {
+      clientX: 22,
+      clientY: 28,
+      pointerId: 7,
+    });
+
+    expect(actions).toEqual([
+      {
+        kind: 'hand_trace',
+        hand: 'left',
+        phase: 'start',
+        pointerId: 7,
+        point: { x: 10, y: 20 },
+        path: [{ x: 10, y: 20 }],
+        source: 'pointer',
+        occurredAtMs: 240,
+      },
+      {
+        kind: 'hand_trace',
+        hand: 'left',
+        phase: 'move',
+        pointerId: 7,
+        point: { x: 18, y: 24 },
+        path: [
+          { x: 10, y: 20 },
+          { x: 18, y: 24 },
+        ],
+        source: 'pointer',
+        occurredAtMs: 240,
+      },
+      {
+        kind: 'hand_trace',
+        hand: 'left',
+        phase: 'end',
+        pointerId: 7,
+        point: { x: 22, y: 28 },
+        path: [
+          { x: 10, y: 20 },
+          { x: 18, y: 24 },
+          { x: 22, y: 28 },
+        ],
+        source: 'pointer',
+        occurredAtMs: 240,
+      },
+    ]);
+
+    controller.dispose();
+  });
+
+  test('maps right mouse drag to a right hand trajectory and prevents context menu', () => {
+    const target = createTarget();
+    const actions: ChildMotionDebugAction[] = [];
+    const controller = createChildMotionDebugInputController({
+      target,
+      onAction: (action) => actions.push(action),
+      now: () => 360,
+    });
+
+    const pointerDown = dispatchPointer(target, 'pointerdown', {
+      button: 2,
+      clientX: 30,
+      clientY: 40,
+      pointerId: 9,
+    });
+    dispatchPointer(target, 'pointermove', {
+      clientX: 44,
+      clientY: 48,
+      pointerId: 9,
+    });
+    dispatchPointer(target, 'pointercancel', {
+      clientX: 48,
+      clientY: 52,
+      pointerId: 9,
+    });
+    const contextMenuEvent = new MouseEvent('contextmenu', {
+      bubbles: true,
+      cancelable: true,
+      button: 2,
+    });
+    target.dispatchEvent(contextMenuEvent);
+
+    expect(pointerDown.defaultPrevented).toBe(true);
+    expect(contextMenuEvent.defaultPrevented).toBe(true);
+    expect(actions.map((action) => action.kind)).toEqual([
+      'hand_trace',
+      'hand_trace',
+      'hand_trace',
+    ]);
+    expect(actions[0]).toMatchObject({
+      hand: 'right',
+      phase: 'start',
+      point: { x: 30, y: 40 },
+    });
+    expect(actions[2]).toMatchObject({
+      hand: 'right',
+      phase: 'cancel',
+      point: { x: 48, y: 52 },
+    });
+
+    controller.dispose();
+  });
+
+  test('can be disabled or disposed without emitting debug actions', () => {
+    const target = createTarget();
+    const onAction = vi.fn();
+    const controller = createChildMotionDebugInputController({
+      target,
+      onAction,
+    });
+
+    controller.setEnabled(false);
+    expect(controller.isEnabled()).toBe(false);
+    dispatchKeyboard(target, { key: 'a', code: 'KeyA' });
+    dispatchPointer(target, 'pointerdown', {
+      button: 0,
+      clientX: 10,
+      clientY: 20,
+    });
+    expect(onAction).not.toHaveBeenCalled();
+
+    controller.setEnabled(true);
+    dispatchKeyboard(target, { key: 'd', code: 'KeyD' });
+    expect(onAction).toHaveBeenCalledTimes(1);
+
+    controller.dispose();
+    dispatchKeyboard(target, { key: ' ', code: 'Space' });
+    expect(onAction).toHaveBeenCalledTimes(1);
+  });
+});

src/services/child-motion-demo/childMotionDebugInput.ts

@@ -0,0 +1,287 @@
+export type ChildMotionDebugMoveDirection = 'left' | 'right';
+export type ChildMotionDebugHand = 'left' | 'right';
+export type ChildMotionDebugHandTracePhase = 'start' | 'move' | 'end' | 'cancel';
+
+export type ChildMotionDebugPoint = {
+  x: number;
+  y: number;
+};
+
+export type ChildMotionDebugMoveAction = {
+  kind: 'move';
+  direction: ChildMotionDebugMoveDirection;
+  source: 'keyboard';
+  occurredAtMs: number;
+};
+
+export type ChildMotionDebugJumpAction = {
+  kind: 'jump';
+  source: 'keyboard';
+  occurredAtMs: number;
+};
+
+export type ChildMotionDebugHandTraceAction = {
+  kind: 'hand_trace';
+  hand: ChildMotionDebugHand;
+  phase: ChildMotionDebugHandTracePhase;
+  pointerId: number;
+  point: ChildMotionDebugPoint;
+  path: ChildMotionDebugPoint[];
+  source: 'pointer';
+  occurredAtMs: number;
+};
+
+export type ChildMotionDebugAction =
+  | ChildMotionDebugMoveAction
+  | ChildMotionDebugJumpAction
+  | ChildMotionDebugHandTraceAction;
+
+type ChildMotionDebugActionPayload =
+  | Omit<ChildMotionDebugMoveAction, 'occurredAtMs'>
+  | Omit<ChildMotionDebugJumpAction, 'occurredAtMs'>
+  | Omit<ChildMotionDebugHandTraceAction, 'occurredAtMs'>;
+
+export type ChildMotionDebugInputTarget = Pick<
+  EventTarget,
+  'addEventListener' | 'removeEventListener'
+>;
+
+export type ChildMotionDebugInputOptions = {
+  target: ChildMotionDebugInputTarget;
+  onAction: (action: ChildMotionDebugAction) => void;
+  enabled?: boolean;
+  now?: () => number;
+  preventContextMenu?: boolean;
+};
+
+export type ChildMotionDebugInputController = {
+  dispose: () => void;
+  isEnabled: () => boolean;
+  setEnabled: (enabled: boolean) => void;
+};
+
+type ActiveHandTrace = {
+  hand: ChildMotionDebugHand;
+  path: ChildMotionDebugPoint[];
+};
+
+const DEFAULT_POINTER_ID = 1;
+
+export function createChildMotionDebugInputController(
+  options: ChildMotionDebugInputOptions,
+): ChildMotionDebugInputController {
+  const { target, onAction, now = () => Date.now() } = options;
+  const preventContextMenu = options.preventContextMenu ?? true;
+  const activeHandTraces = new Map<number, ActiveHandTrace>();
+  let enabled = options.enabled ?? true;
+
+  const emit = (action: ChildMotionDebugActionPayload) => {
+    onAction({
+      ...action,
+      occurredAtMs: now(),
+    });
+  };
+
+  const handleKeyDown = (event: Event) => {
+    if (!enabled) {
+      return;
+    }
+
+    const action = resolveKeyboardDebugAction(event);
+    if (!action) {
+      return;
+    }
+
+    event.preventDefault();
+    emit(action);
+  };
+
+  const handlePointerDown = (event: Event) => {
+    if (!enabled) {
+      return;
+    }
+
+    const hand = resolvePointerHand(event);
+    if (!hand) {
+      return;
+    }
+
+    event.preventDefault();
+    const pointerId = readPointerId(event);
+    const point = readPointerPoint(event);
+    const trace: ActiveHandTrace = {
+      hand,
+      path: [point],
+    };
+    activeHandTraces.set(pointerId, trace);
+    emit({
+      kind: 'hand_trace',
+      hand,
+      phase: 'start',
+      pointerId,
+      point,
+      path: trace.path,
+      source: 'pointer',
+    });
+  };
+
+  const handlePointerMove = (event: Event) => {
+    if (!enabled) {
+      return;
+    }
+
+    const pointerId = readPointerId(event);
+    const trace = activeHandTraces.get(pointerId);
+    if (!trace) {
+      return;
+    }
+
+    event.preventDefault();
+    const point = readPointerPoint(event);
+    trace.path = [...trace.path, point];
+    activeHandTraces.set(pointerId, trace);
+    emit({
+      kind: 'hand_trace',
+      hand: trace.hand,
+      phase: 'move',
+      pointerId,
+      point,
+      path: trace.path,
+      source: 'pointer',
+    });
+  };
+
+  const finishPointerTrace = (
+    event: Event,
+    phase: Extract<ChildMotionDebugHandTracePhase, 'end' | 'cancel'>,
+  ) => {
+    if (!enabled) {
+      return;
+    }
+
+    const pointerId = readPointerId(event);
+    const trace = activeHandTraces.get(pointerId);
+    if (!trace) {
+      return;
+    }
+
+    event.preventDefault();
+    const point = readPointerPoint(event);
+    const path = [...trace.path, point];
+    activeHandTraces.delete(pointerId);
+    emit({
+      kind: 'hand_trace',
+      hand: trace.hand,
+      phase,
+      pointerId,
+      point,
+      path,
+      source: 'pointer',
+    });
+  };
+
+  const handlePointerUp = (event: Event) => finishPointerTrace(event, 'end');
+  const handlePointerCancel = (event: Event) =>
+    finishPointerTrace(event, 'cancel');
+  const handleContextMenu = (event: Event) => {
+    if (enabled && preventContextMenu) {
+      event.preventDefault();
+    }
+  };
+
+  target.addEventListener('keydown', handleKeyDown);
+  target.addEventListener('pointerdown', handlePointerDown);
+  target.addEventListener('pointermove', handlePointerMove);
+  target.addEventListener('pointerup', handlePointerUp);
+  target.addEventListener('pointercancel', handlePointerCancel);
+  target.addEventListener('contextmenu', handleContextMenu);
+
+  return {
+    dispose: () => {
+      activeHandTraces.clear();
+      target.removeEventListener('keydown', handleKeyDown);
+      target.removeEventListener('pointerdown', handlePointerDown);
+      target.removeEventListener('pointermove', handlePointerMove);
+      target.removeEventListener('pointerup', handlePointerUp);
+      target.removeEventListener('pointercancel', handlePointerCancel);
+      target.removeEventListener('contextmenu', handleContextMenu);
+    },
+    isEnabled: () => enabled,
+    setEnabled: (nextEnabled: boolean) => {
+      enabled = nextEnabled;
+      if (!enabled) {
+        activeHandTraces.clear();
+      }
+    },
+  };
+}
+
+export function resolveKeyboardDebugAction(
+  event: Event,
+):
+  | Omit<ChildMotionDebugMoveAction, 'occurredAtMs'>
+  | Omit<ChildMotionDebugJumpAction, 'occurredAtMs'>
+  | null {
+  const keyboardEvent = event as KeyboardEvent;
+  if (keyboardEvent.repeat) {
+    return null;
+  }
+
+  const normalizedKey = keyboardEvent.key?.toLocaleLowerCase('en-US') ?? '';
+  const normalizedCode = keyboardEvent.code ?? '';
+
+  if (normalizedKey === 'a' || normalizedCode === 'KeyA') {
+    return {
+      kind: 'move',
+      direction: 'left',
+      source: 'keyboard',
+    };
+  }
+
+  if (normalizedKey === 'd' || normalizedCode === 'KeyD') {
+    return {
+      kind: 'move',
+      direction: 'right',
+      source: 'keyboard',
+    };
+  }
+
+  if (
+    keyboardEvent.key === ' ' ||
+    keyboardEvent.key === 'Spacebar' ||
+    normalizedCode === 'Space'
+  ) {
+    return {
+      kind: 'jump',
+      source: 'keyboard',
+    };
+  }
+
+  return null;
+}
+
+function resolvePointerHand(event: Event): ChildMotionDebugHand | null {
+  const button = (event as MouseEvent).button;
+  if (button === 0) {
+    return 'left';
+  }
+
+  if (button === 2) {
+    return 'right';
+  }
+
+  return null;
+}
+
+function readPointerId(event: Event) {
+  const pointerId = (event as PointerEvent).pointerId;
+  return typeof pointerId === 'number' ? pointerId : DEFAULT_POINTER_ID;
+}
+
+function readPointerPoint(event: Event): ChildMotionDebugPoint {
+  const mouseEvent = event as MouseEvent;
+  return {
+    x: mouseEvent.clientX,
+    y: mouseEvent.clientY,
+  };
+}

src/services/child-motion-demo/index.ts

@@ -0,0 +1 @@
+export * from './childMotionDebugInput';

src/services/input-devices/index.ts

@@ -0,0 +1,19 @@
+export {
+  createRuntimeDragInputController,
+  type RuntimeDragInputControllerOptions,
+  type RuntimeDragInputMove,
+  type RuntimeDragInputPress,
+  type RuntimeDragInputRelease,
+  type RuntimeDragInputSession,
+  type RuntimeInputDeviceKind,
+  type RuntimeInputPoint,
+} from './runtimeDragInputController';
+export {
+  createRuntimeInputPointFromClient,
+  createRuntimeInputPointFromNormalized,
+  readRuntimeInputElementBounds,
+  resolveRuntimeInputGridCell,
+  type RuntimeInputBounds,
+  type RuntimeInputGridCell,
+  type RuntimeInputGridSpec,
+} from './runtimeInputGeometry';

src/services/input-devices/runtimeDragInputController.test.ts

@@ -0,0 +1,161 @@
+import { describe, expect, test, vi } from 'vitest';
+
+import {
+  createRuntimeDragInputController,
+  createRuntimeInputPointFromNormalized,
+  resolveRuntimeInputGridCell,
+} from './index';
+
+describe('runtime drag input controller', () => {
+  test('pointer-like short press remains a tap', () => {
+    const onTap = vi.fn();
+    const onDrop = vi.fn();
+    const controller = createRuntimeDragInputController({
+      dragThresholdPx: 12,
+      onTap,
+      onDrop,
+    });
+
+    controller.press({
+      targetId: 'piece-1',
+      inputId: 'pointer:1',
+      deviceKind: 'pointer',
+      point: { clientX: 10, clientY: 10 },
+    });
+    controller.move({
+      inputId: 'pointer:1',
+      point: { clientX: 14, clientY: 14 },
+    });
+    controller.release({
+      inputId: 'pointer:1',
+      point: { clientX: 14, clientY: 14 },
+    });
+
+    expect(onTap).toHaveBeenCalledTimes(1);
+    expect(onTap).toHaveBeenCalledWith(
+      expect.objectContaining({ targetId: 'piece-1', dragging: false }),
+    );
+    expect(onDrop).not.toHaveBeenCalled();
+  });
+
+  test('device adapters can force continuous drag semantics', () => {
+    const onDragStart = vi.fn();
+    const onDragMove = vi.fn();
+    const onDrop = vi.fn();
+    const controller = createRuntimeDragInputController({
+      dragThresholdPx: 100,
+      onDragStart,
+      onDragMove,
+      onDrop,
+    });
+
+    controller.press({
+      targetId: 'piece-1',
+      inputId: 'mocap:hand',
+      deviceKind: 'mocap',
+      point: { clientX: 10, clientY: 10 },
+    });
+    controller.move({
+      inputId: 'mocap:hand',
+      point: { clientX: 11, clientY: 11 },
+      forceDragging: true,
+    });
+    controller.release({
+      inputId: 'mocap:hand',
+      point: { clientX: 12, clientY: 12 },
+    });
+
+    expect(onDragStart).toHaveBeenCalledTimes(1);
+    expect(onDragMove).toHaveBeenCalledTimes(1);
+    expect(onDrop).toHaveBeenCalledTimes(1);
+    expect(onDrop).toHaveBeenCalledWith(
+      expect.objectContaining({ deviceKind: 'mocap', dragging: true }),
+    );
+  });
+
+  test('device adapters can force drop on release without converting to tap', () => {
+    const onTap = vi.fn();
+    const onDrop = vi.fn();
+    const controller = createRuntimeDragInputController({
+      dragThresholdPx: 100,
+      onTap,
+      onDrop,
+    });
+
+    controller.press({
+      targetId: 'piece-1',
+      inputId: 'mocap:hand',
+      deviceKind: 'mocap',
+      point: { clientX: 10, clientY: 10 },
+    });
+    controller.release({
+      inputId: 'mocap:hand',
+      point: { clientX: 10, clientY: 10 },
+      forceDrop: true,
+    });
+
+    expect(onDrop).toHaveBeenCalledTimes(1);
+    expect(onDrop).toHaveBeenCalledWith(
+      expect.objectContaining({
+        targetId: 'piece-1',
+        forceDrop: true,
+        dragging: false,
+      }),
+    );
+    expect(onTap).not.toHaveBeenCalled();
+  });
+
+  test('input-scoped cancel keeps unrelated active sessions alive', () => {
+    const onCancel = vi.fn();
+    const onDrop = vi.fn();
+    const controller = createRuntimeDragInputController({
+      dragThresholdPx: 1,
+      onCancel,
+      onDrop,
+    });
+
+    controller.press({
+      targetId: 'piece-1',
+      inputId: 'pointer:1',
+      deviceKind: 'pointer',
+      point: { clientX: 10, clientY: 10 },
+    });
+    controller.cancel('mocap:hand');
+    controller.move({
+      inputId: 'pointer:1',
+      point: { clientX: 20, clientY: 20 },
+    });
+    controller.release({
+      inputId: 'pointer:1',
+      point: { clientX: 20, clientY: 20 },
+    });
+
+    expect(onCancel).not.toHaveBeenCalled();
+    expect(onDrop).toHaveBeenCalledTimes(1);
+    expect(onDrop).toHaveBeenCalledWith(
+      expect.objectContaining({ inputId: 'pointer:1', targetId: 'piece-1' }),
+    );
+  });
+});
+
+describe('runtime input geometry', () => {
+  test('normalised device coordinates map into client coordinates and grid cells', () => {
+    const point = createRuntimeInputPointFromNormalized(0.75, 0.25, {
+      left: 20,
+      top: 10,
+      width: 200,
+      height: 100,
+    });
+
+    expect(point).toEqual({
+      clientX: 170,
+      clientY: 35,
+      normalizedX: 0.75,
+      normalizedY: 0.25,
+    });
+    expect(resolveRuntimeInputGridCell(point, { rows: 4, cols: 4 })).toEqual({
+      row: 1,
+      col: 3,
+    });
+  });
+});

src/services/input-devices/runtimeDragInputController.ts

@@ -0,0 +1,168 @@
+export type RuntimeInputDeviceKind = 'pointer' | 'mocap' | 'keyboard' | 'unknown';
+
+export type RuntimeInputPoint = {
+  clientX: number;
+  clientY: number;
+  normalizedX?: number;
+  normalizedY?: number;
+};
+
+export type RuntimeDragInputSession<TTargetId extends string = string> = {
+  targetId: TTargetId;
+  inputId: string;
+  deviceKind: RuntimeInputDeviceKind;
+  startPoint: RuntimeInputPoint;
+  currentPoint: RuntimeInputPoint;
+  dragging: boolean;
+  forceDrop: boolean;
+};
+
+export type RuntimeDragInputPress<TTargetId extends string = string> = {
+  targetId: TTargetId;
+  inputId: string;
+  deviceKind: RuntimeInputDeviceKind;
+  point: RuntimeInputPoint;
+};
+
+export type RuntimeDragInputMove = {
+  inputId: string;
+  point: RuntimeInputPoint;
+  forceDragging?: boolean;
+};
+
+export type RuntimeDragInputRelease = {
+  inputId: string;
+  point: RuntimeInputPoint;
+  forceDrop?: boolean;
+};
+
+export type RuntimeDragInputControllerOptions<
+  TTargetId extends string = string,
+> = {
+  dragThresholdPx?: number;
+  onPress?: (session: RuntimeDragInputSession<TTargetId>) => void;
+  onDragStart?: (session: RuntimeDragInputSession<TTargetId>) => void;
+  onDragMove?: (session: RuntimeDragInputSession<TTargetId>) => void;
+  onDrop?: (session: RuntimeDragInputSession<TTargetId>) => void;
+  onTap?: (session: RuntimeDragInputSession<TTargetId>) => void;
+  onCancel?: (session: RuntimeDragInputSession<TTargetId>) => void;
+};
+
+const DEFAULT_DRAG_THRESHOLD_PX = 8;
+
+function clonePoint(point: RuntimeInputPoint): RuntimeInputPoint {
+  return { ...point };
+}
+
+function shouldStartDragging(
+  session: RuntimeDragInputSession,
+  point: RuntimeInputPoint,
+  thresholdPx: number,
+  forceDragging = false,
+) {
+  if (session.dragging || forceDragging) {
+    return true;
+  }
+
+  return (
+    Math.hypot(
+      point.clientX - session.startPoint.clientX,
+      point.clientY - session.startPoint.clientY,
+    ) >= thresholdPx
+  );
+}
+
+export function createRuntimeDragInputController<
+  TTargetId extends string = string,
+>(initialOptions: RuntimeDragInputControllerOptions<TTargetId> = {}) {
+  let options = initialOptions;
+  let session: RuntimeDragInputSession<TTargetId> | null = null;
+
+  const setOptions = (
+    nextOptions: RuntimeDragInputControllerOptions<TTargetId>,
+  ) => {
+    options = nextOptions;
+  };
+
+  const cancel = (inputId?: string) => {
+    if (inputId && session?.inputId !== inputId) {
+      return;
+    }
+
+    const activeSession = session;
+    session = null;
+    if (activeSession) {
+      options.onCancel?.(activeSession);
+    }
+  };
+
+  const press = (input: RuntimeDragInputPress<TTargetId>) => {
+    cancel();
+    session = {
+      targetId: input.targetId,
+      inputId: input.inputId,
+      deviceKind: input.deviceKind,
+      startPoint: clonePoint(input.point),
+      currentPoint: clonePoint(input.point),
+      dragging: false,
+      forceDrop: false,
+    };
+    options.onPress?.(session);
+    return session;
+  };
+
+  const move = (input: RuntimeDragInputMove) => {
+    if (!session || session.inputId !== input.inputId) {
+      return null;
+    }
+
+    const wasDragging = session.dragging;
+    session = {
+      ...session,
+      currentPoint: clonePoint(input.point),
+      dragging: shouldStartDragging(
+        session,
+        input.point,
+        options.dragThresholdPx ?? DEFAULT_DRAG_THRESHOLD_PX,
+        input.forceDragging,
+      ),
+    };
+
+    if (!wasDragging && session.dragging) {
+      options.onDragStart?.(session);
+    }
+    if (session.dragging) {
+      options.onDragMove?.(session);
+    }
+
+    return session;
+  };
+
+  const release = (input: RuntimeDragInputRelease) => {
+    if (!session || session.inputId !== input.inputId) {
+      return null;
+    }
+
+    const completedSession = {
+      ...session,
+      currentPoint: clonePoint(input.point),
+      forceDrop: input.forceDrop === true,
+    };
+    session = null;
+    if (completedSession.dragging || completedSession.forceDrop) {
+      options.onDrop?.(completedSession);
+    } else {
+      options.onTap?.(completedSession);
+    }
+    return completedSession;
+  };
+
+  return {
+    cancel,
+    getSession: () => session,
+    move,
+    press,
+    release,
+    setOptions,
+  };
+}

src/services/input-devices/runtimeInputGeometry.ts

@@ -0,0 +1,142 @@
+import type { RuntimeInputPoint } from './runtimeDragInputController';
+
+export type RuntimeInputBounds = {
+  left: number;
+  top: number;
+  width: number;
+  height: number;
+};
+
+export type RuntimeInputGridSpec = {
+  rows: number;
+  cols: number;
+};
+
+export type RuntimeInputGridCell = {
+  row: number;
+  col: number;
+};
+
+function isFiniteNumber(value: number) {
+  return Number.isFinite(value);
+}
+
+function clamp01(value: number) {
+  return Math.min(1, Math.max(0, value));
+}
+
+function hasUsableBounds(
+  bounds: RuntimeInputBounds | null | undefined,
+): bounds is RuntimeInputBounds {
+  return Boolean(
+    bounds &&
+      isFiniteNumber(bounds.left) &&
+      isFiniteNumber(bounds.top) &&
+      isFiniteNumber(bounds.width) &&
+      isFiniteNumber(bounds.height) &&
+      bounds.width > 0 &&
+      bounds.height > 0,
+  );
+}
+
+export function readRuntimeInputElementBounds(
+  element: Element | null | undefined,
+): RuntimeInputBounds | null {
+  if (!element) {
+    return null;
+  }
+
+  const rect = element.getBoundingClientRect();
+  if (!rect.width || !rect.height) {
+    return null;
+  }
+
+  return {
+    left: rect.left,
+    top: rect.top,
+    width: rect.width,
+    height: rect.height,
+  };
+}
+
+export function createRuntimeInputPointFromClient(
+  clientX: number,
+  clientY: number,
+  bounds?: RuntimeInputBounds | null,
+): RuntimeInputPoint {
+  if (!hasUsableBounds(bounds)) {
+    return { clientX, clientY };
+  }
+
+  return {
+    clientX,
+    clientY,
+    normalizedX: (clientX - bounds.left) / bounds.width,
+    normalizedY: (clientY - bounds.top) / bounds.height,
+  };
+}
+
+export function createRuntimeInputPointFromNormalized(
+  normalizedX: number,
+  normalizedY: number,
+  bounds?: RuntimeInputBounds | null,
+): RuntimeInputPoint {
+  const x = clamp01(normalizedX);
+  const y = clamp01(normalizedY);
+  if (!hasUsableBounds(bounds)) {
+    return {
+      clientX: x,
+      clientY: y,
+      normalizedX: x,
+      normalizedY: y,
+    };
+  }
+
+  return {
+    clientX: bounds.left + x * bounds.width,
+    clientY: bounds.top + y * bounds.height,
+    normalizedX: x,
+    normalizedY: y,
+  };
+}
+
+export function resolveRuntimeInputGridCell(
+  point: RuntimeInputPoint,
+  grid: RuntimeInputGridSpec,
+  bounds?: RuntimeInputBounds | null,
+): RuntimeInputGridCell | null {
+  if (grid.rows <= 0 || grid.cols <= 0) {
+    return null;
+  }
+
+  const normalizedX =
+    typeof point.normalizedX === 'number'
+      ? point.normalizedX
+      : hasUsableBounds(bounds)
+        ? (point.clientX - bounds.left) / bounds.width
+        : null;
+  const normalizedY =
+    typeof point.normalizedY === 'number'
+      ? point.normalizedY
+      : hasUsableBounds(bounds)
+        ? (point.clientY - bounds.top) / bounds.height
+        : null;
+
+  if (
+    normalizedX === null ||
+    normalizedY === null ||
+    !isFiniteNumber(normalizedX) ||
+    !isFiniteNumber(normalizedY) ||
+    normalizedX < 0 ||
+    normalizedX > 1 ||
+    normalizedY < 0 ||
+    normalizedY > 1
+  ) {
+    return null;
+  }
+
+  return {
+    row: Math.min(grid.rows - 1, Math.floor(normalizedY * grid.rows)),
+    col: Math.min(grid.cols - 1, Math.floor(normalizedX * grid.cols)),
+  };
+}

src/services/useMocapInput.test.ts

@@ -0,0 +1,81 @@
+import { describe, expect, test } from 'vitest';
+
+import { parseMocapPacket, resolveMocapPalmCenter } from './useMocapInput';
+
+describe('resolveMocapPalmCenter', () => {
+  test('优先用手腕和四个 MCP 点加权计算掌心派生点', () => {
+    const center = resolveMocapPalmCenter([
+      { name: 'wrist', x: 0.1, y: 0.2 },
+      { name: 'index_mcp', x: 0.3, y: 0.4 },
+      { name: 'middle_mcp', x: 0.5, y: 0.6 },
+      { name: 'ring_mcp', x: 0.7, y: 0.8 },
+      { name: 'pinky_mcp', x: 0.9, y: 1 },
+      { name: 'index_finger_tip', x: 1, y: 1 },
+    ]);
+
+    expect(center?.x).toBeCloseTo(0.44);
+    expect(center?.y).toBeCloseTo(0.54);
+  });
+
+  test('可用掌心点少于三个时不返回掌心坐标', () => {
+    expect(
+      resolveMocapPalmCenter([
+        { name: 'wrist', x: 0.1, y: 0.2 },
+        { name: 'index_mcp', x: 0.3, y: 0.4 },
+      ]),
+    ).toBeNull();
+  });
+});
+
+describe('parseMocapPacket', () => {
+  test('解析手部数据时优先把 primaryHand 定位到掌心而不是腕部或指尖', () => {
+    const command = parseMocapPacket({
+      hands: [
+        {
+          state: 'open_palm',
+          x: 0.01,
+          y: 0.02,
+          landmarks: [
+            { name: 'wrist', x: 0.1, y: 0.2 },
+            { name: 'index_mcp', x: 0.3, y: 0.4 },
+            { name: 'middle_mcp', x: 0.5, y: 0.6 },
+            { name: 'ring_mcp', x: 0.7, y: 0.8 },
+            { name: 'pinky_mcp', x: 0.9, y: 1 },
+          ],
+        },
+      ],
+    });
+
+    expect(command.primaryHand?.x).toBeCloseTo(0.44);
+    expect(command.primaryHand?.y).toBeCloseTo(0.54);
+    expect(command.primaryHand).toEqual(
+      expect.objectContaining({
+        state: 'open_palm',
+        source: 'palm_center',
+      }),
+    );
+  });
+
+  test('缺少足够掌心关键点时退回 wrist landmark，再退回 hand 直出坐标', () => {
+    const landmarkFallback = parseMocapPacket({
+      hands: [
+        {
+          state: 'grab',
+          x: 0.9,
+          y: 0.8,
+          landmarks: [{ name: 'wrist', x: 0.25, y: 0.75 }],
+        },
+      ],
+    });
+    expect(landmarkFallback.primaryHand).toEqual(
+      expect.objectContaining({x: 0.25, y: 0.75, source: 'landmark'}),
+    );
+
+    const directFallback = parseMocapPacket({
+      hands: [{ state: 'grab', x: 0.9, y: 0.8 }],
+    });
+    expect(directFallback.primaryHand).toEqual(
+      expect.objectContaining({x: 0.9, y: 0.8, source: 'direct'}),
+    );
+  });
+});

src/services/useMocapInput.ts

@@ -3,14 +3,29 @@ import {useEffect, useMemo, useRef, useState} from 'react';
 export type MocapConnectionStatus = 'idle' | 'connecting' | 'connected' | 'error';
 
 export type MocapHandState = 'open_palm' | 'grab' | 'unknown';
+export type MocapHandSide = 'left' | 'right' | 'unknown';
+export type MocapHandSource = 'palm_center' | 'direct' | 'landmark';
+
+export type MocapHandInput = {
+  x: number;
+  y: number;
+  state: MocapHandState;
+  side: MocapHandSide;
+  source?: MocapHandSource;
+};
+
+export type MocapBodyCenterInput = {
+  x: number;
+  y: number;
+};
 
 export type MocapInputCommand = {
   actions: string[];
-  primaryHand?: {
-    x: number;
-    y: number;
-    state: MocapHandState;
-  } | null;
+  hands?: MocapHandInput[];
+  primaryHand?: MocapHandInput | null;
+  leftHand?: MocapHandInput | null;
+  rightHand?: MocapHandInput | null;
+  bodyCenter?: MocapBodyCenterInput | null;
   parseWarnings?: string[];
 };
 
@@ -32,9 +47,19 @@ export type UseMocapInputOptions = {
   reconnectDelayMs?: number;
 };
 
+type MocapLandmarkRecord = Record<string, unknown>;
+
 const DEFAULT_MOCAP_SERVICE_URL = 'http://127.0.0.1:8876';
 const DEFAULT_RECONNECT_DELAY_MS = 1200;
 const MAX_RAW_PACKET_PREVIEW_LENGTH = 360;
+const PALM_CENTER_WEIGHTS = [
+  ['wrist', 0.25],
+  ['index_mcp', 0.2],
+  ['middle_mcp', 0.25],
+  ['ring_mcp', 0.2],
+  ['pinky_mcp', 0.1],
+] as const;
+const MIN_PALM_CENTER_POINT_COUNT = 3;
 
 function buildRawPacketPreview(rawData: unknown): string {
   const rawText = typeof rawData === 'string' ? rawData : JSON.stringify(rawData);
@@ -57,89 +82,301 @@ function normalizeCoordinate(value: unknown) {
   return Math.min(1, Math.max(0, numericValue));
 }
 
-function resolvePrimaryHand(hands: unknown) {
-  if (!Array.isArray(hands)) {
+function resolveNormalizedPoint(value: unknown) {
+  if (Array.isArray(value)) {
+    const x = normalizeCoordinate(value[0]);
+    const y = normalizeCoordinate(value[1]);
+    if (x === null || y === null) {
+      return null;
+    }
+    return {x, y};
+  }
+
+  if (!value || typeof value !== 'object') {
     return null;
   }
 
-  for (const hand of hands) {
-    if (!hand || typeof hand !== 'object') {
-      continue;
-    }
-
-    const handRecord = hand as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
-    const state = normaliseHandState(handRecord.state);
-    const directX = normalizeCoordinate(handRecord.x);
-    const directY = normalizeCoordinate(handRecord.y);
-    if (directX !== null && directY !== null) {
-      return {x: directX, y: directY, state};
-    }
-    if (!Array.isArray(handRecord.landmarks)) {
-      continue;
-    }
-
-    const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
-    const landmark =
-      landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
-    const x = normalizeCoordinate(landmark?.x);
-    const y = normalizeCoordinate(landmark?.y);
-    if (x === null || y === null) {
-      continue;
-    }
-
-    return {x, y, state};
+  const pointRecord = value as {x?: unknown; y?: unknown};
+  const x = normalizeCoordinate(pointRecord.x);
+  const y = normalizeCoordinate(pointRecord.y);
+  if (x === null || y === null) {
+    return null;
   }
 
-  return null;
+  return {x, y};
 }
 
-function resolveHandLike(record: unknown) {
-  if (!record || typeof record !== 'object') {
-    return null;
-  }
-
-  const handRecord = record as {state?: unknown; landmarks?: unknown; x?: unknown; y?: unknown};
-  const state = normaliseHandState(handRecord.state);
-  const directX = normalizeCoordinate(handRecord.x);
-  const directY = normalizeCoordinate(handRecord.y);
-  if (directX !== null && directY !== null) {
-    return {x: directX, y: directY, state};
-  }
-  if (!Array.isArray(handRecord.landmarks)) {
-    return null;
-  }
-
-  const landmarks = handRecord.landmarks as Array<Record<string, unknown>>;
-  const landmark =
-    landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
+function resolveLandmarkCoordinate(landmark: MocapLandmarkRecord | undefined) {
   const x = normalizeCoordinate(landmark?.x);
   const y = normalizeCoordinate(landmark?.y);
   if (x === null || y === null) {
     return null;
   }
-
-  return {x, y, state};
+  return {x, y};
 }
 
-function normaliseHandState(state: unknown): MocapHandState {
-  if (state === 'grab' || state === 'open_palm') {
-    return state;
+export function resolveMocapPalmCenter(
+  landmarks: Array<MocapLandmarkRecord>,
+): {x: number; y: number} | null {
+  const landmarksByName = new Map(
+    landmarks
+      .filter((landmark) => typeof landmark?.name === 'string')
+      .map((landmark) => [String(landmark.name), landmark]),
+  );
+  const weightedPoints = PALM_CENTER_WEIGHTS.map(([name, weight]) => {
+    const point = resolveLandmarkCoordinate(landmarksByName.get(name));
+    return point ? {...point, weight: Number(weight)} : null;
+  }).filter((point): point is {x: number; y: number; weight: number} =>
+    Boolean(point),
+  );
+
+  if (weightedPoints.length < MIN_PALM_CENTER_POINT_COUNT) {
+    return null;
   }
+
+  const weightTotal = weightedPoints.reduce((sum, point) => sum + point.weight, 0);
+  if (weightTotal <= 0) {
+    return null;
+  }
+
+  return {
+    x: weightedPoints.reduce((sum, point) => sum + point.x * point.weight, 0) / weightTotal,
+    y: weightedPoints.reduce((sum, point) => sum + point.y * point.weight, 0) / weightTotal,
+  };
+}
+
+function normaliseHandSide(side: unknown): MocapHandSide {
+  if (typeof side !== 'string') {
+    return 'unknown';
+  }
+
+  const normalized = side.trim().toLocaleLowerCase('en-US');
+  if (
+    normalized === 'left' ||
+    normalized === 'l' ||
+    normalized === 'left hand' ||
+    normalized === 'left_hand' ||
+    normalized === 'left-hand' ||
+    normalized === '左' ||
+    normalized === '左手'
+  ) {
+    return 'left';
+  }
+
+  if (
+    normalized === 'right' ||
+    normalized === 'r' ||
+    normalized === 'right hand' ||
+    normalized === 'right_hand' ||
+    normalized === 'right-hand' ||
+    normalized === '右' ||
+    normalized === '右手'
+  ) {
+    return 'right';
+  }
+
   return 'unknown';
 }
 
-function parseMocapPacket(packet: unknown): MocapInputCommand {
+function normalizeMocapAction(value: unknown) {
+  if (typeof value !== 'string') {
+    return null;
+  }
+
+  const normalized = value
+    .trim()
+    .toLocaleLowerCase('en-US')
+    .replace(/\s+/gu, '_')
+    .replace(/-/gu, '_');
+
+  return normalized || null;
+}
+
+function addMocapActions(actions: Set<string>, value: unknown) {
+  if (Array.isArray(value)) {
+    value.forEach((item) => addMocapActions(actions, item));
+    return;
+  }
+
+  if (value && typeof value === 'object') {
+    const actionRecord = value as Record<string, unknown>;
+    addMocapActions(actions, actionRecord.action);
+    addMocapActions(actions, actionRecord.actions);
+    addMocapActions(actions, actionRecord.gesture);
+    addMocapActions(actions, actionRecord.gestures);
+    addMocapActions(actions, actionRecord.event);
+    addMocapActions(actions, actionRecord.name);
+    addMocapActions(actions, actionRecord.type);
+    return;
+  }
+
+  const normalized = normalizeMocapAction(value);
+  if (normalized) {
+    actions.add(normalized);
+  }
+}
+
+function normaliseHandState(state: unknown): MocapHandState {
+  if (typeof state !== 'string') {
+    return 'unknown';
+  }
+
+  const normalized = state
+    .trim()
+    .toLocaleLowerCase('en-US')
+    .replace(/\s+/gu, '_')
+    .replace(/-/gu, '_');
+
+  if (
+    normalized === 'grab' ||
+    normalized === 'grabbing' ||
+    normalized === 'close' ||
+    normalized === 'fist' ||
+    normalized === 'closed_fist' ||
+    normalized === 'closed'
+  ) {
+    return 'grab';
+  }
+
+  if (
+    normalized === 'open_palm' ||
+    normalized === 'open_palm_up' ||
+    normalized === 'open' ||
+    normalized === 'palm' ||
+    normalized === 'hand_open'
+  ) {
+    return 'open_palm';
+  }
+
+  return 'unknown';
+}
+
+function resolveMocapHand(record: unknown, fallbackSide: MocapHandSide) {
+  if (!record || typeof record !== 'object') {
+    return null;
+  }
+
+  const handRecord = record as {
+    state?: unknown;
+    landmarks?: unknown;
+    x?: unknown;
+    y?: unknown;
+    side?: unknown;
+    handedness?: unknown;
+    label?: unknown;
+    hand?: unknown;
+  };
+  const state = normaliseHandState(handRecord.state);
+  const detectedSide = normaliseHandSide(
+    handRecord.side ??
+      handRecord.handedness ??
+      handRecord.label ??
+      handRecord.hand,
+  );
+  const side = detectedSide === 'unknown' ? fallbackSide : detectedSide;
+
+  if (Array.isArray(handRecord.landmarks)) {
+    const landmarks = handRecord.landmarks as Array<MocapLandmarkRecord>;
+    const palmCenter = resolveMocapPalmCenter(landmarks);
+    if (palmCenter) {
+      return {...palmCenter, state, side, source: 'palm_center' as const};
+    }
+
+    const landmark =
+      landmarks.find((item) => item?.name === 'wrist') ?? landmarks[0];
+    const fallbackPoint = resolveLandmarkCoordinate(landmark);
+    if (fallbackPoint) {
+      return {...fallbackPoint, state, side, source: 'landmark' as const};
+    }
+  }
+
+  const directX = normalizeCoordinate(handRecord.x);
+  const directY = normalizeCoordinate(handRecord.y);
+  if (directX !== null && directY !== null) {
+    return {x: directX, y: directY, state, side, source: 'direct' as const};
+  }
+
+  return null;
+}
+
+function resolveHands(packetRecord: Record<string, unknown>) {
+  const resolvedHands: MocapHandInput[] = [];
+  const packetHands = packetRecord.hands;
+  if (!Array.isArray(packetHands)) {
+    const leftHand = resolveMocapHand(
+      packetRecord.leftHand ?? packetRecord.left_hand,
+      'left',
+    );
+    const rightHand = resolveMocapHand(
+      packetRecord.rightHand ?? packetRecord.right_hand,
+      'right',
+    );
+    if (leftHand) {
+      resolvedHands.push(leftHand);
+    }
+    if (rightHand) {
+      resolvedHands.push(rightHand);
+    }
+    return resolvedHands;
+  }
+
+  for (const hand of packetHands) {
+    const resolvedHand = resolveMocapHand(hand, 'unknown');
+    if (resolvedHand) {
+      resolvedHands.push(resolvedHand);
+    }
+  }
+
+  return resolvedHands;
+}
+
+function resolveBodyCenter(packetRecord: Record<string, unknown>) {
+  const generalRecord =
+    packetRecord.general && typeof packetRecord.general === 'object'
+      ? (packetRecord.general as Record<string, unknown>)
+      : null;
+  const bodyCandidates = [generalRecord?.body, packetRecord.body];
+
+  for (const bodyCandidate of bodyCandidates) {
+    if (!bodyCandidate || typeof bodyCandidate !== 'object') {
+      continue;
+    }
+
+    const bodyRecord = bodyCandidate as Record<string, unknown>;
+    const center = resolveNormalizedPoint(
+      bodyRecord.center_norm ?? bodyRecord.centerNorm ?? bodyRecord.center,
+    );
+    if (center) {
+      return center;
+    }
+  }
+
+  return null;
+}
+
+export function parseMocapPacket(packet: unknown): MocapInputCommand {
   if (!packet || typeof packet !== 'object') {
     return {actions: [], parseWarnings: ['packet 不是对象']};
   }
 
-  const packetRecord = packet as {hands?: unknown};
-  const primaryHand = resolvePrimaryHand(packetRecord.hands);
+  const packetRecord = packet as Record<string, unknown>;
+  const hands = resolveHands(packetRecord);
+  const primaryHand = hands[0] ?? null;
+  const leftHand = hands.find((hand) => hand.side === 'left') ?? null;
+  const rightHand = hands.find((hand) => hand.side === 'right') ?? null;
+  const bodyCenter = resolveBodyCenter(packetRecord);
   const actions = new Set<string>();
   const parseWarnings: string[] = [];
-  if (!Array.isArray(packetRecord.hands)) {
+  addMocapActions(actions, packetRecord.actions);
+  addMocapActions(actions, packetRecord.action);
+  addMocapActions(actions, packetRecord.gesture);
+  addMocapActions(actions, packetRecord.gestures);
+  addMocapActions(actions, packetRecord.event);
+  addMocapActions(actions, packetRecord.name);
+  addMocapActions(actions, packetRecord.type);
+  if (!Array.isArray(packetRecord.hands) && hands.length === 0 && !bodyCenter) {
     parseWarnings.push('缺少 hands 数组');
-  } else if (!primaryHand) {
+  } else if (!primaryHand && !bodyCenter) {
     parseWarnings.push('hands 中没有可用坐标');
   }
   if (primaryHand?.state === 'grab') {
@@ -148,13 +385,22 @@ function parseMocapPacket(packet: unknown): MocapInputCommand {
   if (primaryHand?.state === 'open_palm') {
     actions.add('open_palm');
   }
+  for (const hand of hands) {
+    if (hand.state !== 'unknown') {
+      actions.add(hand.state);
+    }
+  }
   if (primaryHand && primaryHand.state === 'unknown') {
     parseWarnings.push('手势 state 未识别');
   }
 
   return {
     actions: Array.from(actions),
+    hands,
     primaryHand,
+    leftHand,
+    rightHand,
+    bodyCenter,
     parseWarnings,
   };
 }