import { ArrowLeft, Brush, Check, Eraser, ImagePlus, RotateCcw, Save, Sparkles, } from 'lucide-react'; import { type CSSProperties, type PointerEvent as ReactPointerEvent, useCallback, useEffect, useMemo, useRef, useState, } from 'react'; import type { BabyLoveDrawingPoint, BabyLoveDrawingRecord, BabyLoveDrawingStroke, BabyLoveDrawingTool, } from '../../../packages/shared/src/contracts/edutainmentBabyDrawing'; import { BABY_LOVE_DRAWING_RAINBOW_COLORS } from '../../../packages/shared/src/contracts/edutainmentBabyDrawing'; import { createBabyLoveDrawingMagicImage, saveBabyLoveDrawing, } from '../../services/edutainment-baby-drawing'; import type { MocapHandInput } from '../../services/useMocapInput'; import { useMocapInput } from '../../services/useMocapInput'; import { PlatformRuntimeStatusToast } from '../common/PlatformRuntimeStatusToast'; import { appendPointToStroke, BABY_LOVE_DRAWING_BRUSH_SIZE, BABY_LOVE_DRAWING_DEFAULT_COLOR, BABY_LOVE_DRAWING_ERASER_SIZE, type BabyLoveDrawingBounds, type BabyLoveDrawingHandPoint, type BabyLoveDrawingHoverTarget, type BabyLoveDrawingPhase, createBabyDrawingStroke, hasHoverCompleted, isPointInsideBounds, resolveHoverProgress, toCanvasPoint, } from './babyLoveDrawingModel'; type BabyLoveDrawingRuntimeShellProps = { onBack?: () => void; }; type ActionButtonId = 'finish' | 'magic' | 'save' | 'restart' | 'back'; type RectMap = { canvas: BabyLoveDrawingBounds | null; colors: Record; tools: Record; buttons: Record; }; type ActiveStrokeState = { stroke: BabyLoveDrawingStroke; lastPoint: BabyLoveDrawingPoint; }; const BABY_LOVE_DRAWING_HAND_LOSS_GRACE_MS = 320; const BABY_LOVE_DRAWING_HAND_SMOOTHING = 0.38; const BABY_LOVE_DRAWING_RIGHT_HAND_MAX_FRAME_JUMP = 0.28; const EMPTY_RECT_MAP: RectMap = { canvas: null, colors: {}, tools: { brush: null, eraser: null, }, buttons: { finish: null, magic: null, save: null, restart: null, back: null, }, }; function pointFromPointer( event: ReactPointerEvent, element: HTMLElement, ): BabyLoveDrawingHandPoint { const rect = element.getBoundingClientRect(); const width = rect.width || 1; const height = rect.height || 1; return { x: Math.max(0, Math.min(1, (event.clientX - rect.left) / width)), y: Math.max(0, Math.min(1, (event.clientY - rect.top) / height)), state: event.buttons ? 'grab' : 'open_palm', }; } function handToPoint(hand: MocapHandInput | null | undefined) { if (!hand) { return null; } return { x: hand.x, y: hand.y, state: hand.state, } satisfies BabyLoveDrawingHandPoint; } function commandToPlayerLeftHand(command: { rightHand?: MocapHandInput | null; }) { // 本地 mocap handedness 当前按摄像头视角输出:画面右侧手对应用户身体左手。 return handToPoint(command.rightHand); } function commandToPlayerRightHand(command: { leftHand?: MocapHandInput | null; }) { // 本地 mocap handedness 当前按摄像头视角输出:画面左侧手对应用户身体右手。 return handToPoint(command.leftHand); } function smoothHandPoint( previous: BabyLoveDrawingHandPoint | null, next: BabyLoveDrawingHandPoint, ): BabyLoveDrawingHandPoint { if (!previous) { return next; } return { x: previous.x + (next.x - previous.x) * BABY_LOVE_DRAWING_HAND_SMOOTHING, y: previous.y + (next.y - previous.y) * BABY_LOVE_DRAWING_HAND_SMOOTHING, state: next.state, }; } function getHandPointDistance( left: BabyLoveDrawingHandPoint, right: BabyLoveDrawingHandPoint, ) { return Math.hypot(left.x - right.x, left.y - right.y); } function canAcceptRightHandPoint( previous: BabyLoveDrawingHandPoint | null, next: BabyLoveDrawingHandPoint | null, ) { if (!next || !previous) { return Boolean(next); } return ( getHandPointDistance(previous, next) <= BABY_LOVE_DRAWING_RIGHT_HAND_MAX_FRAME_JUMP ); } function sameHoverTarget( left: BabyLoveDrawingHoverTarget, right: BabyLoveDrawingHoverTarget, ) { if (!left || !right) { return left === right; } return left.kind === right.kind && left.id === right.id; } function findTargetInBounds( point: BabyLoveDrawingHandPoint | null, bounds: Record, ): T | null { if (!point) { return null; } for (const [id, rect] of Object.entries(bounds) as Array< [T, BabyLoveDrawingBounds | null] >) { if (rect && isPointInsideBounds(point, rect)) { return id; } } return null; } function drawStrokeSegment( context: CanvasRenderingContext2D, stroke: BabyLoveDrawingStroke, from: BabyLoveDrawingPoint, to: BabyLoveDrawingPoint, width: number, height: number, ) { context.save(); context.lineCap = 'round'; context.lineJoin = 'round'; context.lineWidth = stroke.tool === 'brush' ? BABY_LOVE_DRAWING_BRUSH_SIZE : BABY_LOVE_DRAWING_ERASER_SIZE; if (stroke.tool === 'eraser') { context.globalCompositeOperation = 'destination-out'; context.strokeStyle = 'rgba(0,0,0,1)'; } else { context.globalCompositeOperation = 'source-over'; context.strokeStyle = stroke.color; } context.beginPath(); context.moveTo(from.x * width, from.y * height); context.lineTo(to.x * width, to.y * height); context.stroke(); context.restore(); } export function BabyLoveDrawingRuntimeShell({ onBack, }: BabyLoveDrawingRuntimeShellProps) { const shellRef = useRef(null); const canvasRef = useRef(null); const rectMapRef = useRef(EMPTY_RECT_MAP); const activeStrokeRef = useRef(null); const hoverTargetRef = useRef(null); const hoverStartedAtRef = useRef(null); const hoverCompletedKeyRef = useRef(null); const previousToolGrabRef = useRef(null); const visibleLeftHandRef = useRef(null); const visibleRightHandRef = useRef(null); const leftHandSeenAtRef = useRef(null); const rightHandSeenAtRef = useRef(null); const [phase, setPhase] = useState('drawing'); const [selectedColor, setSelectedColor] = useState( BABY_LOVE_DRAWING_DEFAULT_COLOR, ); const [selectedTool, setSelectedTool] = useState('brush'); const [strokes, setStrokes] = useState([]); const [rightHandPoint, setRightHandPoint] = useState(null); const [leftHandPoint, setLeftHandPoint] = useState(null); const [hoverTarget, setHoverTarget] = useState(null); const [hoverProgress, setHoverProgress] = useState(0); const [originalImageSrc, setOriginalImageSrc] = useState(null); const [magicImageSrc, setMagicImageSrc] = useState(null); const [savedRecord, setSavedRecord] = useState( null, ); const [error, setError] = useState(null); const { latestCommand } = useMocapInput({ enabled: true }); const canUseMagic = phase === 'finished' || phase === 'magicReady'; const canSave = phase === 'finished' || phase === 'magicReady'; const actionButtons = useMemo( () => [ { id: 'finish' as const, label: '完成', icon: Check, visible: phase === 'drawing', }, { id: 'magic' as const, label: phase === 'magicPending' ? '魔法中' : '使用绘画魔法', icon: Sparkles, visible: phase === 'finished' || phase === 'magicReady' || phase === 'magicPending', }, { id: 'save' as const, label: '保存', icon: Save, visible: canSave, }, { id: 'restart' as const, label: '再画一张', icon: RotateCcw, visible: phase === 'saved', }, { id: 'back' as const, label: '返回', icon: ArrowLeft, visible: phase === 'saved', }, ], [canSave, phase], ); const updateRectMap = useCallback(() => { const shell = shellRef.current; if (!shell) { return; } const shellRect = shell.getBoundingClientRect(); const toUnitBounds = (element: Element | null): BabyLoveDrawingBounds | null => { if (!(element instanceof HTMLElement)) { return null; } const rect = element.getBoundingClientRect(); return { left: (rect.left - shellRect.left) / shellRect.width, top: (rect.top - shellRect.top) / shellRect.height, width: rect.width / shellRect.width, height: rect.height / shellRect.height, }; }; const colors: Record = {}; BABY_LOVE_DRAWING_RAINBOW_COLORS.forEach((color) => { const rect = toUnitBounds( shell.querySelector(`[data-baby-drawing-color="${color.id}"]`), ); if (rect) { colors[color.id] = rect; } }); rectMapRef.current = { canvas: toUnitBounds(shell.querySelector('[data-baby-drawing-canvas]')), colors, tools: { brush: toUnitBounds(shell.querySelector('[data-baby-drawing-tool="brush"]')), eraser: toUnitBounds(shell.querySelector('[data-baby-drawing-tool="eraser"]')), }, buttons: { finish: toUnitBounds(shell.querySelector('[data-baby-drawing-button="finish"]')), magic: toUnitBounds(shell.querySelector('[data-baby-drawing-button="magic"]')), save: toUnitBounds(shell.querySelector('[data-baby-drawing-button="save"]')), restart: toUnitBounds(shell.querySelector('[data-baby-drawing-button="restart"]')), back: toUnitBounds(shell.querySelector('[data-baby-drawing-button="back"]')), }, }; }, []); useEffect(() => { updateRectMap(); window.addEventListener('resize', updateRectMap); return () => window.removeEventListener('resize', updateRectMap); }, [updateRectMap]); useEffect(() => { updateRectMap(); }, [actionButtons, phase, updateRectMap]); useEffect(() => { const canvas = canvasRef.current; if (!canvas) { return; } const resizeCanvas = () => { const rect = canvas.getBoundingClientRect(); const dpr = Math.min(window.devicePixelRatio || 1, 2); const nextWidth = Math.max(1, Math.floor(rect.width * dpr)); const nextHeight = Math.max(1, Math.floor(rect.height * dpr)); if (canvas.width === nextWidth && canvas.height === nextHeight) { return; } const previousImage = canvas.toDataURL('image/png'); canvas.width = nextWidth; canvas.height = nextHeight; const context = canvas.getContext('2d'); if (!context) { return; } context.fillStyle = '#fffdf4'; context.fillRect(0, 0, canvas.width, canvas.height); if (previousImage) { const image = new Image(); image.onload = () => { context.drawImage(image, 0, 0, canvas.width, canvas.height); }; image.src = previousImage; } }; resizeCanvas(); window.addEventListener('resize', resizeCanvas); return () => window.removeEventListener('resize', resizeCanvas); }, []); const clearCanvas = useCallback(() => { const canvas = canvasRef.current; const context = canvas?.getContext('2d'); if (!canvas || !context) { return; } context.globalCompositeOperation = 'source-over'; context.fillStyle = '#fffdf4'; context.fillRect(0, 0, canvas.width, canvas.height); }, []); useEffect(() => { clearCanvas(); }, [clearCanvas]); const captureOriginalImage = useCallback(() => { const canvas = canvasRef.current; if (!canvas) { return null; } return canvas.toDataURL('image/png'); }, []); const finishDrawing = useCallback(() => { const imageSrc = captureOriginalImage(); if (!imageSrc) { return; } activeStrokeRef.current = null; setOriginalImageSrc(imageSrc); setPhase('finished'); setError(null); }, [captureOriginalImage]); const restartDrawing = useCallback(() => { activeStrokeRef.current = null; hoverTargetRef.current = null; hoverStartedAtRef.current = null; hoverCompletedKeyRef.current = null; setPhase('drawing'); setSelectedColor(BABY_LOVE_DRAWING_DEFAULT_COLOR); setSelectedTool('brush'); setStrokes([]); setOriginalImageSrc(null); setMagicImageSrc(null); setSavedRecord(null); setError(null); setHoverTarget(null); setHoverProgress(0); clearCanvas(); }, [clearCanvas]); const saveCurrentDrawing = useCallback(() => { const imageSrc = originalImageSrc ?? captureOriginalImage(); if (!imageSrc) { return; } const response = saveBabyLoveDrawing({ originalImageSrc: imageSrc, magicImageSrc, strokeTrace: strokes, }); setOriginalImageSrc(imageSrc); setSavedRecord(response.record); setPhase('saved'); setError(null); }, [captureOriginalImage, magicImageSrc, originalImageSrc, strokes]); const generateMagicImage = useCallback(async () => { const imageSrc = originalImageSrc ?? captureOriginalImage(); if (!imageSrc || phase === 'magicPending') { return; } setOriginalImageSrc(imageSrc); setPhase('magicPending'); setError(null); try { const response = await createBabyLoveDrawingMagicImage({ originalImageSrc: imageSrc, strokeTrace: strokes, }); setMagicImageSrc(response.magicImageSrc); setPhase('magicReady'); } catch (magicError) { setError( magicError instanceof Error ? magicError.message : '生成宝贝爱画魔法图片失败。', ); setPhase('finished'); } }, [captureOriginalImage, originalImageSrc, phase, strokes]); const triggerButton = useCallback( (buttonId: string) => { if (buttonId === 'finish' && phase === 'drawing') { finishDrawing(); return; } if (buttonId === 'magic' && canUseMagic) { void generateMagicImage(); return; } if (buttonId === 'save' && canSave) { saveCurrentDrawing(); return; } if (buttonId === 'restart' && phase === 'saved') { restartDrawing(); return; } if (buttonId === 'back' && phase === 'saved') { onBack?.(); } }, [ canSave, canUseMagic, finishDrawing, generateMagicImage, onBack, phase, restartDrawing, saveCurrentDrawing, ], ); const applyHoverTarget = useCallback( (nextTarget: BabyLoveDrawingHoverTarget) => { const currentTarget = hoverTargetRef.current; const now = Date.now(); if (!sameHoverTarget(currentTarget, nextTarget)) { hoverTargetRef.current = nextTarget; hoverStartedAtRef.current = nextTarget ? now : null; hoverCompletedKeyRef.current = null; setHoverTarget(nextTarget); setHoverProgress(0); return; } const startedAt = hoverStartedAtRef.current; const progress = resolveHoverProgress(nextTarget, startedAt, now); setHoverProgress(progress); if (!hasHoverCompleted(nextTarget, startedAt, now) || !nextTarget) { return; } const completeKey = `${nextTarget.kind}:${nextTarget.id}`; if (hoverCompletedKeyRef.current === completeKey) { return; } hoverCompletedKeyRef.current = completeKey; if (nextTarget.kind === 'color') { const color = BABY_LOVE_DRAWING_RAINBOW_COLORS.find( (item) => item.id === nextTarget.id, ); if (color) { setSelectedColor(color.value); setSelectedTool('brush'); } return; } triggerButton(nextTarget.id); }, [triggerButton], ); const updateToolFromRightHand = useCallback((point: BabyLoveDrawingHandPoint | null) => { if (!point || point.state !== 'grab') { previousToolGrabRef.current = null; return; } const tool = findTargetInBounds(point, rectMapRef.current.tools); if (!tool) { previousToolGrabRef.current = null; return; } if (previousToolGrabRef.current === tool) { return; } previousToolGrabRef.current = tool; setSelectedTool(tool); }, []); const drawWithRightHand = useCallback( (point: BabyLoveDrawingHandPoint | null) => { const canvasBounds = rectMapRef.current.canvas; const canvas = canvasRef.current; const context = canvas?.getContext('2d'); if ( phase !== 'drawing' || !point || point.state !== 'grab' || !canvasBounds || !canvas || !context || !isPointInsideBounds(point, canvasBounds) ) { activeStrokeRef.current = null; return; } const nextPoint = toCanvasPoint(point, canvasBounds); const activeStroke = activeStrokeRef.current; if (!activeStroke) { const stroke = createBabyDrawingStroke( selectedTool, selectedColor, nextPoint, ); activeStrokeRef.current = { stroke, lastPoint: nextPoint, }; setStrokes((current) => [...current, stroke]); return; } const nextStroke = appendPointToStroke(activeStroke.stroke, nextPoint); drawStrokeSegment( context, nextStroke, activeStroke.lastPoint, nextPoint, canvas.width, canvas.height, ); activeStrokeRef.current = { stroke: nextStroke, lastPoint: nextPoint, }; setStrokes((current) => current.map((stroke) => stroke.strokeId === nextStroke.strokeId ? nextStroke : stroke, ), ); }, [phase, selectedColor, selectedTool], ); const updateInteraction = useCallback( ( nextLeftHand: BabyLoveDrawingHandPoint | null, nextRightHand: BabyLoveDrawingHandPoint | null, ) => { const now = Date.now(); const previousLeftHand = visibleLeftHandRef.current; const previousRightHand = visibleRightHandRef.current; const acceptedRightHand = canAcceptRightHandPoint( previousRightHand, nextRightHand, ) ? nextRightHand : null; const visibleLeftHand = nextLeftHand ? smoothHandPoint(previousLeftHand, nextLeftHand) : previousLeftHand && leftHandSeenAtRef.current !== null && now - leftHandSeenAtRef.current <= BABY_LOVE_DRAWING_HAND_LOSS_GRACE_MS ? previousLeftHand : null; const visibleRightHand = acceptedRightHand ? smoothHandPoint(previousRightHand, acceptedRightHand) : previousRightHand && rightHandSeenAtRef.current !== null && now - rightHandSeenAtRef.current <= BABY_LOVE_DRAWING_HAND_LOSS_GRACE_MS ? previousRightHand : null; const activeRightHand = acceptedRightHand ? visibleRightHand : null; if (nextLeftHand) { leftHandSeenAtRef.current = now; } if (acceptedRightHand) { rightHandSeenAtRef.current = now; } visibleLeftHandRef.current = visibleLeftHand; visibleRightHandRef.current = visibleRightHand; setLeftHandPoint(visibleLeftHand); setRightHandPoint(visibleRightHand); updateToolFromRightHand(activeRightHand); drawWithRightHand(activeRightHand); const colorId = findTargetInBounds( visibleLeftHand, rectMapRef.current.colors, ); const buttonId = findTargetInBounds(visibleLeftHand, rectMapRef.current.buttons) ?? findTargetInBounds(visibleRightHand, rectMapRef.current.buttons); const nextHoverTarget: BabyLoveDrawingHoverTarget = colorId ? { kind: 'color', id: colorId } : buttonId ? { kind: 'button', id: buttonId } : null; applyHoverTarget(nextHoverTarget); }, [applyHoverTarget, drawWithRightHand, updateToolFromRightHand], ); useEffect(() => { if (!latestCommand) { return; } updateInteraction( commandToPlayerLeftHand(latestCommand), commandToPlayerRightHand(latestCommand), ); }, [latestCommand, updateInteraction]); const handlePointerDown = (event: ReactPointerEvent) => { const point = pointFromPointer(event, event.currentTarget); if (event.button === 2) { updateInteraction(leftHandPoint, { ...point, state: 'grab' as const }); return; } updateInteraction({ ...point, state: 'open_palm' as const }, null); }; const handlePointerMove = (event: ReactPointerEvent) => { const point = pointFromPointer(event, event.currentTarget); const nextState: BabyLoveDrawingHandPoint['state'] = event.buttons ? 'grab' : 'open_palm'; const nextPoint: BabyLoveDrawingHandPoint = { ...point, state: nextState, }; if (event.buttons === 2) { updateInteraction(leftHandPoint, nextPoint); return; } updateInteraction(nextPoint, null); }; const handlePointerUp = (event: ReactPointerEvent) => { const point = pointFromPointer(event, event.currentTarget); if (event.button === 2) { updateInteraction(leftHandPoint, { ...point, state: 'open_palm' as const }); return; } updateInteraction({ ...point, state: 'open_palm' as const }, null); }; return (
event.preventDefault()} >
{BABY_LOVE_DRAWING_RAINBOW_COLORS.map((color) => (
{magicImageSrc && phase !== 'drawing' ? ( 绘画魔法结果 ) : null} {phase === 'magicPending' ? (
绘画魔法
) : null}
{actionButtons .filter((button) => button.visible) .map((button) => { const Icon = button.icon; const isHovering = hoverTarget?.kind === 'button' && hoverTarget.id === button.id; return ( ); })}
{error ? ( {error} ) : null} {savedRecord ? ( 已保存 ) : null} {leftHandPoint ? ( ) : null}
{selectedTool === 'brush' ? ( ) : ( )}
); } export default BabyLoveDrawingRuntimeShell;