GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add edutainment drawing and visual package flows

Browse Source
This commit is contained in:
五香丸子
2026-05-14 14:17:10 +08:00
parent 10e8beea80
commit e444266e1e
109 changed files with 8788 additions and 996 deletions
Download Patch File Download Diff File Expand all files Collapse all files

391
src/components/edutainment-runtime/BabyObjectMatchRuntimeShell.tsx
View File

@@ -6,6 +6,7 @@ import {
SkipForward,
} from 'lucide-react';
import {
type CSSProperties,
type PointerEvent as ReactPointerEvent,
useCallback,
useEffect,
@@ -16,6 +17,8 @@ import {
import type {
BabyObjectMatchDraft,
BabyObjectMatchItemAsset,
BabyObjectMatchVisualAsset,
BabyObjectMatchVisualAssetKind,
} from '../../../packages/shared/src/contracts/edutainmentBabyObject';
import type {
MocapHandInput,
@@ -26,7 +29,10 @@ import { useMocapInput } from '../../services/useMocapInput';
import { ResolvedAssetImage } from '../ResolvedAssetImage';
const BABY_OBJECT_MATCH_SUCCESS_TARGET = 20;
const BABY_OBJECT_MATCH_FEEDBACK_DURATION_MS = 760;
const BABY_OBJECT_MATCH_GIFT_APPEAR_DURATION_MS = 620;
const BABY_OBJECT_MATCH_GIFT_OPEN_DURATION_MS = 640;
const BABY_OBJECT_MATCH_ITEM_APPEAR_DURATION_MS = 620;
const BABY_OBJECT_MATCH_FEEDBACK_DURATION_MS = 1180;
const BABY_OBJECT_MATCH_MIN_HORIZONTAL_MOVE_DISTANCE = 0.05;
const BABY_OBJECT_MATCH_HAND_PATH_LIMIT = 16;
@@ -41,7 +47,14 @@ type BabyObjectMatchRuntimeShellProps = {
};
type BasketSide = 'left' | 'right';
type RuntimePhase = 'waiting' | 'active' | 'correct' | 'wrong' | 'complete';
type RuntimePhase =
| 'gift-entering'
| 'gift-opening'
| 'item-appearing'
| 'active'
| 'correct'
| 'wrong'
| 'complete';
type RuntimeRound = {
item: BabyObjectMatchItemAsset;
@@ -65,23 +78,16 @@ type RuntimeMocapHandPaths = {
};
type BabyObjectMatchRandom = () => number;
const OPEN_PALM_ACTIONS = [
'open_palm',
'open_palm_up',
'open',
'palm',
'hand_open',
];
const GRAB_ACTIONS = [
'grab',
'grabbing',
'close',
'fist',
'closed_fist',
'closed',
];
type BabyObjectMatchStageStyle = CSSProperties &
Partial<
Record<
| '--baby-object-ui-frame-image'
| '--baby-object-gift-box-image'
| '--baby-object-basket-image'
| '--baby-object-smoke-image',
string
>
>;
function pickRandomIndex(length: number, random: BabyObjectMatchRandom) {
if (length <= 1) {
@@ -114,10 +120,6 @@ function isHorizontalDrag(dragState: DragState) {
);
}
function hasMocapAction(command: MocapInputCommand, actions: string[]) {
return command.actions.some((action) => actions.includes(action));
}
function mocapHandToRuntimePoint(
hand: MocapHandInput | null | undefined,
): RuntimeHandPoint | null {
@@ -165,26 +167,6 @@ function resolveMocapHandPaths(
} satisfies RuntimeMocapHandPaths;
}
function hasOpenPalmMocapHand(command: MocapInputCommand) {
return (
hasMocapAction(command, OPEN_PALM_ACTIONS) ||
Boolean(command.hands?.some((hand) => hand.state === 'open_palm')) ||
command.leftHand?.state === 'open_palm' ||
command.rightHand?.state === 'open_palm' ||
command.primaryHand?.state === 'open_palm'
);
}
function hasGrabMocapHand(command: MocapInputCommand) {
return (
hasMocapAction(command, GRAB_ACTIONS) ||
Boolean(command.hands?.some((hand) => hand.state === 'grab')) ||
command.leftHand?.state === 'grab' ||
command.rightHand?.state === 'grab' ||
command.primaryHand?.state === 'grab'
);
}
function resolveMocapHorizontalMoveSide(
paths: RuntimeMocapHandPaths,
): BasketSide | null {
@@ -208,6 +190,20 @@ function buildMocapPacketKey(
: JSON.stringify(command);
}
function findVisualAsset(
draft: BabyObjectMatchDraft,
kind: BabyObjectMatchVisualAssetKind,
): BabyObjectMatchVisualAsset | null {
return (
draft.visualPackage?.assets.find((asset) => asset.assetKind === kind) ??
null
);
}
function buildCssImageValue(src: string) {
return `url("${src.replace(/"/gu, '\\"')}")`;
}
export function BabyObjectMatchRuntimeShell({
draft,
embedded = false,
@@ -217,33 +213,92 @@ export function BabyObjectMatchRuntimeShell({
onBack,
onNextLevel,
}: BabyObjectMatchRuntimeShellProps) {
const randomRef = useRef<BabyObjectMatchRandom>(random ?? (() => Math.random()));
const randomRef = useRef<BabyObjectMatchRandom>(
random ?? (() => Math.random()),
);
const introTimerRef = useRef<number | null>(null);
const feedbackTimerRef = useRef<number | null>(null);
const dragStateRef = useRef<DragState | null>(null);
const handledMocapPacketKeyRef = useRef<string | null>(null);
const hasOpenPalmBeforeGrabRef = useRef(false);
const latestMocapPacketKeyRef = useRef<string | null>(null);
const mocapHandPathsRef = useRef<RuntimeMocapHandPaths>({
left: [],
right: [],
});
const [phase, setPhase] = useState<RuntimePhase>('waiting');
const [phase, setPhase] = useState<RuntimePhase>('gift-entering');
const [successCount, setSuccessCount] = useState(0);
const [round, setRound] = useState<RuntimeRound | null>(null);
const [round, setRound] = useState<RuntimeRound | null>(() =>
buildRuntimeRound(draft, randomRef.current),
);
const [feedbackText, setFeedbackText] = useState<string | null>(null);
const [lastTargetSide, setLastTargetSide] = useState<BasketSide | null>(null);
const liveMocapInput = useMocapInput({
enabled: enableMocapInput && !mocapInput,
});
const resolvedMocapInput = mocapInput ?? liveMocapInput;
const backgroundAsset = findVisualAsset(draft, 'background');
const uiFrameAsset = findVisualAsset(draft, 'ui-frame');
const giftBoxAsset = findVisualAsset(draft, 'gift-box');
const basketAsset = findVisualAsset(draft, 'basket');
const smokeAsset = findVisualAsset(draft, 'smoke-puff');
const stageStyle: BabyObjectMatchStageStyle = {
...(uiFrameAsset
? {
'--baby-object-ui-frame-image': buildCssImageValue(
uiFrameAsset.imageSrc,
),
}
: {}),
...(giftBoxAsset
? {
'--baby-object-gift-box-image': buildCssImageValue(
giftBoxAsset.imageSrc,
),
}
: {}),
...(basketAsset
? {
'--baby-object-basket-image': buildCssImageValue(
basketAsset.imageSrc,
),
}
: {}),
...(smokeAsset
? {
'--baby-object-smoke-image': buildCssImageValue(
smokeAsset.imageSrc,
),
}
: {}),
};
const progressText = `${successCount}/${BABY_OBJECT_MATCH_SUCCESS_TARGET}`;
const isComplete = phase === 'complete';
const currentItem = round?.item ?? null;
const isJudgementOpen = phase === 'active';
const shouldShowCurrentItem =
currentItem &&
(phase === 'item-appearing' ||
phase === 'active' ||
phase === 'correct' ||
phase === 'wrong');
const shouldShowGift = phase === 'gift-entering' || phase === 'gift-opening';
const shouldShowSmoke =
phase === 'gift-opening' || phase === 'item-appearing';
useEffect(() => {
randomRef.current = random ?? (() => Math.random());
}, [random]);
useEffect(() => {
latestMocapPacketKeyRef.current = resolvedMocapInput.latestCommand
? buildMocapPacketKey(
resolvedMocapInput.latestCommand,
resolvedMocapInput.rawPacketPreview,
)
: null;
}, [resolvedMocapInput.latestCommand, resolvedMocapInput.rawPacketPreview]);
const clearFeedbackTimer = useCallback(() => {
if (feedbackTimerRef.current !== null) {
window.clearTimeout(feedbackTimerRef.current);
@@ -251,33 +306,65 @@ export function BabyObjectMatchRuntimeShell({
}
}, []);
const openGiftBox = useCallback(() => {
if (phase !== 'waiting') {
return;
const clearIntroTimer = useCallback(() => {
if (introTimerRef.current !== null) {
window.clearTimeout(introTimerRef.current);
introTimerRef.current = null;
}
}, []);
clearFeedbackTimer();
setFeedbackText(null);
setLastTargetSide(null);
setRound(buildRuntimeRound(draft, randomRef.current));
setPhase('active');
}, [clearFeedbackTimer, draft, phase]);
const resetRuntime = useCallback(() => {
clearFeedbackTimer();
const resetInputPaths = useCallback(() => {
dragStateRef.current = null;
handledMocapPacketKeyRef.current = null;
hasOpenPalmBeforeGrabRef.current = false;
mocapHandPathsRef.current = { left: [], right: [] };
}, []);
useEffect(() => {
clearIntroTimer();
if (phase === 'gift-entering') {
introTimerRef.current = window.setTimeout(() => {
introTimerRef.current = null;
setPhase('gift-opening');
}, BABY_OBJECT_MATCH_GIFT_APPEAR_DURATION_MS);
return clearIntroTimer;
}
if (phase === 'gift-opening') {
introTimerRef.current = window.setTimeout(() => {
introTimerRef.current = null;
setPhase('item-appearing');
}, BABY_OBJECT_MATCH_GIFT_OPEN_DURATION_MS);
return clearIntroTimer;
}
if (phase === 'item-appearing') {
introTimerRef.current = window.setTimeout(() => {
introTimerRef.current = null;
resetInputPaths();
handledMocapPacketKeyRef.current = latestMocapPacketKeyRef.current;
setPhase('active');
}, BABY_OBJECT_MATCH_ITEM_APPEAR_DURATION_MS);
return clearIntroTimer;
}
return clearIntroTimer;
}, [clearIntroTimer, phase, resetInputPaths]);
const resetRuntime = useCallback(() => {
clearIntroTimer();
clearFeedbackTimer();
resetInputPaths();
setSuccessCount(0);
setRound(null);
setRound(buildRuntimeRound(draft, randomRef.current));
setFeedbackText(null);
setLastTargetSide(null);
setPhase('waiting');
}, [clearFeedbackTimer]);
setPhase('gift-entering');
}, [clearFeedbackTimer, clearIntroTimer, draft, resetInputPaths]);
const finishFeedback = useCallback(
(nextSuccessCount: number, wasCorrect: boolean) => {
clearIntroTimer();
clearFeedbackTimer();
feedbackTimerRef.current = window.setTimeout(() => {
feedbackTimerRef.current = null;
@@ -289,25 +376,26 @@ export function BabyObjectMatchRuntimeShell({
return;
}
setRound(null);
setRound(buildRuntimeRound(draft, randomRef.current));
setFeedbackText(null);
setLastTargetSide(null);
setPhase('waiting');
resetInputPaths();
setPhase('gift-entering');
return;
}
setFeedbackText(null);
setLastTargetSide(null);
mocapHandPathsRef.current = { left: [], right: [] };
resetInputPaths();
setPhase('active');
}, BABY_OBJECT_MATCH_FEEDBACK_DURATION_MS);
},
[clearFeedbackTimer],
[clearFeedbackTimer, clearIntroTimer, draft, resetInputPaths],
);
const sendItemToBasket = useCallback(
(side: BasketSide) => {
if (phase !== 'active' || !round) {
if (!isJudgementOpen || !round) {
return;
}
@@ -326,18 +414,16 @@ export function BabyObjectMatchRuntimeShell({
setPhase('wrong');
finishFeedback(successCount, false);
},
[finishFeedback, phase, round, successCount],
[finishFeedback, isJudgementOpen, round, successCount],
);
useEffect(() => clearFeedbackTimer, [clearFeedbackTimer]);
useEffect(() => {
if (phase === 'waiting') {
mocapHandPathsRef.current = { left: [], right: [] };
return;
}
hasOpenPalmBeforeGrabRef.current = false;
}, [phase]);
useEffect(
() => () => {
clearIntroTimer();
clearFeedbackTimer();
},
[clearFeedbackTimer, clearIntroTimer],
);
useEffect(() => {
const command = resolvedMocapInput.latestCommand;
@@ -354,60 +440,28 @@ export function BabyObjectMatchRuntimeShell({
}
handledMocapPacketKeyRef.current = packetKey;
if (phase === 'waiting') {
if (hasGrabMocapHand(command) && hasOpenPalmBeforeGrabRef.current) {
hasOpenPalmBeforeGrabRef.current = false;
mocapHandPathsRef.current = { left: [], right: [] };
openGiftBox();
return;
}
if (hasOpenPalmMocapHand(command)) {
hasOpenPalmBeforeGrabRef.current = true;
}
if (!isJudgementOpen) {
resetInputPaths();
return;
}
if (phase !== 'active') {
mocapHandPathsRef.current = { left: [], right: [] };
return;
}
const nextPaths = resolveMocapHandPaths(
command,
mocapHandPathsRef.current,
);
const nextPaths = resolveMocapHandPaths(command, mocapHandPathsRef.current);
mocapHandPathsRef.current = nextPaths;
const targetSide = resolveMocapHorizontalMoveSide(nextPaths);
if (targetSide) {
sendItemToBasket(targetSide);
mocapHandPathsRef.current = { left: [], right: [] };
resetInputPaths();
}
}, [
isComplete,
openGiftBox,
phase,
isJudgementOpen,
resetInputPaths,
resolvedMocapInput.latestCommand,
resolvedMocapInput.rawPacketPreview,
sendItemToBasket,
]);
useEffect(() => {
const handleKeyDown = (event: KeyboardEvent) => {
if (event.key.toLowerCase() !== 'f') {
return;
}
event.preventDefault();
openGiftBox();
};
window.addEventListener('keydown', handleKeyDown);
return () => {
window.removeEventListener('keydown', handleKeyDown);
};
}, [openGiftBox]);
const getPointerUnitX = (
event: ReactPointerEvent<HTMLElement>,
element: HTMLElement,
@@ -418,6 +472,10 @@ export function BabyObjectMatchRuntimeShell({
};
const handlePointerDown = (event: ReactPointerEvent<HTMLElement>) => {
if (!isJudgementOpen) {
return;
}
if (event.button !== 0 && event.button !== 2) {
return;
}
@@ -436,6 +494,11 @@ export function BabyObjectMatchRuntimeShell({
};
const handlePointerMove = (event: ReactPointerEvent<HTMLElement>) => {
if (!isJudgementOpen) {
dragStateRef.current = null;
return;
}
if (!dragStateRef.current) {
return;
}
@@ -469,13 +532,26 @@ export function BabyObjectMatchRuntimeShell({
data-testid="baby-object-match-runtime"
>
<section
className="baby-object-runtime__stage"
className={`baby-object-runtime__stage${
backgroundAsset ? ' baby-object-runtime__stage--skinned' : ''
}`}
style={stageStyle}
onPointerDown={handlePointerDown}
onPointerMove={handlePointerMove}
onPointerUp={handlePointerUp}
onPointerCancel={handlePointerUp}
onContextMenu={(event) => event.preventDefault()}
>
{backgroundAsset ? (
<ResolvedAssetImage
src={backgroundAsset.imageSrc}
alt=""
className="baby-object-runtime__background-image"
data-testid="baby-object-background-image"
aria-hidden="true"
/>
) : null}
{onBack ? (
<button
type="button"
@@ -496,25 +572,65 @@ export function BabyObjectMatchRuntimeShell({
{progressText}
</div>
<div
className={`baby-object-runtime__gift${phase === 'active' || phase === 'correct' || phase === 'wrong' ? ' baby-object-runtime__gift--open' : ''}`}
aria-label="礼物盒"
>
<Gift className="baby-object-runtime__gift-icon" />
</div>
{shouldShowGift ? (
<div
className={`baby-object-runtime__gift${
giftBoxAsset ? ' baby-object-runtime__gift--skinned' : ''
}${
phase === 'gift-entering'
? ' baby-object-runtime__gift--entering'
: ''
}${
phase === 'gift-opening'
? ' baby-object-runtime__gift--opening baby-object-runtime__gift--open'
: ''
}`}
aria-label="礼物盒"
>
{giftBoxAsset ? (
<ResolvedAssetImage
src={giftBoxAsset.imageSrc}
alt="礼物盒"
className="baby-object-runtime__gift-image"
/>
) : (
<Gift className="baby-object-runtime__gift-icon" />
)}
</div>
) : null}
{shouldShowSmoke ? (
<div
className={`baby-object-runtime__smoke${
smokeAsset ? ' baby-object-runtime__smoke--skinned' : ''
}${
phase === 'item-appearing'
? ' baby-object-runtime__smoke--releasing'
: ''
}`}
data-testid="baby-object-smoke-effect"
aria-hidden="true"
/>
) : null}
<div
className={`baby-object-runtime__item${
shouldShowCurrentItem ? ' baby-object-runtime__item--visible' : ''
}${
phase === 'item-appearing'
? ' baby-object-runtime__item--appearing'
: ''
}${
phase === 'correct'
? ` baby-object-runtime__item--to-${lastTargetSide ?? 'left'}`
: phase === 'wrong'
? ` baby-object-runtime__item--wrong-${lastTargetSide ?? 'left'}`
: ''
: ''
}`}
data-testid="baby-object-current-item"
aria-live="polite"
>
{currentItem ? (
{shouldShowCurrentItem ? (
<>
<ResolvedAssetImage
src={currentItem.imageSrc}
@@ -555,12 +671,17 @@ export function BabyObjectMatchRuntimeShell({
<div className="baby-object-runtime__baskets">
{(['left', 'right'] as const).map((side) => {
const basketItem = round?.baskets[side] ?? draft.itemAssets[side === 'left' ? 0 : 1];
const basketItem =
round?.baskets[side] ?? draft.itemAssets[side === 'left' ? 0 : 1];
return (
<div
key={side}
className={`baby-object-runtime__basket baby-object-runtime__basket--${side}`}
className={`baby-object-runtime__basket baby-object-runtime__basket--${side}${
phase === 'correct' && lastTargetSide === side
? ' baby-object-runtime__basket--correct'
: ''
}`}
aria-label={`${side === 'left' ? '左侧' : '右侧'}篮子 ${basketItem.itemName}`}
>
<div className="baby-object-runtime__basket-icon">
@@ -570,7 +691,21 @@ export function BabyObjectMatchRuntimeShell({
className="baby-object-runtime__basket-image"
/>
</div>
<div className="baby-object-runtime__basket-body" />
<div
className={`baby-object-runtime__basket-body${
basketAsset
? ' baby-object-runtime__basket-body--skinned'
: ''
}`}
>
{basketAsset ? (
<ResolvedAssetImage
src={basketAsset.imageSrc}
alt=""
className="baby-object-runtime__basket-shell-image"
/>
) : null}
</div>
</div>
);
})}