feat(edutainment): refresh baby object match flow

This commit is contained in:
2026-05-16 11:29:28 +08:00
parent 49ffa6b901
commit 45daca3647
24 changed files with 6616 additions and 659 deletions

View File

@@ -157,6 +157,7 @@ function dispatchPointerEvent(
options: {
pointerId: number;
button?: number;
buttons?: number;
clientX: number;
clientY: number;
},
@@ -164,9 +165,10 @@ function dispatchPointerEvent(
const event = new Event(type, { bubbles: true, cancelable: true });
Object.assign(event, options);
target.dispatchEvent(event);
return event;
}
function dragHand(stage: HTMLElement, button: 0 | 2) {
function setStageRect(stage: HTMLElement) {
Object.defineProperty(stage, 'getBoundingClientRect', {
configurable: true,
value: () => ({
@@ -181,34 +183,67 @@ function dragHand(stage: HTMLElement, button: 0 | 2) {
toJSON: () => ({}),
}),
});
}
function dragItemWithHand(stage: HTMLElement, button: 0 | 2, targetX: number) {
setStageRect(stage);
act(() => {
dispatchPointerEvent(stage, 'pointerdown', {
pointerId: button + 1,
button,
clientX: 20,
clientY: 140,
buttons: button === 2 ? 2 : 1,
clientX: 160,
clientY: 89,
});
});
act(() => {
dispatchPointerEvent(stage, 'pointermove', {
pointerId: button + 1,
button,
clientX: 120,
clientY: 140,
buttons: button === 2 ? 2 : 1,
clientX: targetX,
clientY: 190,
});
});
act(() => {
dispatchPointerEvent(stage, 'pointerup', {
pointerId: button + 1,
button,
clientX: 120,
clientY: 140,
buttons: 0,
clientX: targetX,
clientY: 190,
});
});
}
async function advanceRoundIntro() {
await advanceInitialTargetPreview();
await advanceGiftIntro();
}
async function advanceInitialTargetPreview() {
await act(async () => {
await vi.advanceTimersByTimeAsync(2000);
});
await act(async () => {
await vi.advanceTimersByTimeAsync(720);
});
await act(async () => {
await vi.advanceTimersByTimeAsync(1000);
});
await act(async () => {
await vi.advanceTimersByTimeAsync(2000);
});
await act(async () => {
await vi.advanceTimersByTimeAsync(720);
});
await act(async () => {
await vi.advanceTimersByTimeAsync(1000);
});
}
async function advanceGiftIntro() {
await act(async () => {
await vi.advanceTimersByTimeAsync(620);
});
@@ -236,6 +271,7 @@ test('shows the first gift item after gift and item animations', async () => {
);
expect(screen.getByText('将物品放入对应的篮子里')).toBeTruthy();
expect(screen.getByTestId('baby-object-intro-item')).toBeTruthy();
expect(screen.getByTestId('baby-object-current-item').textContent).toBe('');
await advanceRoundIntro();
@@ -246,6 +282,56 @@ test('shows the first gift item after gift and item animations', async () => {
vi.useRealTimers();
});
test('previews both target items before the first gift box round', async () => {
vi.useFakeTimers();
render(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={createRandomSequence([0])}
/>,
);
expect(screen.getByTestId('baby-object-intro-item')).toBeTruthy();
expect(
within(screen.getByTestId('baby-object-intro-item')).getByAltText('苹果'),
).toBeTruthy();
expect(screen.queryByLabelText('礼物盒')).toBeNull();
await act(async () => {
await vi.advanceTimersByTimeAsync(2000);
});
expect(screen.getByTestId('baby-object-intro-item').className).toContain(
'baby-object-runtime__intro-item--flying',
);
await act(async () => {
await vi.advanceTimersByTimeAsync(720);
});
await act(async () => {
await vi.advanceTimersByTimeAsync(1000);
});
expect(screen.getByTestId('baby-object-intro-item')).toBeTruthy();
expect(
within(screen.getByTestId('baby-object-intro-item')).getByAltText('香蕉'),
).toBeTruthy();
await act(async () => {
await vi.advanceTimersByTimeAsync(2000);
});
await act(async () => {
await vi.advanceTimersByTimeAsync(720);
});
await act(async () => {
await vi.advanceTimersByTimeAsync(1000);
});
expect(screen.queryByTestId('baby-object-intro-item')).toBeNull();
expect(screen.getByLabelText('礼物盒')).toBeTruthy();
vi.useRealTimers();
});
test('applies generated visual package to stage, gift box, baskets, smoke and hud', async () => {
vi.useFakeTimers();
const { container } = render(
@@ -270,6 +356,8 @@ test('applies generated visual package to stage, gift box, baskets, smoke and hu
expect(stage.style.getPropertyValue('--baby-object-smoke-image')).toContain(
'smoke',
);
await advanceInitialTargetPreview();
expect(screen.getByAltText('礼物盒')).toBeTruthy();
expect(
container.querySelector('.baby-object-runtime__basket-shell-image'),
@@ -283,6 +371,80 @@ test('applies generated visual package to stage, gift box, baskets, smoke and hu
vi.useRealTimers();
});
test('uses default runtime hand indicators instead of per-draft generated hand assets', async () => {
vi.useFakeTimers();
const random = createRandomSequence([0, 0]);
const draftWithLegacyHandAssets: BabyObjectMatchDraft = {
...createVisualPackageDraft(),
visualPackage: {
...createVisualPackageDraft().visualPackage!,
assets: [
...createVisualPackageDraft().visualPackage!.assets,
{
assetId: 'legacy-left-hand',
assetKind: 'left-hand',
imageSrc: 'data:image/png;base64,legacy-left-hand',
assetObjectId: null,
generationProvider: 'vector-engine-gpt-image-2',
prompt: '旧左手',
},
{
assetId: 'legacy-right-hand',
assetKind: 'right-hand',
imageSrc: 'data:image/png;base64,legacy-right-hand',
assetObjectId: null,
generationProvider: 'vector-engine-gpt-image-2',
prompt: '旧右手',
},
],
},
};
const { container, rerender } = render(
<BabyObjectMatchRuntimeShell
draft={draftWithLegacyHandAssets}
random={random}
mocapInput={createMocapInput()}
/>,
);
await advanceRoundIntro();
rerender(
<BabyObjectMatchRuntimeShell
draft={draftWithLegacyHandAssets}
random={random}
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.5, y: 0.37, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'right' },
leftHand: null,
rightHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'right' },
},
rawPacketPreview: {
text: 'show-default-hand',
receivedAtMs: 1,
},
})}
/>,
);
expect(screen.getByTestId('baby-object-left-hand')).toBeTruthy();
const stage = container.querySelector('.baby-object-runtime__stage');
expect(stage).toBeInstanceOf(HTMLElement);
expect(
(stage as HTMLElement).style.getPropertyValue(
'--baby-object-left-hand-image',
),
).toBe('');
expect(
(stage as HTMLElement).style.getPropertyValue(
'--baby-object-right-hand-image',
),
).toBe('');
vi.useRealTimers();
});
test('removes the gift box after smoke releases the current item', async () => {
vi.useFakeTimers();
render(
@@ -292,6 +454,8 @@ test('removes the gift box after smoke releases the current item', async () => {
/>,
);
await advanceInitialTargetPreview();
expect(screen.getByLabelText('礼物盒')).toBeTruthy();
await act(async () => {
@@ -335,10 +499,16 @@ test('keeps left and right baskets fixed while only the gift item is random', as
).toBeTruthy();
expect(screen.getByLabelText('左侧篮子 苹果')).toBeTruthy();
expect(screen.getByLabelText('右侧篮子 香蕉')).toBeTruthy();
expect(
within(screen.getByLabelText('左侧篮子 苹果')).getByText('苹果'),
).toBeTruthy();
expect(
within(screen.getByLabelText('右侧篮子 香蕉')).getByText('香蕉'),
).toBeTruthy();
vi.useRealTimers();
});
test('mocap camera-right hand movement sends the player left hand item into the left basket', async () => {
test('mocap hand must touch the current item before dropping it into a basket', async () => {
vi.useFakeTimers();
const random = createRandomSequence([0, 0]);
const { rerender } = render(
@@ -358,13 +528,211 @@ test('mocap camera-right hand movement sends the player left hand item into the
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.22, y: 0.45, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.22, y: 0.45, state: 'open_palm', side: 'right' },
hands: [{ x: 0.24, y: 0.72, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.24, y: 0.72, state: 'open_palm', side: 'right' },
leftHand: null,
rightHand: { x: 0.22, y: 0.45, state: 'open_palm', side: 'right' },
rightHand: { x: 0.24, y: 0.72, state: 'open_palm', side: 'right' },
},
rawPacketPreview: {
text: 'camera-right-horizontal-1',
text: 'drop-without-grab',
receivedAtMs: 1,
},
})}
/>,
);
expect(screen.queryByText('真棒')).toBeNull();
expect(screen.queryByText('再想一想吧')).toBeNull();
rerender(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={random}
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.5, y: 0.37, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'right' },
leftHand: null,
rightHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'right' },
},
rawPacketPreview: {
text: 'touch-current-item',
receivedAtMs: 2,
},
})}
/>,
);
expect(screen.getByTestId('baby-object-left-hand')).toBeTruthy();
expect(screen.queryByText('真棒')).toBeNull();
rerender(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={random}
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.22, y: 0.78, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.22, y: 0.78, state: 'open_palm', side: 'right' },
leftHand: null,
rightHand: { x: 0.22, y: 0.78, state: 'open_palm', side: 'right' },
},
rawPacketPreview: {
text: 'drop-left-basket',
receivedAtMs: 3,
},
})}
/>,
);
expect(screen.getByText('真棒')).toBeTruthy();
expect(screen.getByLabelText('成功次数').textContent).toBe('1/20');
vi.useRealTimers();
});
test('mocap hand uses skeleton wrist before hand landmark points in baby object runtime', async () => {
vi.useFakeTimers();
const random = createRandomSequence([0, 0]);
const { rerender } = render(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={random}
mocapInput={createMocapInput()}
/>,
);
await advanceRoundIntro();
rerender(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={random}
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [
{
x: 0.5,
y: 0.37,
state: 'open_palm',
side: 'right',
source: 'palm_center',
wrist: { x: 0.62, y: 0.37 },
},
],
primaryHand: {
x: 0.5,
y: 0.37,
state: 'open_palm',
side: 'right',
source: 'palm_center',
wrist: { x: 0.62, y: 0.37 },
},
leftHand: null,
rightHand: {
x: 0.5,
y: 0.37,
state: 'open_palm',
side: 'right',
source: 'palm_center',
wrist: { x: 0.62, y: 0.37 },
},
bodyJoints: {
rightWrist: { x: 0.64, y: 0.37 },
},
},
rawPacketPreview: {
text: 'hand-points-over-item-skeleton-wrist-away',
receivedAtMs: 1,
},
})}
/>,
);
expect(screen.queryByTestId('baby-object-left-hand')).toBeTruthy();
expect(screen.queryByText('真棒')).toBeNull();
rerender(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={random}
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [
{
x: 0.62,
y: 0.37,
state: 'open_palm',
side: 'right',
source: 'palm_center',
wrist: { x: 0.5, y: 0.37 },
},
],
primaryHand: {
x: 0.62,
y: 0.37,
state: 'open_palm',
side: 'right',
source: 'palm_center',
wrist: { x: 0.5, y: 0.37 },
},
leftHand: null,
rightHand: {
x: 0.62,
y: 0.37,
state: 'open_palm',
side: 'right',
source: 'palm_center',
wrist: { x: 0.62, y: 0.37 },
},
bodyJoints: {
rightWrist: { x: 0.5, y: 0.37 },
},
},
rawPacketPreview: {
text: 'skeleton-wrist-over-item-hand-points-away',
receivedAtMs: 2,
},
})}
/>,
);
expect(screen.getByTestId('baby-object-left-hand').className).toContain(
'baby-object-runtime__hand--holding-left-corner',
);
vi.useRealTimers();
});
test('basket judgement accepts the enlarged basket edge while keeping center gap safe', async () => {
vi.useFakeTimers();
const random = createRandomSequence([0, 0]);
const { rerender } = render(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={random}
mocapInput={createMocapInput()}
/>,
);
await advanceRoundIntro();
rerender(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={random}
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.5, y: 0.37, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'right' },
leftHand: null,
rightHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'right' },
},
rawPacketPreview: {
text: 'touch-current-item-before-narrow-zone',
receivedAtMs: 1,
},
})}
@@ -378,40 +746,22 @@ test('mocap camera-right hand movement sends the player left hand item into the
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.24, y: 0.45, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.24, y: 0.45, state: 'open_palm', side: 'right' },
hands: [{ x: 0.37, y: 0.82, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.37, y: 0.82, state: 'open_palm', side: 'right' },
leftHand: null,
rightHand: { x: 0.24, y: 0.45, state: 'open_palm', side: 'right' },
rightHand: { x: 0.37, y: 0.82, state: 'open_palm', side: 'right' },
},
rawPacketPreview: {
text: 'camera-right-horizontal-2',
text: 'outside-enlarged-left-hitbox-center-gap',
receivedAtMs: 2,
},
})}
/>,
);
rerender(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={random}
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.22, y: 0.45, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.22, y: 0.45, state: 'open_palm', side: 'right' },
leftHand: null,
rightHand: { x: 0.22, y: 0.45, state: 'open_palm', side: 'right' },
},
rawPacketPreview: {
text: 'camera-right-horizontal-3',
receivedAtMs: 3,
},
})}
/>,
);
expect(screen.queryByText('真棒')).toBeNull();
expect(screen.queryByText('再想一想吧')).toBeNull();
expect(screen.getByLabelText('成功次数').textContent).toBe('0/20');
rerender(
<BabyObjectMatchRuntimeShell
@@ -420,14 +770,14 @@ test('mocap camera-right hand movement sends the player left hand item into the
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.31, y: 0.45, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.31, y: 0.45, state: 'open_palm', side: 'right' },
hands: [{ x: 0.36, y: 0.62, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.36, y: 0.62, state: 'open_palm', side: 'right' },
leftHand: null,
rightHand: { x: 0.31, y: 0.45, state: 'open_palm', side: 'right' },
rightHand: { x: 0.36, y: 0.62, state: 'open_palm', side: 'right' },
},
rawPacketPreview: {
text: 'camera-right-horizontal-4',
receivedAtMs: 4,
text: 'enlarged-left-hitbox-edge',
receivedAtMs: 3,
},
})}
/>,
@@ -438,7 +788,7 @@ test('mocap camera-right hand movement sends the player left hand item into the
vi.useRealTimers();
});
test('mocap camera-left hand movement sends the player right hand item into the right basket', async () => {
test('either mocap hand can drag the current item into either basket', async () => {
vi.useFakeTimers();
const random = createRandomSequence([0, 0]);
const { rerender } = render(
@@ -458,12 +808,12 @@ test('mocap camera-left hand movement sends the player right hand item into the
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.82, y: 0.45, state: 'open_palm', side: 'left' }],
primaryHand: { x: 0.82, y: 0.45, state: 'open_palm', side: 'left' },
leftHand: { x: 0.82, y: 0.45, state: 'open_palm', side: 'left' },
hands: [{ x: 0.5, y: 0.37, state: 'open_palm', side: 'left' }],
primaryHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'left' },
leftHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'left' },
rightHand: null,
},
rawPacketPreview: { text: 'camera-left-horizontal-1', receivedAtMs: 1 },
rawPacketPreview: { text: 'right-hand-touch-item', receivedAtMs: 1 },
})}
/>,
);
@@ -475,48 +825,12 @@ test('mocap camera-left hand movement sends the player right hand item into the
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.8, y: 0.45, state: 'open_palm', side: 'left' }],
primaryHand: { x: 0.8, y: 0.45, state: 'open_palm', side: 'left' },
leftHand: { x: 0.8, y: 0.45, state: 'open_palm', side: 'left' },
hands: [{ x: 0.78, y: 0.78, state: 'open_palm', side: 'left' }],
primaryHand: { x: 0.78, y: 0.78, state: 'open_palm', side: 'left' },
leftHand: { x: 0.78, y: 0.78, state: 'open_palm', side: 'left' },
rightHand: null,
},
rawPacketPreview: { text: 'camera-left-horizontal-2', receivedAtMs: 2 },
})}
/>,
);
rerender(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={random}
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.82, y: 0.45, state: 'open_palm', side: 'left' }],
primaryHand: { x: 0.82, y: 0.45, state: 'open_palm', side: 'left' },
leftHand: { x: 0.82, y: 0.45, state: 'open_palm', side: 'left' },
rightHand: null,
},
rawPacketPreview: { text: 'camera-left-horizontal-3', receivedAtMs: 3 },
})}
/>,
);
expect(screen.queryByText('再想一想吧')).toBeNull();
rerender(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={random}
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.73, y: 0.45, state: 'open_palm', side: 'left' }],
primaryHand: { x: 0.73, y: 0.45, state: 'open_palm', side: 'left' },
leftHand: { x: 0.73, y: 0.45, state: 'open_palm', side: 'left' },
rightHand: null,
},
rawPacketPreview: { text: 'camera-left-horizontal-4', receivedAtMs: 4 },
rawPacketPreview: { text: 'right-hand-drop-right', receivedAtMs: 2 },
})}
/>,
);
@@ -526,7 +840,84 @@ test('mocap camera-left hand movement sends the player right hand item into the
vi.useRealTimers();
});
test('mocap action names do not select a basket without horizontal hand movement', async () => {
test('holding hand indicator anchors to the lower item corner by hand side', async () => {
vi.useFakeTimers();
const leftHandRandom = createRandomSequence([0, 0]);
const leftHandRuntime = render(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={leftHandRandom}
mocapInput={createMocapInput()}
/>,
);
await advanceRoundIntro();
leftHandRuntime.rerender(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={leftHandRandom}
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.5, y: 0.37, state: 'open_palm', side: 'right' }],
primaryHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'right' },
leftHand: null,
rightHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'right' },
},
rawPacketPreview: {
text: 'left-hand-holding-corner',
receivedAtMs: 1,
},
})}
/>,
);
expect(screen.getByTestId('baby-object-left-hand').className).toContain(
'baby-object-runtime__hand--holding-left-corner',
);
leftHandRuntime.unmount();
const rightHandRandom = createRandomSequence([0, 0]);
const rightHandRuntime = render(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={rightHandRandom}
mocapInput={createMocapInput()}
/>,
);
await advanceRoundIntro();
rightHandRuntime.rerender(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={rightHandRandom}
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x: 0.5, y: 0.37, state: 'open_palm', side: 'left' }],
primaryHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'left' },
leftHand: { x: 0.5, y: 0.37, state: 'open_palm', side: 'left' },
rightHand: null,
},
rawPacketPreview: {
text: 'right-hand-holding-corner',
receivedAtMs: 2,
},
})}
/>,
);
expect(screen.getByTestId('baby-object-right-hand').className).toContain(
'baby-object-runtime__hand--holding-right-corner',
);
rightHandRuntime.unmount();
vi.useRealTimers();
});
test('mocap action names do not select a basket without touching and dragging item', async () => {
vi.useFakeTimers();
const random = createRandomSequence([0, 0]);
const { rerender } = render(
@@ -564,7 +955,7 @@ test('mocap action names do not select a basket without horizontal hand movement
vi.useRealTimers();
});
test('mocap unknown hand horizontal movement does not select a basket', async () => {
test('mocap unknown hand movement does not grab or select a basket', async () => {
vi.useFakeTimers();
const random = createRandomSequence([0, 0]);
const { rerender } = render(
@@ -578,7 +969,8 @@ test('mocap unknown hand horizontal movement does not select a basket', async ()
await advanceRoundIntro();
for (let index = 0; index < 4; index += 1) {
const x = [0.22, 0.24, 0.22, 0.31][index] ?? 0.22;
const x = [0.5, 0.5, 0.22, 0.22][index] ?? 0.5;
const y = [0.37, 0.78, 0.78, 0.37][index] ?? 0.37;
rerender(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
@@ -586,13 +978,13 @@ test('mocap unknown hand horizontal movement does not select a basket', async ()
mocapInput={createMocapInput({
latestCommand: {
actions: [],
hands: [{ x, y: 0.45, state: 'open_palm', side: 'unknown' }],
primaryHand: { x, y: 0.45, state: 'open_palm', side: 'unknown' },
hands: [{ x, y, state: 'open_palm', side: 'unknown' }],
primaryHand: { x, y, state: 'open_palm', side: 'unknown' },
leftHand: null,
rightHand: null,
},
rawPacketPreview: {
text: `unknown-horizontal-${index + 1}`,
text: `unknown-drag-${index + 1}`,
receivedAtMs: index + 1,
},
})}
@@ -608,7 +1000,7 @@ test('mocap unknown hand horizontal movement does not select a basket', async ()
vi.useRealTimers();
});
test('left hand horizontal drag sends a correct item into the left basket', async () => {
test('left mouse hand drags a correct item into the left basket', async () => {
vi.useFakeTimers();
const { container } = render(
<BabyObjectMatchRuntimeShell
@@ -622,7 +1014,7 @@ test('left hand horizontal drag sends a correct item into the left basket', asyn
}
await advanceRoundIntro();
dragHand(stage, 0);
dragItemWithHand(stage, 0, 70);
expect(screen.getByText('真棒')).toBeTruthy();
expect(screen.getByLabelText('成功次数').textContent).toBe('1/20');
@@ -657,19 +1049,55 @@ test('ignores drag input until the item animation finishes', async () => {
throw new Error('Missing baby object runtime stage');
}
dragHand(stage, 0);
dragItemWithHand(stage, 0, 70);
expect(screen.queryByText('真棒')).toBeNull();
expect(screen.getByLabelText('成功次数').textContent).toBe('0/20');
await advanceRoundIntro();
dragHand(stage, 0);
dragItemWithHand(stage, 0, 70);
expect(screen.getByText('真棒')).toBeTruthy();
expect(screen.getByLabelText('成功次数').textContent).toBe('1/20');
vi.useRealTimers();
});
test('keeps the back button outside active gameplay pointer input', async () => {
vi.useFakeTimers();
const onBack = vi.fn();
render(
<BabyObjectMatchRuntimeShell
draft={createDraft()}
random={createRandomSequence([0, 0])}
onBack={onBack}
/>,
);
await advanceRoundIntro();
const backButton = screen.getByRole('button', { name: '返回' });
let pointerDownEvent!: Event;
act(() => {
pointerDownEvent = dispatchPointerEvent(backButton, 'pointerdown', {
pointerId: 9,
button: 0,
buttons: 1,
clientX: 16,
clientY: 16,
});
});
expect(pointerDownEvent.defaultPrevented).toBe(false);
expect(screen.queryByTestId('baby-object-left-hand')).toBeNull();
act(() => {
backButton.click();
});
expect(onBack).toHaveBeenCalledTimes(1);
vi.useRealTimers();
});
test('correct placement automatically shows the next gift item', async () => {
vi.useFakeTimers();
const { container } = render(
@@ -691,7 +1119,7 @@ test('correct placement automatically shows the next gift item', async () => {
within(screen.getByTestId('baby-object-current-item')).getByAltText('苹果'),
).toBeTruthy();
dragHand(stage, 0);
dragItemWithHand(stage, 0, 70);
expect(screen.getByText('真棒')).toBeTruthy();
@@ -722,7 +1150,7 @@ test('wrong basket keeps the item active after feedback', async () => {
}
await advanceRoundIntro();
dragHand(stage, 2);
dragItemWithHand(stage, 2, 250);
expect(screen.getByText('再想一想吧')).toBeTruthy();
expect(screen.getByLabelText('成功次数').textContent).toBe('0/20');
@@ -752,7 +1180,7 @@ test('twenty correct placements completes the level', async () => {
for (let index = 0; index < 20; index += 1) {
await advanceRoundIntro();
dragHand(stage, 0);
dragItemWithHand(stage, 0, 70);
await advanceFeedback();
}

View File

@@ -33,8 +33,15 @@ const BABY_OBJECT_MATCH_GIFT_APPEAR_DURATION_MS = 620;
const BABY_OBJECT_MATCH_GIFT_OPEN_DURATION_MS = 640;
const BABY_OBJECT_MATCH_ITEM_APPEAR_DURATION_MS = 620;
const BABY_OBJECT_MATCH_FEEDBACK_DURATION_MS = 1180;
const BABY_OBJECT_MATCH_MIN_HORIZONTAL_MOVE_DISTANCE = 0.05;
const BABY_OBJECT_MATCH_HAND_PATH_LIMIT = 16;
const BABY_OBJECT_MATCH_INTRO_SHOW_DURATION_MS = 2000;
const BABY_OBJECT_MATCH_INTRO_FLY_DURATION_MS = 720;
const BABY_OBJECT_MATCH_INTRO_READY_PAUSE_MS = 1000;
const BABY_OBJECT_MATCH_ITEM_CENTER: RuntimeHandPoint = { x: 0.5, y: 0.37 };
const BABY_OBJECT_MATCH_ITEM_GRAB_RADIUS = 0.14;
// 篮子仍只认主体附近,但在上一版核心区基础上扩大约 50%,避免贴近篮子后仍难以命中。
const BABY_OBJECT_MATCH_BASKET_DROP_Y = 0.62;
const BABY_OBJECT_MATCH_LEFT_BASKET_MAX_X = 0.36;
const BABY_OBJECT_MATCH_RIGHT_BASKET_MIN_X = 0.64;
type BabyObjectMatchRuntimeShellProps = {
draft: BabyObjectMatchDraft;
@@ -48,6 +55,12 @@ type BabyObjectMatchRuntimeShellProps = {
type BasketSide = 'left' | 'right';
type RuntimePhase =
| 'intro-left-showing'
| 'intro-left-flying'
| 'intro-left-ready'
| 'intro-right-showing'
| 'intro-right-flying'
| 'intro-right-ready'
| 'gift-entering'
| 'gift-opening'
| 'item-appearing'
@@ -61,10 +74,10 @@ type RuntimeRound = {
baskets: Record<BasketSide, BabyObjectMatchItemAsset>;
};
type DragState = {
type RuntimeIntroShowcase = {
side: BasketSide;
startX: number;
lastX: number;
item: BabyObjectMatchItemAsset;
isFlying: boolean;
};
type RuntimeHandPoint = {
@@ -72,9 +85,12 @@ type RuntimeHandPoint = {
y: number;
};
type RuntimeMocapHandPaths = {
left: RuntimeHandPoint[];
right: RuntimeHandPoint[];
type RuntimeHandRole = 'left' | 'right';
type RuntimeHands = Record<RuntimeHandRole, RuntimeHandPoint | null>;
type HeldItemState = {
hand: RuntimeHandRole;
};
type BabyObjectMatchRandom = () => number;
@@ -113,74 +129,72 @@ function buildRuntimeRound(
};
}
function isHorizontalDrag(dragState: DragState) {
return (
Math.abs(dragState.lastX - dragState.startX) >=
BABY_OBJECT_MATCH_MIN_HORIZONTAL_MOVE_DISTANCE
);
}
function mocapHandToRuntimePoint(
hand: MocapHandInput | null | undefined,
skeletonWrist: RuntimeHandPoint | null | undefined,
): RuntimeHandPoint | null {
if (skeletonWrist) {
return clampRuntimePoint(skeletonWrist);
}
if (!hand) {
return null;
}
return { x: hand.x, y: hand.y };
// 骨架 wrist 缺失时再回退到手部 landmarks 的 wrist最后才使用手部派生点。
const point = hand.wrist ?? hand;
return clampRuntimePoint({ x: point.x, y: point.y });
}
function appendRuntimeHandPoint(
points: RuntimeHandPoint[],
point: RuntimeHandPoint,
) {
return [...points, point].slice(-BABY_OBJECT_MATCH_HAND_PATH_LIMIT);
function clampRuntimePoint(point: RuntimeHandPoint): RuntimeHandPoint {
return {
x: Math.max(0, Math.min(1, point.x)),
y: Math.max(0, Math.min(1, point.y)),
};
}
function hasRuntimeHorizontalMovePath(points: RuntimeHandPoint[]) {
if (points.length < 3) {
return false;
}
function isRuntimePointTouchingItem(point: RuntimeHandPoint) {
const dx = point.x - BABY_OBJECT_MATCH_ITEM_CENTER.x;
const dy = point.y - BABY_OBJECT_MATCH_ITEM_CENTER.y;
return Math.sqrt(dx * dx + dy * dy) <= BABY_OBJECT_MATCH_ITEM_GRAB_RADIUS;
}
const xValues = points.map((point) => point.x);
function isRuntimeControlPointerTarget(target: EventTarget | null) {
return (
Math.max(...xValues) - Math.min(...xValues) >=
BABY_OBJECT_MATCH_MIN_HORIZONTAL_MOVE_DISTANCE
target instanceof Element &&
target.closest(
'button, a, input, select, textarea, [role="button"], [data-baby-object-runtime-control="true"]',
) !== null
);
}
function resolveMocapHandPaths(
command: MocapInputCommand,
currentPaths: RuntimeMocapHandPaths,
) {
// 本地 mocap 当前按摄像头视角输出 handedness这里换回用户身体视角再选篮。
const leftPoint = mocapHandToRuntimePoint(command.rightHand);
const rightPoint = mocapHandToRuntimePoint(command.leftHand);
return {
left: leftPoint
? appendRuntimeHandPoint(currentPaths.left, leftPoint)
: currentPaths.left,
right: rightPoint
? appendRuntimeHandPoint(currentPaths.right, rightPoint)
: currentPaths.right,
} satisfies RuntimeMocapHandPaths;
}
function resolveMocapHorizontalMoveSide(
paths: RuntimeMocapHandPaths,
): BasketSide | null {
if (hasRuntimeHorizontalMovePath(paths.left)) {
function resolveBasketSideForPoint(point: RuntimeHandPoint): BasketSide | null {
if (point.y < BABY_OBJECT_MATCH_BASKET_DROP_Y) {
return null;
}
if (point.x <= BABY_OBJECT_MATCH_LEFT_BASKET_MAX_X) {
return 'left';
}
if (hasRuntimeHorizontalMovePath(paths.right)) {
if (point.x >= BABY_OBJECT_MATCH_RIGHT_BASKET_MIN_X) {
return 'right';
}
return null;
}
function resolveMocapRuntimeHands(command: MocapInputCommand): RuntimeHands {
// 本地 mocap 当前按摄像头视角输出 handedness这里换回用户身体视角用于显示双手。
return {
left: mocapHandToRuntimePoint(
command.rightHand,
command.bodyJoints?.rightWrist,
),
right: mocapHandToRuntimePoint(
command.leftHand,
command.bodyJoints?.leftWrist,
),
};
}
function buildMocapPacketKey(
command: MocapInputCommand,
rawPacketPreview: UseMocapInputResult['rawPacketPreview'],
@@ -204,6 +218,44 @@ function buildCssImageValue(src: string) {
return `url("${src.replace(/"/gu, '\\"')}")`;
}
function resolveIntroShowcase(
phase: RuntimePhase,
draft: BabyObjectMatchDraft,
): RuntimeIntroShowcase | null {
if (phase === 'intro-left-showing' || phase === 'intro-left-flying') {
const item = draft.itemAssets[0];
return item
? { side: 'left', item, isFlying: phase === 'intro-left-flying' }
: null;
}
if (phase === 'intro-right-showing' || phase === 'intro-right-flying') {
const item = draft.itemAssets[1];
return item
? { side: 'right', item, isFlying: phase === 'intro-right-flying' }
: null;
}
return null;
}
function isBasketOptionReadyInIntro(side: BasketSide, phase: RuntimePhase) {
if (!phase.startsWith('intro-')) {
return true;
}
if (side === 'left') {
return (
phase === 'intro-left-ready' ||
phase === 'intro-right-showing' ||
phase === 'intro-right-flying' ||
phase === 'intro-right-ready'
);
}
return phase === 'intro-right-ready';
}
export function BabyObjectMatchRuntimeShell({
draft,
embedded = false,
@@ -218,20 +270,20 @@ export function BabyObjectMatchRuntimeShell({
);
const introTimerRef = useRef<number | null>(null);
const feedbackTimerRef = useRef<number | null>(null);
const dragStateRef = useRef<DragState | null>(null);
const handledMocapPacketKeyRef = useRef<string | null>(null);
const latestMocapPacketKeyRef = useRef<string | null>(null);
const mocapHandPathsRef = useRef<RuntimeMocapHandPaths>({
left: [],
right: [],
});
const [phase, setPhase] = useState<RuntimePhase>('gift-entering');
const [phase, setPhase] = useState<RuntimePhase>('intro-left-showing');
const [successCount, setSuccessCount] = useState(0);
const [round, setRound] = useState<RuntimeRound | null>(() =>
buildRuntimeRound(draft, randomRef.current),
);
const [feedbackText, setFeedbackText] = useState<string | null>(null);
const [lastTargetSide, setLastTargetSide] = useState<BasketSide | null>(null);
const [runtimeHands, setRuntimeHands] = useState<RuntimeHands>({
left: null,
right: null,
});
const [heldItem, setHeldItem] = useState<HeldItemState | null>(null);
const liveMocapInput = useMocapInput({
enabled: enableMocapInput && !mocapInput,
});
@@ -276,6 +328,8 @@ export function BabyObjectMatchRuntimeShell({
const isComplete = phase === 'complete';
const currentItem = round?.item ?? null;
const isJudgementOpen = phase === 'active';
const introShowcase = resolveIntroShowcase(phase, draft);
const heldPoint = heldItem ? runtimeHands[heldItem.hand] : null;
const shouldShowCurrentItem =
currentItem &&
(phase === 'item-appearing' ||
@@ -314,14 +368,61 @@ export function BabyObjectMatchRuntimeShell({
}, []);
const resetInputPaths = useCallback(() => {
dragStateRef.current = null;
handledMocapPacketKeyRef.current = null;
mocapHandPathsRef.current = { left: [], right: [] };
setHeldItem(null);
}, []);
useEffect(() => {
clearIntroTimer();
if (phase === 'intro-left-showing') {
introTimerRef.current = window.setTimeout(() => {
introTimerRef.current = null;
setPhase('intro-left-flying');
}, BABY_OBJECT_MATCH_INTRO_SHOW_DURATION_MS);
return clearIntroTimer;
}
if (phase === 'intro-left-flying') {
introTimerRef.current = window.setTimeout(() => {
introTimerRef.current = null;
setPhase('intro-left-ready');
}, BABY_OBJECT_MATCH_INTRO_FLY_DURATION_MS);
return clearIntroTimer;
}
if (phase === 'intro-left-ready') {
introTimerRef.current = window.setTimeout(() => {
introTimerRef.current = null;
setPhase('intro-right-showing');
}, BABY_OBJECT_MATCH_INTRO_READY_PAUSE_MS);
return clearIntroTimer;
}
if (phase === 'intro-right-showing') {
introTimerRef.current = window.setTimeout(() => {
introTimerRef.current = null;
setPhase('intro-right-flying');
}, BABY_OBJECT_MATCH_INTRO_SHOW_DURATION_MS);
return clearIntroTimer;
}
if (phase === 'intro-right-flying') {
introTimerRef.current = window.setTimeout(() => {
introTimerRef.current = null;
setPhase('intro-right-ready');
}, BABY_OBJECT_MATCH_INTRO_FLY_DURATION_MS);
return clearIntroTimer;
}
if (phase === 'intro-right-ready') {
introTimerRef.current = window.setTimeout(() => {
introTimerRef.current = null;
setPhase('gift-entering');
}, BABY_OBJECT_MATCH_INTRO_READY_PAUSE_MS);
return clearIntroTimer;
}
if (phase === 'gift-entering') {
introTimerRef.current = window.setTimeout(() => {
introTimerRef.current = null;
@@ -359,7 +460,7 @@ export function BabyObjectMatchRuntimeShell({
setRound(buildRuntimeRound(draft, randomRef.current));
setFeedbackText(null);
setLastTargetSide(null);
setPhase('gift-entering');
setPhase('intro-left-showing');
}, [clearFeedbackTimer, clearIntroTimer, draft, resetInputPaths]);
const finishFeedback = useCallback(
@@ -440,20 +541,38 @@ export function BabyObjectMatchRuntimeShell({
}
handledMocapPacketKeyRef.current = packetKey;
const nextHands = resolveMocapRuntimeHands(command);
setRuntimeHands(nextHands);
if (!isJudgementOpen) {
resetInputPaths();
return;
}
const nextPaths = resolveMocapHandPaths(command, mocapHandPathsRef.current);
mocapHandPathsRef.current = nextPaths;
const targetSide = resolveMocapHorizontalMoveSide(nextPaths);
if (targetSide) {
const currentHeldItem = heldItem;
if (currentHeldItem) {
const heldHandPoint = nextHands[currentHeldItem.hand];
const targetSide = heldHandPoint
? resolveBasketSideForPoint(heldHandPoint)
: null;
if (!targetSide) {
return;
}
sendItemToBasket(targetSide);
resetInputPaths();
return;
}
for (const hand of ['left', 'right'] as const) {
const point = nextHands[hand];
if (!point || !isRuntimePointTouchingItem(point)) {
continue;
}
setHeldItem({ hand });
return;
}
}, [
heldItem,
isComplete,
isJudgementOpen,
resetInputPaths,
@@ -462,16 +581,24 @@ export function BabyObjectMatchRuntimeShell({
sendItemToBasket,
]);
const getPointerUnitX = (
const getPointerUnitPoint = (
event: ReactPointerEvent<HTMLElement>,
element: HTMLElement,
) => {
): RuntimeHandPoint => {
const rect = element.getBoundingClientRect();
const width = rect.width || 1;
return Math.max(0, Math.min(1, (event.clientX - rect.left) / width));
const height = rect.height || 1;
return clampRuntimePoint({
x: (event.clientX - rect.left) / width,
y: (event.clientY - rect.top) / height,
});
};
const handlePointerDown = (event: ReactPointerEvent<HTMLElement>) => {
if (isRuntimeControlPointerTarget(event.target)) {
return;
}
if (!isJudgementOpen) {
return;
}
@@ -480,13 +607,12 @@ export function BabyObjectMatchRuntimeShell({
return;
}
const side: BasketSide = event.button === 2 ? 'right' : 'left';
const pointerX = getPointerUnitX(event, event.currentTarget);
dragStateRef.current = {
side,
startX: pointerX,
lastX: pointerX,
};
const hand: RuntimeHandRole = event.button === 2 ? 'right' : 'left';
const point = getPointerUnitPoint(event, event.currentTarget);
setRuntimeHands((current) => ({ ...current, [hand]: point }));
if (isRuntimePointTouchingItem(point)) {
setHeldItem({ hand });
}
event.preventDefault();
if (typeof event.currentTarget.setPointerCapture === 'function') {
event.currentTarget.setPointerCapture(event.pointerId);
@@ -494,36 +620,44 @@ export function BabyObjectMatchRuntimeShell({
};
const handlePointerMove = (event: ReactPointerEvent<HTMLElement>) => {
if (isRuntimeControlPointerTarget(event.target)) {
return;
}
if (!isJudgementOpen) {
dragStateRef.current = null;
return;
}
if (!dragStateRef.current) {
if (event.buttons !== 1 && event.buttons !== 2) {
return;
}
dragStateRef.current = {
...dragStateRef.current,
lastX: getPointerUnitX(event, event.currentTarget),
};
const hand: RuntimeHandRole = event.buttons === 2 ? 'right' : 'left';
const point = getPointerUnitPoint(event, event.currentTarget);
setRuntimeHands((current) => ({ ...current, [hand]: point }));
if (!heldItem && isRuntimePointTouchingItem(point)) {
setHeldItem({ hand });
return;
}
if (!heldItem || heldItem.hand !== hand) {
return;
}
const targetSide = resolveBasketSideForPoint(point);
if (targetSide) {
sendItemToBasket(targetSide);
resetInputPaths();
}
};
const handlePointerUp = (event: ReactPointerEvent<HTMLElement>) => {
const dragState = dragStateRef.current;
dragStateRef.current = null;
if (
typeof event.currentTarget.hasPointerCapture === 'function' &&
event.currentTarget.hasPointerCapture(event.pointerId)
) {
event.currentTarget.releasePointerCapture(event.pointerId);
}
if (!dragState || !isHorizontalDrag(dragState)) {
return;
}
sendItemToBasket(dragState.side);
};
return (
@@ -556,6 +690,7 @@ export function BabyObjectMatchRuntimeShell({
<button
type="button"
className="baby-object-runtime__back"
data-baby-object-runtime-control="true"
onClick={onBack}
aria-label="返回"
title="返回"
@@ -613,10 +748,31 @@ export function BabyObjectMatchRuntimeShell({
/>
) : null}
{introShowcase ? (
<div
className={`baby-object-runtime__intro-item baby-object-runtime__intro-item--${introShowcase.side}${
introShowcase.isFlying
? ' baby-object-runtime__intro-item--flying'
: ''
}`}
data-testid="baby-object-intro-item"
aria-live="polite"
>
<ResolvedAssetImage
src={introShowcase.item.imageSrc}
alt={introShowcase.item.itemName}
className="baby-object-runtime__intro-item-image"
/>
<span className="baby-object-runtime__intro-item-name">
{introShowcase.item.itemName}
</span>
</div>
) : null}
<div
className={`baby-object-runtime__item${
shouldShowCurrentItem ? ' baby-object-runtime__item--visible' : ''
}${
}${heldPoint ? ' baby-object-runtime__item--held' : ''}${
phase === 'item-appearing'
? ' baby-object-runtime__item--appearing'
: ''
@@ -629,6 +785,14 @@ export function BabyObjectMatchRuntimeShell({
}`}
data-testid="baby-object-current-item"
aria-live="polite"
style={
heldPoint
? ({
'--baby-object-held-x': `${heldPoint.x * 100}%`,
'--baby-object-held-y': `${heldPoint.y * 100}%`,
} as CSSProperties)
: undefined
}
>
{shouldShowCurrentItem ? (
<>
@@ -644,6 +808,34 @@ export function BabyObjectMatchRuntimeShell({
) : null}
</div>
<div className="baby-object-runtime__hands" aria-hidden="true">
{(['left', 'right'] as const).map((hand) => {
const point = runtimeHands[hand];
const isHoldingHand = heldItem?.hand === hand;
if (!point) {
return null;
}
return (
<div
key={hand}
className={`baby-object-runtime__hand baby-object-runtime__hand--${hand}${
isHoldingHand
? ` baby-object-runtime__hand--holding baby-object-runtime__hand--holding-${hand}-corner`
: ''
}`}
data-testid={`baby-object-${hand}-hand`}
style={
{
'--baby-object-hand-x': `${point.x * 100}%`,
'--baby-object-hand-y': `${point.y * 100}%`,
} as CSSProperties
}
/>
);
})}
</div>
{feedbackText ? (
<div
className={`baby-object-runtime__feedback baby-object-runtime__feedback--${phase}`}
@@ -673,6 +865,7 @@ export function BabyObjectMatchRuntimeShell({
{(['left', 'right'] as const).map((side) => {
const basketItem =
round?.baskets[side] ?? draft.itemAssets[side === 'left' ? 0 : 1];
const isOptionReady = isBasketOptionReadyInIntro(side, phase);
return (
<div
@@ -684,12 +877,23 @@ export function BabyObjectMatchRuntimeShell({
}`}
aria-label={`${side === 'left' ? '左侧' : '右侧'} ${basketItem.itemName}`}
>
<div className="baby-object-runtime__basket-icon">
<ResolvedAssetImage
src={basketItem.imageSrc}
alt={basketItem.itemName}
className="baby-object-runtime__basket-image"
/>
<div
className={`baby-object-runtime__basket-option${
isOptionReady
? ' baby-object-runtime__basket-option--ready'
: ''
}`}
>
<div className="baby-object-runtime__basket-icon">
<ResolvedAssetImage
src={basketItem.imageSrc}
alt={basketItem.itemName}
className="baby-object-runtime__basket-image"
/>
</div>
<span className="baby-object-runtime__basket-name">
{basketItem.itemName}
</span>
</div>
<div
className={`baby-object-runtime__basket-body${