GenarrativeAI/Genarrative
6
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Update Match3D/image-generation docs & code

Browse Source 
Adds/updates documentation, assets and implementation for Match3D and puzzle image generation workflows. Key changes: decision logs and pitfalls updated to prefer VectorEngine Gemini for Match3D material sheets and to require edits (multipart) for 1:1 container reference images; guidance added for when to use APIMart vs VectorEngine. .env.example clarified APIMart/Responses config. Many new public assets and PPT visuals added. Code changes across frontend and backend: updated shared contracts, server-rs match3d/puzzle/image-generation handlers, VectorEngine/OpenAI image generation clients, and multiple React components/tests to handle UI/background/container image signing, edits workflow, and puzzle UI background resolution. Added src/services/puzzle-runtime/puzzleUiBackgroundSource.ts and related test updates. Includes notes about multipart HTTP/1.1 requirement and test/verification commands in docs.
This commit is contained in:
高物
2026-05-14 20:34:45 +08:00
parent d33c937ebc
commit 548db78ca7
103 changed files with 6687 additions and 3270 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
src/components/puzzle-runtime/PuzzleRuntimeShell.test.tsx
View File

@@ -1,7 +1,7 @@
/* @vitest-environment jsdom */
import { act, fireEvent, render, screen, within } from '@testing-library/react';
import { beforeEach, expect, test, vi } from 'vitest';
import { expect, test, vi } from 'vitest';
import type { PuzzleRunSnapshot } from '../../../packages/shared/src/contracts/puzzleRuntimeSession';
import { AuthUiContext } from '../auth/AuthUiContext';
@@ -33,41 +33,6 @@ vi.mock('../ResolvedAssetImage', () => ({
}) => (src ? <img src={src} alt={alt} className={className} /> : null),
}));
const mocapMock = vi.hoisted(() => ({
state: 'grab',
x: 0.42,
y: 0.58,
}));
const debugModeMock = vi.hoisted(() => ({
enabled: true,
}));
vi.mock('../../config/debugMode', () => ({
IS_DEBUG_MODE: debugModeMock.enabled,
isDebugMode: () => debugModeMock.enabled,
}));
vi.mock('../../services/useMocapInput', () => ({
useMocapInput: () => ({
status: 'connected',
latestCommand: {
actions: [mocapMock.state],
primaryHand: {x: mocapMock.x, y: mocapMock.y, state: mocapMock.state, source: 'palm_center'},
parseWarnings: [],
},
rawPacketPreview: {text: '{"hands":[{"state":"grab"}]}', receivedAtMs: 1},
error: null,
}),
}));
beforeEach(() => {
debugModeMock.enabled = true;
mocapMock.state = 'grab';
mocapMock.x = 0.42;
mocapMock.y = 0.58;
});
function createAuthValue() {
return {
user: null,
@@ -181,42 +146,7 @@ const clearedRun: PuzzleRunSnapshot = {
},
};
test('调试模式下拼图界面折叠展示 mocap 连接状态，展开后显示最近动作调试信息', () => {
renderPuzzleRuntime(
<PuzzleRuntimeShell
run={{
...clearedRun,
currentLevel: {
...clearedRun.currentLevel!,
status: 'playing',
},
}}
onBack={vi.fn()}
onSwapPieces={vi.fn()}
onDragPiece={vi.fn()}
onAdvanceNextLevel={vi.fn()}
/>,
);
const debugPanel = screen.getByTestId('puzzle-mocap-debug');
expect(within(debugPanel).getByText('mocap: connected')).toBeTruthy();
const toggleButton = within(debugPanel).getByRole('button', {
name: 'mocap: connected',
});
expect(toggleButton.getAttribute('aria-expanded')).toBe('false');
expect(within(debugPanel).queryByText('动作: grab')).toBeNull();
fireEvent.click(toggleButton);
expect(toggleButton.getAttribute('aria-expanded')).toBe('true');
expect(within(debugPanel).getByText('动作: grab')).toBeTruthy();
expect(within(debugPanel).getByText('手势: grab @ 0.42, 0.58')).toBeTruthy();
expect(within(debugPanel).getByText('解析: 无')).toBeTruthy();
expect(within(debugPanel).getByText(/原始:/)).toBeTruthy();
});
test('非调试模式下拼图界面不渲染 mocap 调试面板', () => {
debugModeMock.enabled = false;
test('拼图界面不调用 mocap，也不渲染 mocap 光标或调试面板', () => {
renderPuzzleRuntime(
<PuzzleRuntimeShell
run={{
@@ -234,44 +164,10 @@ test('非调试模式下拼图界面不渲染 mocap 调试面板', () => {
);
expect(screen.queryByTestId('puzzle-mocap-debug')).toBeNull();
expect(screen.queryByTestId('puzzle-mocap-cursor')).toBeNull();
});
test('拼图界面在 mocap open_palm 时显示体感光标', () => {
mocapMock.state = 'open_palm';
mocapMock.x = 0.42;
mocapMock.y = 0.58;
renderPuzzleRuntime(
<PuzzleRuntimeShell
run={{
...clearedRun,
currentLevel: {
...clearedRun.currentLevel!,
status: 'playing',
startedAtMs: Date.now(),
remainingMs: 300_000,
timeLimitMs: 300_000,
},
}}
onBack={vi.fn()}
onSwapPieces={vi.fn()}
onDragPiece={vi.fn()}
onAdvanceNextLevel={vi.fn()}
/>,
);
const cursor = screen.getByTestId('puzzle-mocap-cursor');
expect(cursor).toBeTruthy();
expect(Number.parseFloat(cursor.style.left)).toBeCloseTo(42);
expect(Number.parseFloat(cursor.style.top)).toBeCloseTo(58);
mocapMock.state = 'grab';
mocapMock.x = 0.42;
mocapMock.y = 0.58;
});
test('抓握时会触发拖拽提交并在松开时落子', () => {
mocapMock.state = 'grab';
mocapMock.x = 0.34;
mocapMock.y = 0.34;
test('指针拖拽时会触发拖拽提交并在松开时落子', () => {
const onDragPiece = vi.fn();
const playingRun: PuzzleRunSnapshot = {
...clearedRun,
@@ -358,12 +254,9 @@ test('抓握时会触发拖拽提交并在松开时落子', () => {
);
});
test('mocap 抓握合并大块时按大块锚点提交拖拽', () => {
test('指针拖拽合并大块时按大块锚点提交拖拽', () => {
const originalRequestAnimationFrame = window.requestAnimationFrame;
const originalCancelAnimationFrame = window.cancelAnimationFrame;
mocapMock.state = 'open_palm';
mocapMock.x = 0.2;
mocapMock.y = 0.2;
const onDragPiece = vi.fn();
const mergedRun: PuzzleRunSnapshot = {
...clearedRun,
@@ -405,7 +298,7 @@ test('mocap 抓握合并大块时按大块锚点提交拖拽', () => {
value: vi.fn(),
});
const { container, rerender, unmount } = renderPuzzleRuntime(
const { container, unmount } = renderPuzzleRuntime(
<PuzzleRuntimeShell
run={mergedRun}
onBack={vi.fn()}
@@ -432,48 +325,34 @@ test('mocap 抓握合并大块时按大块锚点提交拖拽', () => {
height: 300,
toJSON: () => ({}),
}) as DOMRect;
const mergedPiece = container.querySelector(
'[data-merged-piece-outline="true"]',
) as HTMLElement | null;
if (!mergedPiece) {
throw new Error('缺少测试合并拼图片');
}
mocapMock.state = 'grab';
mocapMock.x = 0.2;
mocapMock.y = 0.2;
rerender(
<AuthUiContext.Provider value={createAuthValue()}>
<PuzzleRuntimeShell
run={mergedRun}
onBack={vi.fn()}
onSwapPieces={vi.fn()}
onDragPiece={onDragPiece}
onAdvanceNextLevel={vi.fn()}
/>
</AuthUiContext.Provider>,
);
mocapMock.x = 0.7;
mocapMock.y = 0.7;
rerender(
<AuthUiContext.Provider value={createAuthValue()}>
<PuzzleRuntimeShell
run={mergedRun}
onBack={vi.fn()}
onSwapPieces={vi.fn()}
onDragPiece={onDragPiece}
onAdvanceNextLevel={vi.fn()}
/>
</AuthUiContext.Provider>,
);
mocapMock.state = 'open_palm';
rerender(
<AuthUiContext.Provider value={createAuthValue()}>
<PuzzleRuntimeShell
run={mergedRun}
onBack={vi.fn()}
onSwapPieces={vi.fn()}
onDragPiece={onDragPiece}
onAdvanceNextLevel={vi.fn()}
/>
</AuthUiContext.Provider>,
);
act(() => {
dispatchPointerEvent(mergedPiece, 'pointerdown', {
pointerId: 12,
clientX: 60,
clientY: 60,
});
});
act(() => {
dispatchPointerEvent(mergedPiece, 'pointermove', {
pointerId: 12,
clientX: 210,
clientY: 210,
});
});
act(() => {
dispatchPointerEvent(mergedPiece, 'pointerup', {
pointerId: 12,
clientX: 210,
clientY: 210,
});
});
expect(onDragPiece).toHaveBeenCalledTimes(1);
expect(onDragPiece).toHaveBeenCalledWith({
@@ -491,9 +370,6 @@ test('mocap 抓握合并大块时按大块锚点提交拖拽', () => {
configurable: true,
value: originalCancelAnimationFrame,
});
mocapMock.state = 'grab';
mocapMock.x = 0.42;
mocapMock.y = 0.58;
});
test('通关后显示结算弹窗、排行榜和下一关按钮', () => {
@@ -661,6 +537,37 @@ test('运行态优先把关卡 UI 背景渲染为舞台背景', () => {
expect(backgroundImage).toBeTruthy();
});
test('运行态在只有 UI 背景 objectKey 时仍渲染生成背景', () => {
const runWithUiBackground: PuzzleRunSnapshot = {
...clearedRun,
currentLevel: {
...clearedRun.currentLevel!,
status: 'playing',
coverImageSrc: '/generated-puzzle-assets/session/cover.png',
uiBackgroundImageSrc: null,
uiBackgroundImageObjectKey:
'generated-puzzle-assets/session/ui/background-object-key.png',
remainingMs: 300_000,
timeLimitMs: 300_000,
},
};
const { container } = renderPuzzleRuntime(
<PuzzleRuntimeShell
run={runWithUiBackground}
onBack={vi.fn()}
onSwapPieces={vi.fn()}
onDragPiece={vi.fn()}
onAdvanceNextLevel={vi.fn()}
/>,
);
const backgroundImage = container.querySelector(
'img[src="/generated-puzzle-assets/session/ui/background-object-key.png"]',
);
expect(backgroundImage).toBeTruthy();
});
test('关闭通关弹窗后保留底部下一关入口', () => {
vi.useFakeTimers();
const onAdvanceNextLevel = vi.fn();
@@ -1046,9 +953,6 @@ test('移动端点击拼图片时立即触发一次震动反馈', () => {
const originalRequestAnimationFrame = window.requestAnimationFrame;
const originalCancelAnimationFrame = window.cancelAnimationFrame;
const vibrate = vi.fn();
mocapMock.state = 'open_palm';
mocapMock.x = 0.42;
mocapMock.y = 0.58;
const playingRun: PuzzleRunSnapshot = {
...clearedRun,
currentLevel: {

261
src/components/puzzle-runtime/PuzzleRuntimeShell.tsx
View File

@@ -1,8 +1,6 @@
import {
ArrowLeft,
ArrowRight,
ChevronDown,
ChevronUp,
Clock,
Eye,
Lightbulb,
@@ -24,12 +22,11 @@ import type {
PuzzleRuntimePropKind,
SwapPuzzlePiecesRequest,
} from '../../../packages/shared/src/contracts/puzzleRuntimeSession';
import { isDebugMode } from '../../config/debugMode';
import { useResolvedAssetReadUrl } from '../../hooks/useResolvedAssetReadUrl';
import { resolvePuzzleUiBackgroundSource } from '../../services/puzzle-runtime/puzzleUiBackgroundSource';
import {
createRuntimeDragInputController,
createRuntimeInputPointFromClient,
createRuntimeInputPointFromNormalized,
readRuntimeInputElementBounds,
resolveRuntimeInputGridCell,
type RuntimeDragInputSession,
@@ -42,7 +39,6 @@ import {
playRuntimeLevelClearSound,
resolveRuntimeCountdownSecondBucket,
} from '../../services/runtimeAudioFeedback';
import { useMocapInput } from '../../services/useMocapInput';
import { CHROME_ICONS, getNineSliceStyle, UI_CHROME } from '../../uiAssets';
import { useAuthUi } from '../auth/AuthUiContext';
import { PixelIcon } from '../PixelIcon';
@@ -230,8 +226,6 @@ const PUZZLE_HINT_DEMO_DURATION_MS = 1_250;
const PUZZLE_PIECE_PRESS_HAPTIC_PATTERN_MS = 12;
const PUZZLE_EXIT_REMODEL_PROMPT_STORAGE_PREFIX =
'genarrative.puzzle-runtime.exit-remodel-prompt.v1';
const PUZZLE_MOCAP_DRAG_INPUT_ID = 'mocap:primary-hand';
const PUZZLE_MOCAP_CURSOR_FRAME_MS = 1000 / 60;
const shownExitRemodelPromptProfileIds = new Set<string>();
@@ -305,16 +299,6 @@ type PuzzleHintDemoState = {
offsetYPercent: number;
};
type PuzzleMocapCursorState = {
x: number;
y: number;
state: string;
};
type PuzzleMocapCursorSample = PuzzleMocapCursorState & {
receivedAtMs: number;
};
type PuzzleRuntimeDragTargetState = {
pieceId: string;
groupId: string | null;
@@ -376,7 +360,6 @@ export function PuzzleRuntimeShell({
const [isFreezeEffectVisible, setIsFreezeEffectVisible] = useState(false);
const [isPropConfirming, setIsPropConfirming] = useState(false);
const [propConfirmError, setPropConfirmError] = useState<string | null>(null);
const [isMocapDebugExpanded, setIsMocapDebugExpanded] = useState(false);
const [hintDemo, setHintDemo] = useState<PuzzleHintDemoState | null>(null);
const [mergeFlash, setMergeFlash] = useState<PuzzleMergeFlashState | null>(
null,
@@ -414,17 +397,6 @@ export function PuzzleRuntimeShell({
pieceId: string;
groupId: string | null;
} | null>(null);
const [mocapCursor, setMocapCursor] = useState<PuzzleMocapCursorState | null>(
null,
);
const mocapCursorPreviousSampleRef = useRef<PuzzleMocapCursorSample | null>(
null,
);
const mocapCursorTargetSampleRef = useRef<PuzzleMocapCursorSample | null>(null);
const mocapCursorIntervalRef = useRef<number | null>(null);
const updateMocapCursorSampleRef = useRef<(
nextSample: PuzzleMocapCursorSample,
) => void>(() => {});
const runtimeDragInputControllerRef = useRef(
createRuntimeDragInputController<string>(),
);
@@ -470,7 +442,7 @@ export function PuzzleRuntimeShell({
currentLevel?.coverImageSrc ?? null,
);
const { resolvedUrl: resolvedUiBackgroundImage } = useResolvedAssetReadUrl(
currentLevel?.uiBackgroundImageSrc ?? null,
resolvePuzzleUiBackgroundSource(currentLevel) ?? null,
);
const tryPlayBackgroundMusic = useCallback(() => {
const audio = backgroundAudioRef.current;
@@ -483,26 +455,6 @@ export function PuzzleRuntimeShell({
audio.volume = Math.max(0, Math.min(1, musicVolume));
void audio.play().catch(() => {});
}, [musicVolume, resolvedBackgroundMusicSrc, runtimeStatus]);
const mocapInput = useMocapInput({enabled: runtimeStatus === 'playing'});
const primaryMocapHand = mocapInput.latestCommand?.primaryHand;
const primaryMocapHandState = primaryMocapHand?.state;
const primaryMocapHandX = primaryMocapHand?.x;
const primaryMocapHandY = primaryMocapHand?.y;
const mocapActionsLabel =
mocapInput.latestCommand?.actions.length
? mocapInput.latestCommand.actions.join(', ')
: '无';
const mocapHandLabel =
primaryMocapHandState &&
typeof primaryMocapHandX === 'number' &&
typeof primaryMocapHandY === 'number'
? `${primaryMocapHandState} @ ${primaryMocapHandX.toFixed(2)}, ${primaryMocapHandY.toFixed(2)}`
: '无';
const mocapParseWarningLabel = mocapInput.latestCommand?.parseWarnings?.length
? mocapInput.latestCommand.parseWarnings.join('；')
: '无';
const mocapRawPacketLabel = mocapInput.rawPacketPreview?.text ?? '未收到';
const shouldShowMocapDebugPanel = isDebugMode();
useEffect(() => {
currentLevelRef.current = currentLevel;
@@ -1018,31 +970,6 @@ export function PuzzleRuntimeShell({
readRuntimeInputElementBounds(boardRef.current),
);
const resolveBoardInputPointFromNormalized = (
normalizedX: number,
normalizedY: number,
) =>
createRuntimeInputPointFromNormalized(
normalizedX,
normalizedY,
readRuntimeInputElementBounds(boardRef.current),
);
const resetMocapCursorInterpolation = () => {
mocapCursorPreviousSampleRef.current = null;
mocapCursorTargetSampleRef.current = null;
setMocapCursor(null);
};
updateMocapCursorSampleRef.current = (nextSample: PuzzleMocapCursorSample) => {
const previousTarget = mocapCursorTargetSampleRef.current;
mocapCursorPreviousSampleRef.current = previousTarget ?? nextSample;
mocapCursorTargetSampleRef.current = nextSample;
if (!previousTarget) {
setMocapCursor(nextSample);
}
};
const syncRuntimeDragFromController = (
session: RuntimeDragInputSession<string> | null,
) => {
@@ -1103,136 +1030,6 @@ export function PuzzleRuntimeShell({
},
});
useEffect(() => {
const activeSession = runtimeDragInputControllerRef.current.getSession();
if (!board || runtimeStatus !== 'playing' || isInteractionLocked) {
runtimeDragInputControllerRef.current.cancel();
resetMocapCursorInterpolation();
return;
}
if (
!primaryMocapHandState ||
typeof primaryMocapHandX !== 'number' ||
typeof primaryMocapHandY !== 'number'
) {
runtimeDragInputControllerRef.current.cancel(PUZZLE_MOCAP_DRAG_INPUT_ID);
resetMocapCursorInterpolation();
return;
}
const nextSample = {
x: primaryMocapHandX,
y: primaryMocapHandY,
state: primaryMocapHandState,
receivedAtMs: performance.now(),
};
updateMocapCursorSampleRef.current(nextSample);
const handPoint = resolveBoardInputPointFromNormalized(nextSample.x, nextSample.y);
if (primaryMocapHandState === 'grab') {
if (activeSession?.inputId !== PUZZLE_MOCAP_DRAG_INPUT_ID) {
const sourceCell = resolveRuntimeInputGridCell(handPoint, board);
const sourcePiece = sourceCell
? pieceByCell.get(`${sourceCell.row}:${sourceCell.col}`) ?? null
: null;
if (!sourcePiece) {
runtimeDragInputControllerRef.current.cancel(
PUZZLE_MOCAP_DRAG_INPUT_ID,
);
return;
}
runtimeDragInputControllerRef.current.press({
targetId: sourcePiece.pieceId,
inputId: PUZZLE_MOCAP_DRAG_INPUT_ID,
deviceKind: 'mocap',
point: handPoint,
});
return;
}
runtimeDragInputControllerRef.current.move({
inputId: PUZZLE_MOCAP_DRAG_INPUT_ID,
point: handPoint,
forceDragging: true,
});
return;
}
if (activeSession?.inputId === PUZZLE_MOCAP_DRAG_INPUT_ID) {
runtimeDragInputControllerRef.current.release({
inputId: PUZZLE_MOCAP_DRAG_INPUT_ID,
point: handPoint,
forceDrop: activeSession.deviceKind === 'mocap',
});
}
}, [
board,
isInteractionLocked,
pieceByCell,
primaryMocapHandState,
primaryMocapHandX,
primaryMocapHandY,
runtimeStatus,
]);
useEffect(() => {
if (!board || runtimeStatus !== 'playing') {
if (mocapCursorIntervalRef.current !== null) {
window.clearInterval(mocapCursorIntervalRef.current);
mocapCursorIntervalRef.current = null;
}
return;
}
const tickMocapCursor = () => {
const targetSample = mocapCursorTargetSampleRef.current;
if (!targetSample) {
return;
}
const previousSample = mocapCursorPreviousSampleRef.current ?? targetSample;
const durationMs = Math.max(
PUZZLE_MOCAP_CURSOR_FRAME_MS,
targetSample.receivedAtMs - previousSample.receivedAtMs,
);
const progress = targetSample.receivedAtMs === previousSample.receivedAtMs
? 1
: Math.min(
1,
Math.max(0, (performance.now() - targetSample.receivedAtMs) / durationMs),
);
const nextCursor = {
x: previousSample.x + (targetSample.x - previousSample.x) * progress,
y: previousSample.y + (targetSample.y - previousSample.y) * progress,
state: targetSample.state,
};
const nextPoint = resolveBoardInputPointFromNormalized(
nextCursor.x,
nextCursor.y,
);
setMocapCursor(nextCursor);
const activeSession = runtimeDragInputControllerRef.current.getSession();
if (activeSession?.inputId === PUZZLE_MOCAP_DRAG_INPUT_ID) {
runtimeDragInputControllerRef.current.move({
inputId: PUZZLE_MOCAP_DRAG_INPUT_ID,
point: nextPoint,
forceDragging: true,
});
}
};
tickMocapCursor();
mocapCursorIntervalRef.current = window.setInterval(
tickMocapCursor,
PUZZLE_MOCAP_CURSOR_FRAME_MS,
);
return () => {
if (mocapCursorIntervalRef.current !== null) {
window.clearInterval(mocapCursorIntervalRef.current);
mocapCursorIntervalRef.current = null;
}
};
}, [board, runtimeStatus]);
if (!run || !currentLevel || !board) {
return (
<div
@@ -1810,21 +1607,6 @@ export function PuzzleRuntimeShell({
/>
</div>
) : null}
{mocapCursor ? (
<div
data-testid="puzzle-mocap-cursor"
className={`pointer-events-none absolute z-[70] flex h-8 w-8 -translate-x-1/2 -translate-y-1/2 items-center justify-center rounded-full border-2 ${
mocapCursor.state === 'grab'
? 'border-amber-200 bg-amber-400/90 text-amber-950'
: 'border-cyan-200 bg-cyan-300/90 text-cyan-950'
} shadow-[0_10px_24px_rgba(15,23,42,0.25)]`}
style={{left: `${mocapCursor.x * 100}%`, top: `${mocapCursor.y * 100}%`}}
>
<span className="text-[10px] font-black leading-none">
{mocapCursor.state === 'grab' ? '抓' : '手'}
</span>
</div>
) : null}
{mergeFlash ? (
<div
key={mergeFlash.key}
@@ -1852,45 +1634,6 @@ export function PuzzleRuntimeShell({
已选择
</div>
) : null}
{shouldShowMocapDebugPanel ? (
<section
data-testid="puzzle-mocap-debug"
className="w-[min(92vw,34rem)] overflow-hidden rounded-[0.9rem] border border-white/20 bg-slate-950/70 font-mono text-[10px] leading-4 text-white shadow-[0_12px_32px_rgba(15,23,42,0.25)] backdrop-blur"
>
<button
type="button"
aria-expanded={isMocapDebugExpanded}
aria-controls="puzzle-mocap-debug-content"
onClick={() => {
setIsMocapDebugExpanded((current) => !current);
}}
className="flex min-h-9 w-full items-center justify-between gap-3 px-3 py-2 text-left transition hover:bg-white/10"
>
<span className="min-w-0 truncate">
mocap: {mocapInput.status}
</span>
{isMocapDebugExpanded ? (
<ChevronDown className="h-3.5 w-3.5 shrink-0" />
) : (
<ChevronUp className="h-3.5 w-3.5 shrink-0" />
)}
</button>
{isMocapDebugExpanded ? (
<div
id="puzzle-mocap-debug-content"
className="border-t border-white/10 px-3 pb-2 pt-2"
>
<div>动作: {mocapActionsLabel}</div>
<div>手势: {mocapHandLabel}</div>
<div>解析: {mocapParseWarningLabel}</div>
<div className="max-h-20 overflow-auto break-all text-white/75">
原始: {mocapRawPacketLabel}
</div>
{mocapInput.error ? <div>错误: {mocapInput.error}</div> : null}
</div>
) : null}
</section>
) : null}
{canShowNextAction ? (
<button
type="button"