Switch to VectorEngine gpt-image-2 and edits
Replace uses of the legacy `gpt-image-2-all` model with `gpt-image-2` and standardize image workflows: no-reference generation uses POST /v1/images/generations, any-reference flows use POST /v1/images/edits with multipart `image` parts. Update SKILLs, generation scripts, decision logs, and docs to reflect the contract change and edits-vs-generations guidance. Apply corresponding changes across backend (api-server match3d/puzzle modules, openai image adapter, mappers, telemetry, spacetime client/module), frontend components and services (Match3D, Puzzle, CreativeImageInputPanel, runtime shells), and add new spritesheet/parser files and tests. Also add media/logo.png. These changes align repository code and documentation with the VectorEngine image API contract and update generation/upload handling (green-screen -> alpha processing, spritesheet handling, and related tests).
This commit is contained in:
471
src/services/puzzle-runtime/puzzleUiSpritesheetParser.ts
Normal file
471
src/services/puzzle-runtime/puzzleUiSpritesheetParser.ts
Normal file
@@ -0,0 +1,471 @@
|
||||
import type { CSSProperties } from 'react';
|
||||
|
||||
import { readAssetBytes } from '../assetReadUrlService';
|
||||
|
||||
export type PuzzleUiSpriteKind =
|
||||
| 'back'
|
||||
| 'settings'
|
||||
| 'next'
|
||||
| 'hint'
|
||||
| 'reference'
|
||||
| 'freezeTime';
|
||||
|
||||
export type PuzzleUiSpriteRegion = {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
|
||||
export type PuzzleUiSpritesheetLayout = {
|
||||
width: number;
|
||||
height: number;
|
||||
regions: Partial<Record<PuzzleUiSpriteKind, PuzzleUiSpriteRegion>>;
|
||||
hitRegions?: Partial<Record<PuzzleUiSpriteKind, PuzzleUiSpriteRegion>>;
|
||||
};
|
||||
|
||||
export type DetectPuzzleUiSpritesheetLayoutInput = {
|
||||
alpha: ArrayLike<number>;
|
||||
width: number;
|
||||
height: number;
|
||||
minArea?: number;
|
||||
alphaThreshold?: number;
|
||||
hitAlphaThreshold?: number;
|
||||
};
|
||||
|
||||
export type BuildPuzzleUiSpriteBackgroundStyleInput = {
|
||||
src: string;
|
||||
kind: PuzzleUiSpriteKind;
|
||||
layout: PuzzleUiSpritesheetLayout | null;
|
||||
};
|
||||
|
||||
export type BuildPuzzleUiSpriteHitZoneStyleInput = {
|
||||
kind: PuzzleUiSpriteKind;
|
||||
layout: PuzzleUiSpritesheetLayout | null;
|
||||
};
|
||||
|
||||
export type LoadPuzzleUiSpritesheetLayoutOptions = {
|
||||
signal?: AbortSignal;
|
||||
expireSeconds?: number;
|
||||
minArea?: number;
|
||||
alphaThreshold?: number;
|
||||
hitAlphaThreshold?: number;
|
||||
};
|
||||
|
||||
type PuzzleUiDetectedComponent = PuzzleUiSpriteRegion & {
|
||||
area: number;
|
||||
hitRegion?: PuzzleUiSpriteRegion;
|
||||
};
|
||||
|
||||
const PUZZLE_UI_SPRITE_ORDER = [
|
||||
'back',
|
||||
'settings',
|
||||
'next',
|
||||
'hint',
|
||||
'reference',
|
||||
'freezeTime',
|
||||
] as const satisfies readonly PuzzleUiSpriteKind[];
|
||||
|
||||
const PUZZLE_UI_FIXED_GRID_INDEX: Record<PuzzleUiSpriteKind, number> = {
|
||||
back: 0,
|
||||
settings: 1,
|
||||
next: 2,
|
||||
hint: 3,
|
||||
reference: 4,
|
||||
freezeTime: 5,
|
||||
};
|
||||
|
||||
/**
|
||||
* 中文注释：AI 生成的拼图 UI spritesheet 不稳定落在固定六宫格内，
|
||||
* 因此这里以 alpha 连通域检测真实按钮矩形，再按原图位置映射到按钮语义。
|
||||
*/
|
||||
export function detectPuzzleUiSpritesheetLayout({
|
||||
alpha,
|
||||
width,
|
||||
height,
|
||||
minArea = 1,
|
||||
alphaThreshold = 0,
|
||||
hitAlphaThreshold = Math.max(192, alphaThreshold),
|
||||
}: DetectPuzzleUiSpritesheetLayoutInput): PuzzleUiSpritesheetLayout | null {
|
||||
const pixelCount = width * height;
|
||||
if (width <= 0 || height <= 0 || alpha.length < pixelCount) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const visited = new Uint8Array(pixelCount);
|
||||
const components: PuzzleUiDetectedComponent[] = [];
|
||||
|
||||
for (let start = 0; start < pixelCount; start += 1) {
|
||||
const alphaValue = alpha[start];
|
||||
if (
|
||||
visited[start] ||
|
||||
alphaValue === undefined ||
|
||||
alphaValue <= alphaThreshold
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const component = floodFillPuzzleUiSpriteComponent({
|
||||
alpha,
|
||||
visited,
|
||||
width,
|
||||
height,
|
||||
start,
|
||||
alphaThreshold,
|
||||
hitAlphaThreshold,
|
||||
});
|
||||
if (component.area >= minArea) {
|
||||
components.push(component);
|
||||
}
|
||||
}
|
||||
|
||||
if (components.length < PUZZLE_UI_SPRITE_ORDER.length) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sortedComponents = sortPuzzleUiSpriteComponentsByOriginalPosition(
|
||||
components,
|
||||
).slice(0, PUZZLE_UI_SPRITE_ORDER.length);
|
||||
const regions: Partial<Record<PuzzleUiSpriteKind, PuzzleUiSpriteRegion>> = {};
|
||||
const hitRegions: Partial<Record<PuzzleUiSpriteKind, PuzzleUiSpriteRegion>> =
|
||||
{};
|
||||
sortedComponents.forEach((component, index) => {
|
||||
const kind = PUZZLE_UI_SPRITE_ORDER[index];
|
||||
if (!kind) {
|
||||
return;
|
||||
}
|
||||
const region = {
|
||||
x: component.x,
|
||||
y: component.y,
|
||||
width: component.width,
|
||||
height: component.height,
|
||||
};
|
||||
regions[kind] = region;
|
||||
if (component.hitRegion) {
|
||||
hitRegions[kind] = component.hitRegion;
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
width,
|
||||
height,
|
||||
regions,
|
||||
hitRegions,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildPuzzleUiSpriteBackgroundStyle({
|
||||
src,
|
||||
kind,
|
||||
layout,
|
||||
}: BuildPuzzleUiSpriteBackgroundStyleInput): CSSProperties {
|
||||
const region = layout?.regions[kind];
|
||||
if (!layout || !region) {
|
||||
const index = PUZZLE_UI_FIXED_GRID_INDEX[kind];
|
||||
return {
|
||||
backgroundImage: `url("${src}")`,
|
||||
backgroundSize: '200% 300%',
|
||||
backgroundPosition: `${(index % 2) * 100}% ${
|
||||
Math.floor(index / 2) * 50
|
||||
}%`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
backgroundImage: `url("${src}")`,
|
||||
backgroundSize: `${(layout.width / region.width) * 100}% ${
|
||||
(layout.height / region.height) * 100
|
||||
}%`,
|
||||
backgroundPosition: `${resolvePuzzleUiSpriteBackgroundAxisPosition(
|
||||
region.x,
|
||||
layout.width,
|
||||
region.width,
|
||||
)}% ${resolvePuzzleUiSpriteBackgroundAxisPosition(
|
||||
region.y,
|
||||
layout.height,
|
||||
region.height,
|
||||
)}%`,
|
||||
aspectRatio: `${region.width} / ${region.height}`,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildPuzzleUiSpriteHitZoneStyle({
|
||||
kind,
|
||||
layout,
|
||||
}: BuildPuzzleUiSpriteHitZoneStyleInput): CSSProperties {
|
||||
const region = layout?.regions[kind];
|
||||
const hitRegion = layout?.hitRegions?.[kind];
|
||||
if (!region || !hitRegion) {
|
||||
return {
|
||||
inset: 0,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
left: `${resolvePuzzleUiSpriteHitZoneOffset(
|
||||
hitRegion.x,
|
||||
region.x,
|
||||
region.width,
|
||||
)}%`,
|
||||
top: `${resolvePuzzleUiSpriteHitZoneOffset(
|
||||
hitRegion.y,
|
||||
region.y,
|
||||
region.height,
|
||||
)}%`,
|
||||
width: `${resolvePuzzleUiSpriteHitZoneSize(
|
||||
hitRegion.width,
|
||||
region.width,
|
||||
)}%`,
|
||||
height: `${resolvePuzzleUiSpriteHitZoneSize(
|
||||
hitRegion.height,
|
||||
region.height,
|
||||
)}%`,
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadPuzzleUiSpritesheetLayout(
|
||||
source: string,
|
||||
options: LoadPuzzleUiSpritesheetLayoutOptions = {},
|
||||
) {
|
||||
const response = await readAssetBytes(source, {
|
||||
signal: options.signal,
|
||||
expireSeconds: options.expireSeconds,
|
||||
});
|
||||
const blob = await response.blob();
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
|
||||
try {
|
||||
const image = await loadPuzzleUiSpritesheetImage(objectUrl);
|
||||
const width = image.naturalWidth || image.width;
|
||||
const height = image.naturalHeight || image.height;
|
||||
if (width <= 0 || height <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
const context = canvas.getContext('2d');
|
||||
if (!context) {
|
||||
return null;
|
||||
}
|
||||
context.drawImage(image, 0, 0, width, height);
|
||||
const imageData = context.getImageData(0, 0, width, height);
|
||||
const alpha = new Uint8ClampedArray(width * height);
|
||||
for (let index = 0; index < alpha.length; index += 1) {
|
||||
alpha[index] = imageData.data[index * 4 + 3] ?? 0;
|
||||
}
|
||||
|
||||
return detectPuzzleUiSpritesheetLayout({
|
||||
alpha,
|
||||
width,
|
||||
height,
|
||||
minArea:
|
||||
options.minArea ?? Math.max(16, Math.floor(width * height * 0.0002)),
|
||||
alphaThreshold: options.alphaThreshold ?? 16,
|
||||
hitAlphaThreshold: options.hitAlphaThreshold ?? 192,
|
||||
});
|
||||
} finally {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
}
|
||||
}
|
||||
|
||||
function sortPuzzleUiSpriteComponentsByOriginalPosition(
|
||||
components: PuzzleUiDetectedComponent[],
|
||||
) {
|
||||
const averageHeight =
|
||||
components.reduce((total, component) => total + component.height, 0) /
|
||||
components.length;
|
||||
const rowTolerance = Math.max(2, averageHeight * 0.65);
|
||||
const rows: PuzzleUiDetectedComponent[][] = [];
|
||||
|
||||
for (const component of components
|
||||
.slice()
|
||||
.sort((left, right) => left.y - right.y)) {
|
||||
const centerY = component.y + component.height / 2;
|
||||
const row = rows.find((items) => {
|
||||
const rowCenter =
|
||||
items.reduce((total, item) => total + item.y + item.height / 2, 0) /
|
||||
items.length;
|
||||
return Math.abs(rowCenter - centerY) <= rowTolerance;
|
||||
});
|
||||
|
||||
if (row) {
|
||||
row.push(component);
|
||||
} else {
|
||||
rows.push([component]);
|
||||
}
|
||||
}
|
||||
|
||||
return rows.flatMap((row) => row.sort((left, right) => left.x - right.x));
|
||||
}
|
||||
|
||||
function resolvePuzzleUiSpriteBackgroundAxisPosition(
|
||||
offset: number,
|
||||
imageSize: number,
|
||||
regionSize: number,
|
||||
) {
|
||||
const movableSize = imageSize - regionSize;
|
||||
if (movableSize <= 0) {
|
||||
return 0;
|
||||
}
|
||||
return (offset / movableSize) * 100;
|
||||
}
|
||||
|
||||
function resolvePuzzleUiSpriteHitZoneOffset(
|
||||
hitOffset: number,
|
||||
regionOffset: number,
|
||||
regionSize: number,
|
||||
) {
|
||||
if (regionSize <= 0) {
|
||||
return 0;
|
||||
}
|
||||
return clampPuzzleUiSpritePercent(
|
||||
((hitOffset - regionOffset) / regionSize) * 100,
|
||||
);
|
||||
}
|
||||
|
||||
function resolvePuzzleUiSpriteHitZoneSize(
|
||||
hitSize: number,
|
||||
regionSize: number,
|
||||
) {
|
||||
if (regionSize <= 0) {
|
||||
return 100;
|
||||
}
|
||||
return clampPuzzleUiSpritePercent((hitSize / regionSize) * 100);
|
||||
}
|
||||
|
||||
function clampPuzzleUiSpritePercent(value: number) {
|
||||
return Math.min(100, Math.max(0, value));
|
||||
}
|
||||
|
||||
function loadPuzzleUiSpritesheetImage(src: string) {
|
||||
return new Promise<HTMLImageElement>((resolve, reject) => {
|
||||
const image = new Image();
|
||||
image.onload = () => resolve(image);
|
||||
image.onerror = () => reject(new Error('拼图 UI spritesheet 图片解码失败'));
|
||||
image.src = src;
|
||||
});
|
||||
}
|
||||
|
||||
function floodFillPuzzleUiSpriteComponent({
|
||||
alpha,
|
||||
visited,
|
||||
width,
|
||||
height,
|
||||
start,
|
||||
alphaThreshold,
|
||||
hitAlphaThreshold,
|
||||
}: {
|
||||
alpha: ArrayLike<number>;
|
||||
visited: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
start: number;
|
||||
alphaThreshold: number;
|
||||
hitAlphaThreshold: number;
|
||||
}): PuzzleUiDetectedComponent {
|
||||
const stack = [start];
|
||||
visited[start] = 1;
|
||||
|
||||
let minX = start % width;
|
||||
let maxX = minX;
|
||||
let minY = Math.floor(start / width);
|
||||
let maxY = minY;
|
||||
let area = 0;
|
||||
let hitMinX = Number.POSITIVE_INFINITY;
|
||||
let hitMaxX = Number.NEGATIVE_INFINITY;
|
||||
let hitMinY = Number.POSITIVE_INFINITY;
|
||||
let hitMaxY = Number.NEGATIVE_INFINITY;
|
||||
let hitArea = 0;
|
||||
|
||||
while (stack.length > 0) {
|
||||
const index = stack.pop()!;
|
||||
const x = index % width;
|
||||
const y = Math.floor(index / width);
|
||||
const alphaValue = alpha[index] ?? 0;
|
||||
area += 1;
|
||||
minX = Math.min(minX, x);
|
||||
maxX = Math.max(maxX, x);
|
||||
minY = Math.min(minY, y);
|
||||
maxY = Math.max(maxY, y);
|
||||
if (alphaValue > hitAlphaThreshold) {
|
||||
hitArea += 1;
|
||||
hitMinX = Math.min(hitMinX, x);
|
||||
hitMaxX = Math.max(hitMaxX, x);
|
||||
hitMinY = Math.min(hitMinY, y);
|
||||
hitMaxY = Math.max(hitMaxY, y);
|
||||
}
|
||||
|
||||
visitPuzzleUiSpriteNeighbor(
|
||||
index - 1,
|
||||
x > 0,
|
||||
alpha,
|
||||
visited,
|
||||
stack,
|
||||
alphaThreshold,
|
||||
);
|
||||
visitPuzzleUiSpriteNeighbor(
|
||||
index + 1,
|
||||
x + 1 < width,
|
||||
alpha,
|
||||
visited,
|
||||
stack,
|
||||
alphaThreshold,
|
||||
);
|
||||
visitPuzzleUiSpriteNeighbor(
|
||||
index - width,
|
||||
y > 0,
|
||||
alpha,
|
||||
visited,
|
||||
stack,
|
||||
alphaThreshold,
|
||||
);
|
||||
visitPuzzleUiSpriteNeighbor(
|
||||
index + width,
|
||||
y + 1 < height,
|
||||
alpha,
|
||||
visited,
|
||||
stack,
|
||||
alphaThreshold,
|
||||
);
|
||||
}
|
||||
|
||||
const component: PuzzleUiDetectedComponent = {
|
||||
area,
|
||||
x: minX,
|
||||
y: minY,
|
||||
width: maxX - minX + 1,
|
||||
height: maxY - minY + 1,
|
||||
};
|
||||
if (hitArea > 0) {
|
||||
component.hitRegion = {
|
||||
x: hitMinX,
|
||||
y: hitMinY,
|
||||
width: hitMaxX - hitMinX + 1,
|
||||
height: hitMaxY - hitMinY + 1,
|
||||
};
|
||||
}
|
||||
return component;
|
||||
}
|
||||
|
||||
function visitPuzzleUiSpriteNeighbor(
|
||||
index: number,
|
||||
inBounds: boolean,
|
||||
alpha: ArrayLike<number>,
|
||||
visited: Uint8Array,
|
||||
stack: number[],
|
||||
alphaThreshold: number,
|
||||
) {
|
||||
const alphaValue = alpha[index];
|
||||
if (
|
||||
!inBounds ||
|
||||
visited[index] ||
|
||||
alphaValue === undefined ||
|
||||
alphaValue <= alphaThreshold
|
||||
) {
|
||||
return;
|
||||
}
|
||||
visited[index] = 1;
|
||||
stack.push(index);
|
||||
}
|
||||
Reference in New Issue
Block a user