Replace uses of the legacy `gpt-image-2-all` model with `gpt-image-2` and standardize image workflows: no-reference generation uses POST /v1/images/generations, any-reference flows use POST /v1/images/edits with multipart `image` parts. Update SKILLs, generation scripts, decision logs, and docs to reflect the contract change and edits-vs-generations guidance. Apply corresponding changes across backend (api-server match3d/puzzle modules, openai image adapter, mappers, telemetry, spacetime client/module), frontend components and services (Match3D, Puzzle, CreativeImageInputPanel, runtime shells), and add new spritesheet/parser files and tests. Also add media/logo.png. These changes align repository code and documentation with the VectorEngine image API contract and update generation/upload handling (green-screen -> alpha processing, spritesheet handling, and related tests).
472 lines
11 KiB
TypeScript
472 lines
11 KiB
TypeScript
import type { CSSProperties } from 'react';
|
||
|
||
import { readAssetBytes } from '../assetReadUrlService';
|
||
|
||
export type PuzzleUiSpriteKind =
|
||
| 'back'
|
||
| 'settings'
|
||
| 'next'
|
||
| 'hint'
|
||
| 'reference'
|
||
| 'freezeTime';
|
||
|
||
export type PuzzleUiSpriteRegion = {
|
||
x: number;
|
||
y: number;
|
||
width: number;
|
||
height: number;
|
||
};
|
||
|
||
export type PuzzleUiSpritesheetLayout = {
|
||
width: number;
|
||
height: number;
|
||
regions: Partial<Record<PuzzleUiSpriteKind, PuzzleUiSpriteRegion>>;
|
||
hitRegions?: Partial<Record<PuzzleUiSpriteKind, PuzzleUiSpriteRegion>>;
|
||
};
|
||
|
||
export type DetectPuzzleUiSpritesheetLayoutInput = {
|
||
alpha: ArrayLike<number>;
|
||
width: number;
|
||
height: number;
|
||
minArea?: number;
|
||
alphaThreshold?: number;
|
||
hitAlphaThreshold?: number;
|
||
};
|
||
|
||
export type BuildPuzzleUiSpriteBackgroundStyleInput = {
|
||
src: string;
|
||
kind: PuzzleUiSpriteKind;
|
||
layout: PuzzleUiSpritesheetLayout | null;
|
||
};
|
||
|
||
export type BuildPuzzleUiSpriteHitZoneStyleInput = {
|
||
kind: PuzzleUiSpriteKind;
|
||
layout: PuzzleUiSpritesheetLayout | null;
|
||
};
|
||
|
||
export type LoadPuzzleUiSpritesheetLayoutOptions = {
|
||
signal?: AbortSignal;
|
||
expireSeconds?: number;
|
||
minArea?: number;
|
||
alphaThreshold?: number;
|
||
hitAlphaThreshold?: number;
|
||
};
|
||
|
||
type PuzzleUiDetectedComponent = PuzzleUiSpriteRegion & {
|
||
area: number;
|
||
hitRegion?: PuzzleUiSpriteRegion;
|
||
};
|
||
|
||
const PUZZLE_UI_SPRITE_ORDER = [
|
||
'back',
|
||
'settings',
|
||
'next',
|
||
'hint',
|
||
'reference',
|
||
'freezeTime',
|
||
] as const satisfies readonly PuzzleUiSpriteKind[];
|
||
|
||
const PUZZLE_UI_FIXED_GRID_INDEX: Record<PuzzleUiSpriteKind, number> = {
|
||
back: 0,
|
||
settings: 1,
|
||
next: 2,
|
||
hint: 3,
|
||
reference: 4,
|
||
freezeTime: 5,
|
||
};
|
||
|
||
/**
|
||
* 中文注释:AI 生成的拼图 UI spritesheet 不稳定落在固定六宫格内,
|
||
* 因此这里以 alpha 连通域检测真实按钮矩形,再按原图位置映射到按钮语义。
|
||
*/
|
||
export function detectPuzzleUiSpritesheetLayout({
|
||
alpha,
|
||
width,
|
||
height,
|
||
minArea = 1,
|
||
alphaThreshold = 0,
|
||
hitAlphaThreshold = Math.max(192, alphaThreshold),
|
||
}: DetectPuzzleUiSpritesheetLayoutInput): PuzzleUiSpritesheetLayout | null {
|
||
const pixelCount = width * height;
|
||
if (width <= 0 || height <= 0 || alpha.length < pixelCount) {
|
||
return null;
|
||
}
|
||
|
||
const visited = new Uint8Array(pixelCount);
|
||
const components: PuzzleUiDetectedComponent[] = [];
|
||
|
||
for (let start = 0; start < pixelCount; start += 1) {
|
||
const alphaValue = alpha[start];
|
||
if (
|
||
visited[start] ||
|
||
alphaValue === undefined ||
|
||
alphaValue <= alphaThreshold
|
||
) {
|
||
continue;
|
||
}
|
||
|
||
const component = floodFillPuzzleUiSpriteComponent({
|
||
alpha,
|
||
visited,
|
||
width,
|
||
height,
|
||
start,
|
||
alphaThreshold,
|
||
hitAlphaThreshold,
|
||
});
|
||
if (component.area >= minArea) {
|
||
components.push(component);
|
||
}
|
||
}
|
||
|
||
if (components.length < PUZZLE_UI_SPRITE_ORDER.length) {
|
||
return null;
|
||
}
|
||
|
||
const sortedComponents = sortPuzzleUiSpriteComponentsByOriginalPosition(
|
||
components,
|
||
).slice(0, PUZZLE_UI_SPRITE_ORDER.length);
|
||
const regions: Partial<Record<PuzzleUiSpriteKind, PuzzleUiSpriteRegion>> = {};
|
||
const hitRegions: Partial<Record<PuzzleUiSpriteKind, PuzzleUiSpriteRegion>> =
|
||
{};
|
||
sortedComponents.forEach((component, index) => {
|
||
const kind = PUZZLE_UI_SPRITE_ORDER[index];
|
||
if (!kind) {
|
||
return;
|
||
}
|
||
const region = {
|
||
x: component.x,
|
||
y: component.y,
|
||
width: component.width,
|
||
height: component.height,
|
||
};
|
||
regions[kind] = region;
|
||
if (component.hitRegion) {
|
||
hitRegions[kind] = component.hitRegion;
|
||
}
|
||
});
|
||
|
||
return {
|
||
width,
|
||
height,
|
||
regions,
|
||
hitRegions,
|
||
};
|
||
}
|
||
|
||
export function buildPuzzleUiSpriteBackgroundStyle({
|
||
src,
|
||
kind,
|
||
layout,
|
||
}: BuildPuzzleUiSpriteBackgroundStyleInput): CSSProperties {
|
||
const region = layout?.regions[kind];
|
||
if (!layout || !region) {
|
||
const index = PUZZLE_UI_FIXED_GRID_INDEX[kind];
|
||
return {
|
||
backgroundImage: `url("${src}")`,
|
||
backgroundSize: '200% 300%',
|
||
backgroundPosition: `${(index % 2) * 100}% ${
|
||
Math.floor(index / 2) * 50
|
||
}%`,
|
||
};
|
||
}
|
||
|
||
return {
|
||
backgroundImage: `url("${src}")`,
|
||
backgroundSize: `${(layout.width / region.width) * 100}% ${
|
||
(layout.height / region.height) * 100
|
||
}%`,
|
||
backgroundPosition: `${resolvePuzzleUiSpriteBackgroundAxisPosition(
|
||
region.x,
|
||
layout.width,
|
||
region.width,
|
||
)}% ${resolvePuzzleUiSpriteBackgroundAxisPosition(
|
||
region.y,
|
||
layout.height,
|
||
region.height,
|
||
)}%`,
|
||
aspectRatio: `${region.width} / ${region.height}`,
|
||
};
|
||
}
|
||
|
||
export function buildPuzzleUiSpriteHitZoneStyle({
|
||
kind,
|
||
layout,
|
||
}: BuildPuzzleUiSpriteHitZoneStyleInput): CSSProperties {
|
||
const region = layout?.regions[kind];
|
||
const hitRegion = layout?.hitRegions?.[kind];
|
||
if (!region || !hitRegion) {
|
||
return {
|
||
inset: 0,
|
||
};
|
||
}
|
||
|
||
return {
|
||
left: `${resolvePuzzleUiSpriteHitZoneOffset(
|
||
hitRegion.x,
|
||
region.x,
|
||
region.width,
|
||
)}%`,
|
||
top: `${resolvePuzzleUiSpriteHitZoneOffset(
|
||
hitRegion.y,
|
||
region.y,
|
||
region.height,
|
||
)}%`,
|
||
width: `${resolvePuzzleUiSpriteHitZoneSize(
|
||
hitRegion.width,
|
||
region.width,
|
||
)}%`,
|
||
height: `${resolvePuzzleUiSpriteHitZoneSize(
|
||
hitRegion.height,
|
||
region.height,
|
||
)}%`,
|
||
};
|
||
}
|
||
|
||
export async function loadPuzzleUiSpritesheetLayout(
|
||
source: string,
|
||
options: LoadPuzzleUiSpritesheetLayoutOptions = {},
|
||
) {
|
||
const response = await readAssetBytes(source, {
|
||
signal: options.signal,
|
||
expireSeconds: options.expireSeconds,
|
||
});
|
||
const blob = await response.blob();
|
||
const objectUrl = URL.createObjectURL(blob);
|
||
|
||
try {
|
||
const image = await loadPuzzleUiSpritesheetImage(objectUrl);
|
||
const width = image.naturalWidth || image.width;
|
||
const height = image.naturalHeight || image.height;
|
||
if (width <= 0 || height <= 0) {
|
||
return null;
|
||
}
|
||
|
||
const canvas = document.createElement('canvas');
|
||
canvas.width = width;
|
||
canvas.height = height;
|
||
const context = canvas.getContext('2d');
|
||
if (!context) {
|
||
return null;
|
||
}
|
||
context.drawImage(image, 0, 0, width, height);
|
||
const imageData = context.getImageData(0, 0, width, height);
|
||
const alpha = new Uint8ClampedArray(width * height);
|
||
for (let index = 0; index < alpha.length; index += 1) {
|
||
alpha[index] = imageData.data[index * 4 + 3] ?? 0;
|
||
}
|
||
|
||
return detectPuzzleUiSpritesheetLayout({
|
||
alpha,
|
||
width,
|
||
height,
|
||
minArea:
|
||
options.minArea ?? Math.max(16, Math.floor(width * height * 0.0002)),
|
||
alphaThreshold: options.alphaThreshold ?? 16,
|
||
hitAlphaThreshold: options.hitAlphaThreshold ?? 192,
|
||
});
|
||
} finally {
|
||
URL.revokeObjectURL(objectUrl);
|
||
}
|
||
}
|
||
|
||
function sortPuzzleUiSpriteComponentsByOriginalPosition(
|
||
components: PuzzleUiDetectedComponent[],
|
||
) {
|
||
const averageHeight =
|
||
components.reduce((total, component) => total + component.height, 0) /
|
||
components.length;
|
||
const rowTolerance = Math.max(2, averageHeight * 0.65);
|
||
const rows: PuzzleUiDetectedComponent[][] = [];
|
||
|
||
for (const component of components
|
||
.slice()
|
||
.sort((left, right) => left.y - right.y)) {
|
||
const centerY = component.y + component.height / 2;
|
||
const row = rows.find((items) => {
|
||
const rowCenter =
|
||
items.reduce((total, item) => total + item.y + item.height / 2, 0) /
|
||
items.length;
|
||
return Math.abs(rowCenter - centerY) <= rowTolerance;
|
||
});
|
||
|
||
if (row) {
|
||
row.push(component);
|
||
} else {
|
||
rows.push([component]);
|
||
}
|
||
}
|
||
|
||
return rows.flatMap((row) => row.sort((left, right) => left.x - right.x));
|
||
}
|
||
|
||
function resolvePuzzleUiSpriteBackgroundAxisPosition(
|
||
offset: number,
|
||
imageSize: number,
|
||
regionSize: number,
|
||
) {
|
||
const movableSize = imageSize - regionSize;
|
||
if (movableSize <= 0) {
|
||
return 0;
|
||
}
|
||
return (offset / movableSize) * 100;
|
||
}
|
||
|
||
function resolvePuzzleUiSpriteHitZoneOffset(
|
||
hitOffset: number,
|
||
regionOffset: number,
|
||
regionSize: number,
|
||
) {
|
||
if (regionSize <= 0) {
|
||
return 0;
|
||
}
|
||
return clampPuzzleUiSpritePercent(
|
||
((hitOffset - regionOffset) / regionSize) * 100,
|
||
);
|
||
}
|
||
|
||
function resolvePuzzleUiSpriteHitZoneSize(
|
||
hitSize: number,
|
||
regionSize: number,
|
||
) {
|
||
if (regionSize <= 0) {
|
||
return 100;
|
||
}
|
||
return clampPuzzleUiSpritePercent((hitSize / regionSize) * 100);
|
||
}
|
||
|
||
function clampPuzzleUiSpritePercent(value: number) {
|
||
return Math.min(100, Math.max(0, value));
|
||
}
|
||
|
||
function loadPuzzleUiSpritesheetImage(src: string) {
|
||
return new Promise<HTMLImageElement>((resolve, reject) => {
|
||
const image = new Image();
|
||
image.onload = () => resolve(image);
|
||
image.onerror = () => reject(new Error('拼图 UI spritesheet 图片解码失败'));
|
||
image.src = src;
|
||
});
|
||
}
|
||
|
||
function floodFillPuzzleUiSpriteComponent({
|
||
alpha,
|
||
visited,
|
||
width,
|
||
height,
|
||
start,
|
||
alphaThreshold,
|
||
hitAlphaThreshold,
|
||
}: {
|
||
alpha: ArrayLike<number>;
|
||
visited: Uint8Array;
|
||
width: number;
|
||
height: number;
|
||
start: number;
|
||
alphaThreshold: number;
|
||
hitAlphaThreshold: number;
|
||
}): PuzzleUiDetectedComponent {
|
||
const stack = [start];
|
||
visited[start] = 1;
|
||
|
||
let minX = start % width;
|
||
let maxX = minX;
|
||
let minY = Math.floor(start / width);
|
||
let maxY = minY;
|
||
let area = 0;
|
||
let hitMinX = Number.POSITIVE_INFINITY;
|
||
let hitMaxX = Number.NEGATIVE_INFINITY;
|
||
let hitMinY = Number.POSITIVE_INFINITY;
|
||
let hitMaxY = Number.NEGATIVE_INFINITY;
|
||
let hitArea = 0;
|
||
|
||
while (stack.length > 0) {
|
||
const index = stack.pop()!;
|
||
const x = index % width;
|
||
const y = Math.floor(index / width);
|
||
const alphaValue = alpha[index] ?? 0;
|
||
area += 1;
|
||
minX = Math.min(minX, x);
|
||
maxX = Math.max(maxX, x);
|
||
minY = Math.min(minY, y);
|
||
maxY = Math.max(maxY, y);
|
||
if (alphaValue > hitAlphaThreshold) {
|
||
hitArea += 1;
|
||
hitMinX = Math.min(hitMinX, x);
|
||
hitMaxX = Math.max(hitMaxX, x);
|
||
hitMinY = Math.min(hitMinY, y);
|
||
hitMaxY = Math.max(hitMaxY, y);
|
||
}
|
||
|
||
visitPuzzleUiSpriteNeighbor(
|
||
index - 1,
|
||
x > 0,
|
||
alpha,
|
||
visited,
|
||
stack,
|
||
alphaThreshold,
|
||
);
|
||
visitPuzzleUiSpriteNeighbor(
|
||
index + 1,
|
||
x + 1 < width,
|
||
alpha,
|
||
visited,
|
||
stack,
|
||
alphaThreshold,
|
||
);
|
||
visitPuzzleUiSpriteNeighbor(
|
||
index - width,
|
||
y > 0,
|
||
alpha,
|
||
visited,
|
||
stack,
|
||
alphaThreshold,
|
||
);
|
||
visitPuzzleUiSpriteNeighbor(
|
||
index + width,
|
||
y + 1 < height,
|
||
alpha,
|
||
visited,
|
||
stack,
|
||
alphaThreshold,
|
||
);
|
||
}
|
||
|
||
const component: PuzzleUiDetectedComponent = {
|
||
area,
|
||
x: minX,
|
||
y: minY,
|
||
width: maxX - minX + 1,
|
||
height: maxY - minY + 1,
|
||
};
|
||
if (hitArea > 0) {
|
||
component.hitRegion = {
|
||
x: hitMinX,
|
||
y: hitMinY,
|
||
width: hitMaxX - hitMinX + 1,
|
||
height: hitMaxY - hitMinY + 1,
|
||
};
|
||
}
|
||
return component;
|
||
}
|
||
|
||
function visitPuzzleUiSpriteNeighbor(
|
||
index: number,
|
||
inBounds: boolean,
|
||
alpha: ArrayLike<number>,
|
||
visited: Uint8Array,
|
||
stack: number[],
|
||
alphaThreshold: number,
|
||
) {
|
||
const alphaValue = alpha[index];
|
||
if (
|
||
!inBounds ||
|
||
visited[index] ||
|
||
alphaValue === undefined ||
|
||
alphaValue <= alphaThreshold
|
||
) {
|
||
return;
|
||
}
|
||
visited[index] = 1;
|
||
stack.push(index);
|
||
}
|