097170836d
Introduce a new UI design asset extraction flow: add POST /api/editor/ui-designs/assets/extractions handler that always uses gpt-image-2 with the fixed prompt "提取画面中的所有独立并整理成spritesheet"; parse Data URL references, call OpenAI image edits, return a spritesheet and sliced icon assets. Increase request body limit for image Data URLs and wire the route with bearer auth. Add platform-image slicing: slice_generated_icon_spritesheet_all_by_connected_components (with auto-naming "素材 1..N", foreground estimation and heuristics) and export it. Add Rust tests for the handler, prompt constant, and slicing behavior. Update frontend client and integration tests to call the new extractEditorUiDesignAssets API and verify spritesheet + icon layers are added to the canvas and persisted. Update multiple docs to describe the extraction UI, toolbar placement, and related audio/playback pitfalls. Minor related updates: toolbar button ordering and audio duration / signed read-url notes.
475 lines
13 KiB
TypeScript
475 lines
13 KiB
TypeScript
import type {
|
|
EditorAudioGenerationResult,
|
|
EditorCharacterAnimationGenerationResult,
|
|
EditorIconSpritesheetGenerationResult,
|
|
EditorIconSpritesheetIconResult,
|
|
EditorImageGenerationResult,
|
|
EditorVideoGenerationResult,
|
|
} from '../../services/image-editor/editorProjectClient';
|
|
import { resolveLayerResolutionSize } from './ImageCanvasEditorModel';
|
|
import type {
|
|
CanvasGenerationInputs,
|
|
CanvasLayer,
|
|
CanvasViewport,
|
|
GenerateDialogState,
|
|
} from './ImageCanvasEditorTypes';
|
|
import {
|
|
ICON_FRAME_DISPLAY_SIZE,
|
|
ICON_FRAME_ORIGINAL_SIZE,
|
|
} from './ImageCanvasGenerationModel';
|
|
|
|
type CanvasSize = { width: number; height: number };
|
|
|
|
type GeneratedResultLayerOptions = {
|
|
generated: EditorImageGenerationResult;
|
|
generatedIndex: number;
|
|
canvasSize: CanvasSize;
|
|
viewport: CanvasViewport;
|
|
sourceLayer?: CanvasLayer;
|
|
frame?: GenerateDialogState['placeholder'];
|
|
assetKind?: CanvasLayer['assetKind'];
|
|
title?: string;
|
|
generationInputs?: CanvasGenerationInputs;
|
|
};
|
|
|
|
type QuickEditResultLayerOptions = {
|
|
generated: EditorImageGenerationResult;
|
|
generatedIndex: number;
|
|
sourceLayer: CanvasLayer;
|
|
generationInputs: CanvasGenerationInputs;
|
|
mode?: 'quick-edit' | 'redraw';
|
|
frame?: GenerateDialogState['placeholder'];
|
|
};
|
|
|
|
type IconSpritesheetResultLayerOptions = {
|
|
generated: EditorIconSpritesheetGenerationResult;
|
|
iconResults: EditorIconSpritesheetIconResult[];
|
|
startIndex: number;
|
|
canvasSize: CanvasSize;
|
|
viewport: CanvasViewport;
|
|
generationInputs: CanvasGenerationInputs;
|
|
frame?: GenerateDialogState['placeholder'];
|
|
spritesheetTitle?: string;
|
|
};
|
|
|
|
type VideoResultLayerOptions = {
|
|
generated: EditorVideoGenerationResult;
|
|
generatedIndex: number;
|
|
title: string;
|
|
canvasSize: CanvasSize;
|
|
viewport: CanvasViewport;
|
|
generationInputs: CanvasGenerationInputs;
|
|
frame?: GenerateDialogState['placeholder'];
|
|
};
|
|
|
|
type AudioResultLayerOptions = {
|
|
generated: EditorAudioGenerationResult;
|
|
generatedIndex: number;
|
|
title: string;
|
|
canvasSize: CanvasSize;
|
|
viewport: CanvasViewport;
|
|
generationInputs: CanvasGenerationInputs;
|
|
frame?: GenerateDialogState['placeholder'];
|
|
};
|
|
|
|
type CharacterAnimationResultLayerOptions = {
|
|
generated: EditorCharacterAnimationGenerationResult;
|
|
generatedIndex: number;
|
|
title: string;
|
|
canvasSize: CanvasSize;
|
|
viewport: CanvasViewport;
|
|
generationInputs: CanvasGenerationInputs;
|
|
frame?: GenerateDialogState['placeholder'];
|
|
};
|
|
|
|
function getViewportWorldCenter({
|
|
canvasSize,
|
|
viewport,
|
|
}: {
|
|
canvasSize: CanvasSize;
|
|
viewport: CanvasViewport;
|
|
}) {
|
|
const safeScale = viewport.scale > 0 ? viewport.scale : 1;
|
|
return {
|
|
x: (canvasSize.width / 2 - viewport.x) / safeScale,
|
|
y: (canvasSize.height / 2 - viewport.y) / safeScale,
|
|
};
|
|
}
|
|
|
|
function applyGeneratedMetadata(
|
|
layer: CanvasLayer,
|
|
generated: EditorImageGenerationResult,
|
|
generationInputs: CanvasGenerationInputs | undefined,
|
|
): CanvasLayer {
|
|
return {
|
|
...layer,
|
|
prompt: generated.prompt,
|
|
actualPrompt: generated.actualPrompt ?? generated.prompt,
|
|
model: generated.model,
|
|
provider: generated.provider,
|
|
taskId: generated.taskId,
|
|
objectKey: generated.objectKey,
|
|
assetObjectId: generated.assetObjectId,
|
|
generationInputs,
|
|
};
|
|
}
|
|
|
|
export function createGeneratedResultLayer({
|
|
generated,
|
|
generatedIndex,
|
|
canvasSize,
|
|
viewport,
|
|
sourceLayer,
|
|
frame,
|
|
assetKind,
|
|
title,
|
|
generationInputs,
|
|
}: GeneratedResultLayerOptions): CanvasLayer {
|
|
const originalWidth = generated.width || 1024;
|
|
const originalHeight = generated.height || 1024;
|
|
const { width, height } = resolveLayerResolutionSize(
|
|
originalWidth,
|
|
originalHeight,
|
|
{ width: 1024, height: 1024 },
|
|
);
|
|
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
|
|
const frameX =
|
|
frame && frame.width > 0
|
|
? frame.x + frame.width / 2 - width / 2
|
|
: undefined;
|
|
const frameY =
|
|
frame && frame.height > 0
|
|
? frame.y + frame.height / 2 - height / 2
|
|
: undefined;
|
|
|
|
return applyGeneratedMetadata(
|
|
{
|
|
id: sourceLayer
|
|
? `layer-edit-${generatedIndex}`
|
|
: `layer-generated-${generatedIndex}`,
|
|
resourceId: sourceLayer
|
|
? `local-resource-edit-${generatedIndex}`
|
|
: `local-resource-generated-${generatedIndex}`,
|
|
title: sourceLayer
|
|
? `${sourceLayer.title} 修改结果`
|
|
: (title ?? `生成图片 ${generatedIndex}`),
|
|
src: generated.imageSrc,
|
|
x: sourceLayer
|
|
? sourceLayer.x + sourceLayer.width + 32
|
|
: (frameX ?? worldCenter.x - width / 2),
|
|
y: sourceLayer ? sourceLayer.y : (frameY ?? worldCenter.y - height / 2),
|
|
width,
|
|
height,
|
|
originalWidth,
|
|
originalHeight,
|
|
zIndex: generatedIndex + 10,
|
|
sourceType: generated.sourceType,
|
|
assetKind,
|
|
sourceResourceId: sourceLayer?.resourceId,
|
|
},
|
|
generated,
|
|
generationInputs,
|
|
);
|
|
}
|
|
|
|
export function createQuickEditResultLayer({
|
|
generated,
|
|
generatedIndex,
|
|
sourceLayer,
|
|
generationInputs,
|
|
mode = 'quick-edit',
|
|
frame,
|
|
}: QuickEditResultLayerOptions): CanvasLayer {
|
|
const originalWidth = generated.width || sourceLayer.originalWidth || 1024;
|
|
const originalHeight = generated.height || sourceLayer.originalHeight || 1024;
|
|
const { width, height } = resolveLayerResolutionSize(
|
|
originalWidth,
|
|
originalHeight,
|
|
{
|
|
width: sourceLayer.width,
|
|
height: sourceLayer.height,
|
|
},
|
|
);
|
|
const frameX =
|
|
frame && frame.width > 0
|
|
? frame.x + frame.width / 2 - width / 2
|
|
: undefined;
|
|
const frameY =
|
|
frame && frame.height > 0
|
|
? frame.y + frame.height / 2 - height / 2
|
|
: undefined;
|
|
|
|
return applyGeneratedMetadata(
|
|
{
|
|
id: `layer-quick-edit-${generatedIndex}`,
|
|
resourceId: `local-resource-quick-edit-${generatedIndex}`,
|
|
title: `${sourceLayer.title} ${mode === 'redraw' ? '重绘' : '快速编辑'}`,
|
|
src: generated.imageSrc,
|
|
x: frameX ?? sourceLayer.x + sourceLayer.width + 32,
|
|
y: frameY ?? sourceLayer.y,
|
|
width,
|
|
height,
|
|
originalWidth,
|
|
originalHeight,
|
|
zIndex: generatedIndex + 10,
|
|
sourceType: generated.sourceType,
|
|
sourceResourceId: sourceLayer.resourceId,
|
|
groupId: sourceLayer.groupId,
|
|
assetKind: sourceLayer.assetKind,
|
|
},
|
|
generated,
|
|
generationInputs,
|
|
);
|
|
}
|
|
|
|
export function createIconSpritesheetResultLayers({
|
|
generated,
|
|
iconResults,
|
|
startIndex,
|
|
canvasSize,
|
|
viewport,
|
|
generationInputs,
|
|
frame,
|
|
spritesheetTitle,
|
|
}: IconSpritesheetResultLayerOptions): CanvasLayer[] {
|
|
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
|
|
const startX =
|
|
frame?.x ?? worldCenter.x - ICON_FRAME_DISPLAY_SIZE.width / 2;
|
|
const startY =
|
|
frame?.y ?? worldCenter.y - ICON_FRAME_DISPLAY_SIZE.height / 2;
|
|
const spacing = 24;
|
|
const maxRowWidth = 560;
|
|
const spritesheetOriginalWidth =
|
|
generated.spritesheetWidth || ICON_FRAME_ORIGINAL_SIZE.width;
|
|
const spritesheetOriginalHeight =
|
|
generated.spritesheetHeight || ICON_FRAME_ORIGINAL_SIZE.height;
|
|
const spritesheetDisplaySize = resolveLayerResolutionSize(
|
|
spritesheetOriginalWidth,
|
|
spritesheetOriginalHeight,
|
|
ICON_FRAME_DISPLAY_SIZE,
|
|
);
|
|
const spritesheetLayer: CanvasLayer = {
|
|
id: `layer-icon-spritesheet-${startIndex}`,
|
|
resourceId: `local-resource-icon-spritesheet-${startIndex}`,
|
|
title: spritesheetTitle ?? `图标素材图集 ${startIndex}`,
|
|
src: generated.spritesheetImageSrc,
|
|
x: startX,
|
|
y: startY,
|
|
width: spritesheetDisplaySize.width,
|
|
height: spritesheetDisplaySize.height,
|
|
originalWidth: spritesheetOriginalWidth,
|
|
originalHeight: spritesheetOriginalHeight,
|
|
zIndex: startIndex + 10,
|
|
sourceType: 'generated',
|
|
prompt: generated.prompt,
|
|
actualPrompt: generated.actualPrompt ?? generated.prompt,
|
|
model: generated.model,
|
|
provider: generated.provider,
|
|
taskId: generated.taskId,
|
|
assetKind: 'icon-spritesheet',
|
|
generationInputs,
|
|
};
|
|
const iconStartX = startX + spritesheetDisplaySize.width + 32;
|
|
let cursorX = iconStartX;
|
|
let cursorY = startY;
|
|
let rowHeight = 0;
|
|
|
|
const iconLayers = iconResults.map((icon, index) => {
|
|
const originalWidth = icon.width || 128;
|
|
const originalHeight = icon.height || 128;
|
|
const { width, height } = resolveLayerResolutionSize(
|
|
originalWidth,
|
|
originalHeight,
|
|
{ width: 128, height: 128 },
|
|
);
|
|
if (cursorX > iconStartX && cursorX + width - iconStartX > maxRowWidth) {
|
|
cursorX = iconStartX;
|
|
cursorY += rowHeight + spacing;
|
|
rowHeight = 0;
|
|
}
|
|
|
|
const generatedIndex = startIndex + index + 1;
|
|
const layer: CanvasLayer = {
|
|
id: `layer-icon-${generatedIndex}`,
|
|
resourceId: `local-resource-icon-${generatedIndex}`,
|
|
title: icon.name,
|
|
src: icon.imageSrc,
|
|
x: cursorX,
|
|
y: cursorY,
|
|
width,
|
|
height,
|
|
originalWidth,
|
|
originalHeight,
|
|
zIndex: generatedIndex + 10,
|
|
sourceType: 'generated',
|
|
prompt: generated.prompt,
|
|
actualPrompt: generated.actualPrompt ?? generated.prompt,
|
|
model: generated.model,
|
|
provider: generated.provider,
|
|
taskId: generated.taskId,
|
|
assetKind: 'icon',
|
|
generationInputs,
|
|
};
|
|
|
|
cursorX += width + spacing;
|
|
rowHeight = Math.max(rowHeight, height);
|
|
return layer;
|
|
});
|
|
|
|
return [spritesheetLayer, ...iconLayers];
|
|
}
|
|
|
|
export function createVideoResultLayer({
|
|
generated,
|
|
generatedIndex,
|
|
title,
|
|
canvasSize,
|
|
viewport,
|
|
generationInputs,
|
|
frame,
|
|
}: VideoResultLayerOptions): CanvasLayer {
|
|
const originalWidth = generated.width || 1280;
|
|
const originalHeight = generated.height || 720;
|
|
const { width, height } = resolveLayerResolutionSize(
|
|
originalWidth,
|
|
originalHeight,
|
|
{ width: 560, height: 315 },
|
|
);
|
|
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
|
|
const frameX =
|
|
frame && frame.width > 0
|
|
? frame.x + frame.width / 2 - width / 2
|
|
: undefined;
|
|
const frameY =
|
|
frame && frame.height > 0
|
|
? frame.y + frame.height / 2 - height / 2
|
|
: undefined;
|
|
|
|
return {
|
|
id: `layer-video-${generatedIndex}`,
|
|
resourceId: `local-resource-video-${generatedIndex}`,
|
|
title,
|
|
src: generated.videoSrc,
|
|
mediaType: 'video',
|
|
assetKind: 'video',
|
|
x: frameX ?? worldCenter.x - width / 2,
|
|
y: frameY ?? worldCenter.y - height / 2,
|
|
width,
|
|
height,
|
|
originalWidth,
|
|
originalHeight,
|
|
zIndex: generatedIndex + 10,
|
|
sourceType: generated.sourceType,
|
|
prompt: generated.prompt,
|
|
actualPrompt: generated.actualPrompt ?? generated.prompt,
|
|
model: generated.model,
|
|
provider: generated.provider,
|
|
taskId: generated.taskId,
|
|
generationInputs,
|
|
};
|
|
}
|
|
|
|
export function createAudioResultLayer({
|
|
generated,
|
|
generatedIndex,
|
|
title,
|
|
canvasSize,
|
|
viewport,
|
|
generationInputs,
|
|
frame,
|
|
}: AudioResultLayerOptions): CanvasLayer {
|
|
const originalWidth = generated.width || 420;
|
|
const originalHeight = generated.height || 120;
|
|
const { width, height } = resolveLayerResolutionSize(
|
|
originalWidth,
|
|
originalHeight,
|
|
{ width: 420, height: 120 },
|
|
);
|
|
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
|
|
const frameX =
|
|
frame && frame.width > 0
|
|
? frame.x + frame.width / 2 - width / 2
|
|
: undefined;
|
|
const frameY =
|
|
frame && frame.height > 0
|
|
? frame.y + frame.height / 2 - height / 2
|
|
: undefined;
|
|
|
|
return {
|
|
id: `layer-audio-${generatedIndex}`,
|
|
resourceId: `local-resource-audio-${generatedIndex}`,
|
|
title,
|
|
src: generated.audioSrc,
|
|
mediaType: 'audio',
|
|
assetKind: generated.audioKind,
|
|
x: frameX ?? worldCenter.x - width / 2,
|
|
y: frameY ?? worldCenter.y - height / 2,
|
|
width,
|
|
height,
|
|
originalWidth,
|
|
originalHeight,
|
|
zIndex: generatedIndex + 10,
|
|
sourceType: generated.sourceType,
|
|
prompt: generated.prompt,
|
|
actualPrompt: generated.actualPrompt ?? generated.prompt,
|
|
model: generated.model,
|
|
provider: generated.provider,
|
|
taskId: generated.taskId,
|
|
durationSeconds: generated.durationSeconds ?? undefined,
|
|
generationInputs,
|
|
};
|
|
}
|
|
|
|
export function createCharacterAnimationResultLayer({
|
|
generated,
|
|
generatedIndex,
|
|
title,
|
|
canvasSize,
|
|
viewport,
|
|
generationInputs,
|
|
frame,
|
|
}: CharacterAnimationResultLayerOptions): CanvasLayer | null {
|
|
const previewFrame = generated.frames[0];
|
|
if (!previewFrame) {
|
|
return null;
|
|
}
|
|
const originalWidth = previewFrame.width || 1024;
|
|
const originalHeight = previewFrame.height || 1024;
|
|
const { width, height } = resolveLayerResolutionSize(
|
|
originalWidth,
|
|
originalHeight,
|
|
{ width: 420, height: 420 },
|
|
);
|
|
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
|
|
const frameX =
|
|
frame && frame.width > 0
|
|
? frame.x + frame.width / 2 - width / 2
|
|
: undefined;
|
|
const frameY =
|
|
frame && frame.height > 0
|
|
? frame.y + frame.height / 2 - height / 2
|
|
: undefined;
|
|
|
|
return {
|
|
id: `layer-character-animation-${generatedIndex}`,
|
|
resourceId: `local-resource-character-animation-${generatedIndex}`,
|
|
title,
|
|
src: previewFrame.imageSrc,
|
|
x: frameX ?? worldCenter.x - width / 2,
|
|
y: frameY ?? worldCenter.y - height / 2,
|
|
width,
|
|
height,
|
|
originalWidth,
|
|
originalHeight,
|
|
zIndex: generatedIndex + 10,
|
|
sourceType: 'generated',
|
|
prompt: generated.prompt,
|
|
actualPrompt: generated.prompt,
|
|
model: generated.model,
|
|
provider: 'ark',
|
|
taskId: generated.taskId,
|
|
assetKind: 'character-animation',
|
|
generationInputs,
|
|
};
|
|
}
|