Files
Genarrative/src/components/image-editor/ImageCanvasGenerationLayerModel.ts
T
kdletters 6d4033f044 让音频生成进入素材库
音效和背景音乐生成复用媒体素材持久化路径

前端提交音频生成时携带素材库上下文并消费返回素材快照

更新音频生成契约、OpenAPI 和相关文档

补充音频素材库链路回归测试
2026-07-02 20:28:26 +08:00

571 lines
17 KiB
TypeScript

import type {
EditorAudioGenerationResult,
EditorCharacterAnimationGenerationResult,
EditorIconSpritesheetGenerationResult,
EditorIconSpritesheetIconResult,
EditorImageGenerationResult,
EditorVideoGenerationResult,
} from '../../services/image-editor/editorProjectClient';
import { resolveLayerResolutionSize } from './ImageCanvasEditorModel';
import type {
CanvasGenerationInputs,
CanvasLayer,
CanvasViewport,
GenerateDialogState,
} from './ImageCanvasEditorTypes';
import {
ICON_FRAME_DISPLAY_SIZE,
ICON_FRAME_ORIGINAL_SIZE,
} from './ImageCanvasGenerationModel';
type CanvasSize = { width: number; height: number };
type GeneratedResultLayerOptions = {
generated: EditorImageGenerationResult;
generatedIndex: number;
canvasSize: CanvasSize;
viewport: CanvasViewport;
sourceLayer?: CanvasLayer;
frame?: GenerateDialogState['placeholder'];
assetKind?: CanvasLayer['assetKind'];
title?: string;
generationInputs?: CanvasGenerationInputs;
};
type QuickEditResultLayerOptions = {
generated: EditorImageGenerationResult;
generatedIndex: number;
sourceLayer: CanvasLayer;
generationInputs: CanvasGenerationInputs;
mode?: 'quick-edit' | 'redraw';
frame?: GenerateDialogState['placeholder'];
};
type IconSpritesheetResultLayerOptions = {
generated: EditorIconSpritesheetGenerationResult;
iconResults?: EditorIconSpritesheetIconResult[] | null;
startIndex: number;
canvasSize: CanvasSize;
viewport: CanvasViewport;
generationInputs: CanvasGenerationInputs;
frame?: GenerateDialogState['placeholder'];
spritesheetTitle?: string;
};
type VideoResultLayerOptions = {
generated: EditorVideoGenerationResult;
generatedIndex: number;
title: string;
canvasSize: CanvasSize;
viewport: CanvasViewport;
generationInputs: CanvasGenerationInputs;
frame?: GenerateDialogState['placeholder'];
sourceLayer?: CanvasLayer;
assetKind?: CanvasLayer['assetKind'];
};
type AudioResultLayerOptions = {
generated: EditorAudioGenerationResult;
generatedIndex: number;
title: string;
canvasSize: CanvasSize;
viewport: CanvasViewport;
generationInputs: CanvasGenerationInputs;
frame?: GenerateDialogState['placeholder'];
};
type CharacterAnimationResultLayerOptions = {
generated: EditorCharacterAnimationGenerationResult;
generatedIndex: number;
title: string;
canvasSize: CanvasSize;
viewport: CanvasViewport;
generationInputs: CanvasGenerationInputs;
frame?: GenerateDialogState['placeholder'];
sourceLayer?: CanvasLayer;
};
function getViewportWorldCenter({
canvasSize,
viewport,
}: {
canvasSize: CanvasSize;
viewport: CanvasViewport;
}) {
const safeScale = viewport.scale > 0 ? viewport.scale : 1;
return {
x: (canvasSize.width / 2 - viewport.x) / safeScale,
y: (canvasSize.height / 2 - viewport.y) / safeScale,
};
}
function applyGeneratedMetadata(
layer: CanvasLayer,
generated: EditorImageGenerationResult,
generationInputs: CanvasGenerationInputs | undefined,
): CanvasLayer {
const resource = generated.resource;
const asset = generated.asset;
return {
...layer,
resourceId: resource?.resourceId ?? layer.resourceId,
sourceAssetId: asset?.assetId ?? layer.sourceAssetId,
prompt: generated.prompt,
actualPrompt: generated.actualPrompt ?? generated.prompt,
model: generated.model,
provider: generated.provider,
taskId: generated.taskId,
objectKey: resource?.objectKey ?? generated.objectKey,
assetObjectId: resource?.assetObjectId ?? generated.assetObjectId,
assetKind:
(resource?.assetKind as CanvasLayer['assetKind'] | undefined) ??
layer.assetKind,
generationInputs: resource?.generationInputs ?? generationInputs,
generatedAssetSnapshot: asset,
};
}
export function createGeneratedResultLayer({
generated,
generatedIndex,
canvasSize,
viewport,
sourceLayer,
frame,
assetKind,
title,
generationInputs,
}: GeneratedResultLayerOptions): CanvasLayer {
const originalWidth = frame?.originalWidth || generated.width || 1024;
const originalHeight = frame?.originalHeight || generated.height || 1024;
const { width, height } = resolveLayerResolutionSize(
originalWidth,
originalHeight,
{ width: 1024, height: 1024 },
);
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
const frameX =
frame && frame.width > 0
? frame.x + frame.width / 2 - width / 2
: undefined;
const frameY =
frame && frame.height > 0
? frame.y + frame.height / 2 - height / 2
: undefined;
return applyGeneratedMetadata(
{
id: sourceLayer
? `layer-edit-${generatedIndex}`
: `layer-generated-${generatedIndex}`,
resourceId: sourceLayer
? `local-resource-edit-${generatedIndex}`
: `local-resource-generated-${generatedIndex}`,
title: sourceLayer
? `${sourceLayer.title} 修改结果`
: (title ?? `生成图片 ${generatedIndex}`),
src: generated.imageSrc,
x: sourceLayer
? sourceLayer.x + sourceLayer.width + 32
: (frameX ?? worldCenter.x - width / 2),
y: sourceLayer ? sourceLayer.y : (frameY ?? worldCenter.y - height / 2),
width,
height,
originalWidth,
originalHeight,
zIndex: generatedIndex + 10,
sourceType: generated.sourceType,
assetKind,
sourceResourceId: sourceLayer?.resourceId,
},
generated,
generationInputs,
);
}
export function createQuickEditResultLayer({
generated,
generatedIndex,
sourceLayer,
generationInputs,
mode = 'quick-edit',
frame,
}: QuickEditResultLayerOptions): CanvasLayer {
const originalWidth =
frame?.originalWidth || generated.width || sourceLayer.originalWidth || 1024;
const originalHeight =
frame?.originalHeight ||
generated.height ||
sourceLayer.originalHeight ||
1024;
const { width, height } = resolveLayerResolutionSize(
originalWidth,
originalHeight,
{
width: sourceLayer.width,
height: sourceLayer.height,
},
);
const frameX =
frame && frame.width > 0
? frame.x + frame.width / 2 - width / 2
: undefined;
const frameY =
frame && frame.height > 0
? frame.y + frame.height / 2 - height / 2
: undefined;
return applyGeneratedMetadata(
{
id: `layer-quick-edit-${generatedIndex}`,
resourceId: `local-resource-quick-edit-${generatedIndex}`,
title: `${sourceLayer.title} ${mode === 'redraw' ? '重绘' : '快速编辑'}`,
src: generated.imageSrc,
x: frameX ?? sourceLayer.x + sourceLayer.width + 32,
y: frameY ?? sourceLayer.y,
width,
height,
originalWidth,
originalHeight,
zIndex: generatedIndex + 10,
sourceType: generated.sourceType,
sourceResourceId: sourceLayer.resourceId,
groupId: sourceLayer.groupId,
assetKind: sourceLayer.assetKind,
},
generated,
generationInputs,
);
}
export function applyImageEditResultToSourceLayer({
generated,
sourceLayer,
generationInputs,
}: {
generated: EditorImageGenerationResult;
sourceLayer: CanvasLayer;
generationInputs: CanvasGenerationInputs;
}): CanvasLayer {
const originalWidth =
generated.width || sourceLayer.originalWidth || sourceLayer.width;
const originalHeight =
generated.height || sourceLayer.originalHeight || sourceLayer.height;
const { width, height } = resolveLayerResolutionSize(
originalWidth,
originalHeight,
{
width: sourceLayer.width,
height: sourceLayer.height,
},
);
return applyGeneratedMetadata(
{
...sourceLayer,
src: generated.imageSrc,
width,
height,
originalWidth,
originalHeight,
sourceType: generated.sourceType,
sourceResourceId: sourceLayer.resourceId,
},
generated,
generationInputs,
);
}
export function createIconSpritesheetResultLayers({
generated,
iconResults,
startIndex,
canvasSize,
viewport,
generationInputs,
frame,
spritesheetTitle,
}: IconSpritesheetResultLayerOptions): CanvasLayer[] {
const spritesheetResource = generated.spritesheetResource;
const spritesheetAsset = generated.spritesheetAsset;
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
const startX =
frame?.x ?? worldCenter.x - ICON_FRAME_DISPLAY_SIZE.width / 2;
const startY =
frame?.y ?? worldCenter.y - ICON_FRAME_DISPLAY_SIZE.height / 2;
const spacing = 24;
const maxRowWidth = 560;
const spritesheetOriginalWidth =
generated.spritesheetWidth || ICON_FRAME_ORIGINAL_SIZE.width;
const spritesheetOriginalHeight =
generated.spritesheetHeight || ICON_FRAME_ORIGINAL_SIZE.height;
const spritesheetDisplaySize = resolveLayerResolutionSize(
spritesheetOriginalWidth,
spritesheetOriginalHeight,
ICON_FRAME_DISPLAY_SIZE,
);
const spritesheetLayer: CanvasLayer = {
id: `layer-icon-spritesheet-${startIndex}`,
resourceId:
spritesheetResource?.resourceId ??
`local-resource-icon-spritesheet-${startIndex}`,
title: spritesheetTitle ?? `图标素材图集 ${startIndex}`,
src: generated.spritesheetImageSrc,
x: startX,
y: startY,
width: spritesheetDisplaySize.width,
height: spritesheetDisplaySize.height,
originalWidth: spritesheetOriginalWidth,
originalHeight: spritesheetOriginalHeight,
zIndex: startIndex + 10,
sourceType: 'generated',
prompt: generated.prompt,
actualPrompt: generated.actualPrompt ?? generated.prompt,
model: generated.model,
provider: generated.provider,
taskId: generated.taskId,
assetKind: 'icon-spritesheet',
sourceAssetId: spritesheetAsset?.assetId,
objectKey: spritesheetResource?.objectKey ?? undefined,
assetObjectId: spritesheetResource?.assetObjectId ?? undefined,
generationInputs:
spritesheetResource?.generationInputs ?? generationInputs,
generatedAssetSnapshot: spritesheetAsset,
};
const iconStartX = startX + spritesheetDisplaySize.width + 32;
let cursorX = iconStartX;
let cursorY = startY;
let rowHeight = 0;
const normalizedIconResults = Array.isArray(iconResults) ? iconResults : [];
const iconLayers = normalizedIconResults.map((icon, index) => {
const resource = icon.resource;
const asset = icon.asset;
const originalWidth = icon.width || 128;
const originalHeight = icon.height || 128;
const { width, height } = resolveLayerResolutionSize(
originalWidth,
originalHeight,
{ width: 128, height: 128 },
);
if (cursorX > iconStartX && cursorX + width - iconStartX > maxRowWidth) {
cursorX = iconStartX;
cursorY += rowHeight + spacing;
rowHeight = 0;
}
const generatedIndex = startIndex + index + 1;
const layer: CanvasLayer = {
id: `layer-icon-${generatedIndex}`,
resourceId: resource?.resourceId ?? `local-resource-icon-${generatedIndex}`,
title: icon.name,
src: icon.imageSrc,
x: cursorX,
y: cursorY,
width,
height,
originalWidth,
originalHeight,
zIndex: generatedIndex + 10,
sourceType: 'generated',
prompt: generated.prompt,
actualPrompt: generated.actualPrompt ?? generated.prompt,
model: generated.model,
provider: generated.provider,
taskId: generated.taskId,
assetKind: 'icon',
sourceAssetId: asset?.assetId,
objectKey: resource?.objectKey ?? undefined,
assetObjectId: resource?.assetObjectId ?? undefined,
generationInputs: resource?.generationInputs ?? generationInputs,
generatedAssetSnapshot: asset,
};
cursorX += width + spacing;
rowHeight = Math.max(rowHeight, height);
return layer;
});
return [spritesheetLayer, ...iconLayers];
}
export function createVideoResultLayer({
generated,
generatedIndex,
title,
canvasSize,
viewport,
generationInputs,
frame,
sourceLayer,
assetKind = 'video',
}: VideoResultLayerOptions): CanvasLayer {
const originalWidth = generated.width || 1280;
const originalHeight = generated.height || 720;
const { width, height } = resolveLayerResolutionSize(
originalWidth,
originalHeight,
{ width: 560, height: 315 },
);
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
const frameX =
frame && frame.width > 0
? frame.x + frame.width / 2 - width / 2
: undefined;
const frameY =
frame && frame.height > 0
? frame.y + frame.height / 2 - height / 2
: undefined;
const resource = generated.resource;
const asset = generated.asset;
return {
id: `layer-video-${generatedIndex}`,
resourceId: resource?.resourceId ?? `local-resource-video-${generatedIndex}`,
title,
src: generated.videoSrc,
mediaType: 'video',
assetKind: (resource?.assetKind as CanvasLayer['assetKind'] | undefined) ?? assetKind,
x: frameX ?? worldCenter.x - width / 2,
y: frameY ?? worldCenter.y - height / 2,
width,
height,
originalWidth,
originalHeight,
zIndex: generatedIndex + 10,
sourceType: generated.sourceType,
prompt: generated.prompt,
actualPrompt: generated.actualPrompt ?? generated.prompt,
model: generated.model,
provider: generated.provider,
taskId: generated.taskId,
objectKey: resource?.objectKey ?? generated.objectKey,
assetObjectId: resource?.assetObjectId ?? generated.assetObjectId,
thumbnailSrc: generated.thumbnailSrc ?? generated.asset?.thumbnailSrc ?? undefined,
sourceResourceId: sourceLayer?.resourceId,
sourceAssetId: asset?.assetId,
groupId: sourceLayer?.groupId,
generationInputs: resource?.generationInputs ?? generationInputs,
durationSeconds: generated.durationSeconds,
generatedAssetSnapshot: asset ?? generated.asset ?? undefined,
};
}
export function createAudioResultLayer({
generated,
generatedIndex,
title,
canvasSize,
viewport,
generationInputs,
frame,
}: AudioResultLayerOptions): CanvasLayer {
const resource = generated.resource;
const asset = generated.asset;
const originalWidth = generated.width || 420;
const originalHeight = generated.height || 120;
const { width, height } = resolveLayerResolutionSize(
originalWidth,
originalHeight,
{ width: 420, height: 120 },
);
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
const frameX =
frame && frame.width > 0
? frame.x + frame.width / 2 - width / 2
: undefined;
const frameY =
frame && frame.height > 0
? frame.y + frame.height / 2 - height / 2
: undefined;
return {
id: `layer-audio-${generatedIndex}`,
resourceId:
resource?.resourceId ?? `local-resource-audio-${generatedIndex}`,
title,
src: resource?.imageSrc ?? generated.audioSrc,
mediaType: 'audio',
assetKind: generated.audioKind,
x: frameX ?? worldCenter.x - width / 2,
y: frameY ?? worldCenter.y - height / 2,
width,
height,
originalWidth,
originalHeight,
zIndex: generatedIndex + 10,
sourceType: generated.sourceType,
prompt: generated.prompt,
actualPrompt: generated.actualPrompt ?? generated.prompt,
model: generated.model,
provider: generated.provider,
taskId: generated.taskId,
objectKey: resource?.objectKey ?? generated.objectKey,
assetObjectId: resource?.assetObjectId ?? generated.assetObjectId,
sourceAssetId: asset?.assetId,
durationSeconds: generated.durationSeconds ?? undefined,
generationInputs: resource?.generationInputs ?? generationInputs,
generatedAssetSnapshot: asset ?? undefined,
};
}
export function createCharacterAnimationResultLayer({
generated,
generatedIndex,
title,
canvasSize,
viewport,
generationInputs,
frame,
sourceLayer,
}: CharacterAnimationResultLayerOptions): CanvasLayer | null {
const previewFrame = generated.frames[0];
if (!previewFrame) {
return null;
}
const originalWidth = previewFrame.width || 1024;
const originalHeight = previewFrame.height || 1024;
const { width, height } = resolveLayerResolutionSize(
originalWidth,
originalHeight,
{ width: 420, height: 420 },
);
const worldCenter = getViewportWorldCenter({ canvasSize, viewport });
const frameX =
frame && frame.width > 0
? frame.x + frame.width / 2 - width / 2
: undefined;
const frameY =
frame && frame.height > 0
? frame.y + frame.height / 2 - height / 2
: undefined;
return {
id: `layer-character-animation-${generatedIndex}`,
resourceId: `local-resource-character-animation-${generatedIndex}`,
title,
src: previewFrame.imageSrc,
mediaType: 'image-sequence',
thumbnailSrc: previewFrame.imageSrc,
imageSequenceFrames: generated.frames,
previewVideoPath: generated.previewVideoPath,
x: frameX ?? worldCenter.x - width / 2,
y: frameY ?? worldCenter.y - height / 2,
width,
height,
originalWidth,
originalHeight,
zIndex: generatedIndex + 10,
sourceType: 'generated',
prompt: generated.prompt,
actualPrompt: generated.prompt,
model: generated.model,
provider: 'ark',
taskId: generated.taskId,
assetKind: 'character-animation',
durationSeconds: generated.durationSeconds,
sourceResourceId: sourceLayer?.resourceId,
groupId: sourceLayer?.groupId,
generationInputs,
};
}