Image editor: Seedance2.0 refs and audio changes

Adds Seedance 2.0 reference-media constraints and forbids video data URLs; updates docs/decision-log and pitfalls to record the new rules. Introduces typed audio params (type: one-shot|loop, tempo: number|null), generator-specific placeholder styles/avoidance and UI changes for generation panels. Implements related front-end changes (new crop/ raster edit components, many image-editor models, tests and workflow hooks) and server-side contract/handler updates across api-server, shared-contracts, platform-audio, platform-image and vector_engine_audio_generation to support the new submission/asset flow and validations.
This commit is contained in:
2026-06-18 17:31:35 +08:00
parent 6f1ae6caef
commit 9c7b3ad16d
68 changed files with 3262 additions and 1294 deletions
@@ -1,4 +1,4 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { afterEach, describe, expect, it, vi } from 'vitest';
import {
createEditorAsset,
@@ -9,8 +9,8 @@ import {
deleteEditorAssetFolder,
deleteEditorProject,
editEditorImage,
generateEditorCharacterAnimation,
generateEditorBackgroundMusic,
generateEditorCharacterAnimation,
generateEditorIconSpritesheet,
generateEditorImage,
generateEditorSoundEffect,
@@ -591,11 +591,13 @@ describe('editorProjectClient', () => {
model: 'gemini-3.1-flash-image-preview',
provider: 'VectorEngine',
taskId: 'icon-spritesheet-task-1',
priceMudPoints: 12,
});
const result = await generateEditorIconSpritesheet({
referenceImageSrc: 'data:image/png;base64,spec',
iconDescriptions: ['返回按钮', '设置按钮'],
priceMudPoints: 12,
});
expect(result.taskId).toBe('icon-spritesheet-task-1');
@@ -608,6 +610,7 @@ describe('editorProjectClient', () => {
referenceImageSrc: 'data:image/png;base64,spec',
iconDescriptions: ['返回按钮', '设置按钮'],
model: 'gemini-3.1-flash-image-preview',
priceMudPoints: 12,
}),
}),
'生成图标素材失败',
@@ -629,6 +632,7 @@ describe('editorProjectClient', () => {
model: 'gpt-image-2',
provider: 'VectorEngine',
taskId: 'icon-spritesheet-task-2',
priceMudPoints: 12,
});
await generateEditorIconSpritesheet({
@@ -637,6 +641,7 @@ describe('editorProjectClient', () => {
model: 'gpt-image-2',
aspectRatio: '1:1',
imageSize: '2K',
priceMudPoints: 12,
});
expect(requestJsonMock).toHaveBeenCalledWith(
@@ -650,6 +655,7 @@ describe('editorProjectClient', () => {
model: 'gpt-image-2',
aspectRatio: '1:1',
imageSize: '2K',
priceMudPoints: 12,
}),
}),
'生成图标素材失败',
@@ -842,8 +848,12 @@ describe('editorProjectClient', () => {
sound: 'off',
priceMudPoints: 100,
referenceImageSrcs: ['data:image/png;base64,image'],
referenceVideoSrcs: ['data:video/mp4;base64,video'],
referenceAudioSrcs: ['data:audio/mpeg;base64,audio'],
referenceVideoSrcs: [
'generated-character-drafts/editor/seedance-references/video/video.mp4',
],
referenceAudioSrcs: [
'generated-character-drafts/editor/seedance-references/audio/audio.mp3',
],
});
expect(requestJsonMock).toHaveBeenCalledWith(
@@ -861,8 +871,12 @@ describe('editorProjectClient', () => {
sound: 'off',
priceMudPoints: 100,
referenceImageSrcs: ['data:image/png;base64,image'],
referenceVideoSrcs: ['data:video/mp4;base64,video'],
referenceAudioSrcs: ['data:audio/mpeg;base64,audio'],
referenceVideoSrcs: [
'generated-character-drafts/editor/seedance-references/video/video.mp4',
],
referenceAudioSrcs: [
'generated-character-drafts/editor/seedance-references/audio/audio.mp3',
],
}),
}),
'生成视频失败',
@@ -887,8 +901,8 @@ describe('editorProjectClient', () => {
const result = await generateEditorSoundEffect({
sound: '金币掉落叮当声',
type: 'ui',
tempo: 'fast',
type: 'one-shot',
tempo: 120,
priceMudPoints: 10,
});
@@ -900,8 +914,8 @@ describe('editorProjectClient', () => {
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sound: '金币掉落叮当声',
type: 'ui',
tempo: 'fast',
type: 'one-shot',
tempo: 120,
priceMudPoints: 10,
}),
}),
@@ -913,6 +927,43 @@ describe('editorProjectClient', () => {
);
});
it('keeps editor sound effect tempo as null when BPM is not selected', async () => {
requestJsonMock.mockResolvedValueOnce({
audioSrc: '/generated-character-drafts/editor-audios/sfx-null.mp3',
width: 420,
height: 120,
sourceType: 'generated',
prompt: '按钮确认短促音',
actualPrompt: '按钮确认短促音',
model: 'audio1.0',
provider: 'VectorEngine',
taskId: 'sound-task-null',
priceMudPoints: 10,
audioKind: 'sound-effect',
});
await generateEditorSoundEffect({
sound: '按钮确认短促音',
type: 'one-shot',
tempo: null,
priceMudPoints: 10,
});
expect(requestJsonMock).toHaveBeenCalledWith(
'/api/editor/audios/sound-effects/generations',
expect.objectContaining({
body: JSON.stringify({
sound: '按钮确认短促音',
type: 'one-shot',
tempo: null,
priceMudPoints: 10,
}),
}),
'生成游戏音效失败',
expect.any(Object),
);
});
it('generates editor background music through the backend BFF', async () => {
requestJsonMock.mockResolvedValueOnce({
audioSrc: '/generated-character-drafts/editor-audios/bgm.mp3',
@@ -101,6 +101,7 @@ export type EditorIconSpritesheetGenerationInput = {
model?: string;
aspectRatio?: string;
imageSize?: string;
priceMudPoints: number;
};
export type EditorImageEditInput = {
@@ -141,6 +142,7 @@ export type EditorIconSpritesheetGenerationResult = {
model: string;
provider: string;
taskId: string;
priceMudPoints: number;
};
export type EditorCharacterAnimationResolution = '480p' | '720p';
@@ -227,8 +229,8 @@ export type EditorVideoGenerationResult = {
export type EditorSoundEffectGenerationInput = {
sound: string;
type: 'ui' | 'foley' | 'ambient';
tempo: 'slow' | 'medium' | 'fast';
type: 'one-shot' | 'loop';
tempo: number | null;
priceMudPoints: number;
};
@@ -558,6 +560,7 @@ export async function generateEditorIconSpritesheet(
model: input.model?.trim() || EDITOR_IMAGE_MODEL_NANOBANANA2,
...(input.aspectRatio ? { aspectRatio: input.aspectRatio } : {}),
...(input.imageSize ? { imageSize: input.imageSize } : {}),
priceMudPoints: input.priceMudPoints,
}),
'生成图标素材失败',
{