GenarrativeAI/Genarrative
11
0
Fork 0
Code Issues 18 Pull Requests 3 Actions Packages Projects 3 Releases Wiki Activity

实现 SFX V2 T1 契约与提示词基础

Browse Source 
统一前后端 SFX Prompt Unicode canonicalization、字符计数和 2048 边界
固化 52 个音效预设、追加规则及跨语言测试向量
演进共享音频请求响应、duration、Loop 和 V2 metadata 契约
增加 External model 输入矩阵纯函数并保持正式接线归属 T5
补齐前端提交、读取、深拷贝与请求边界回归测试
更新共享计划并标记 T1 完成且当前不可发布
This commit is contained in:
孔令弘
2026-08-06 13:05:48 +00:00
parent 6fd731f56c
commit cd80c085ff
24 changed files with 1559 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/image-editor/ImageCanvasEditorModel.test.ts
+42
View File
@@ -13,6 +13,7 @@ import {
DEFAULT_CANVAS_BACKGROUND_COLOR,
dropDeadInlineGenerationPlaceholders,
formatCanvasDisplayScalePercent,
generationInputsOrNull,
hydrateCanvasGenerationDialog,
hydrateLayer,
INLINE_GENERATION_PLACEHOLDER_LIVE_WINDOW_MS,
@@ -971,6 +972,47 @@ describe('ImageCanvasEditorModel', () => {
expect(hydrated).not.toHaveProperty('durationSeconds');
});
it('keeps only valid authoritative SFX V2 metadata in generation inputs', () => {
const soundEffect = {
schemaVersion: 2 as const,
userPrompt: '金币落地，轻快可爱',
actualPrompt: 'A bright, cute coin landing chime',
model: 'eleven_text_to_sound_v2' as const,
durationMode: 'manual' as const,
requestedDurationSeconds: 5,
actualDurationSeconds: 5.12,
loop: false,
};
expect(
generationInputsOrNull({ fields: [], references: [], soundEffect }),
).toEqual({ fields: [], references: [], soundEffect });
for (const invalidSoundEffect of [
{ ...soundEffect, schemaVersion: 1 },
{ ...soundEffect, model: 'audio1.0' },
{ ...soundEffect, userPrompt: ' 金币落地' },
{ ...soundEffect, actualDurationSeconds: 600.1 },
{
...soundEffect,
durationMode: 'auto',
requestedDurationSeconds: 5,
},
{
...soundEffect,
durationMode: 'manual',
requestedDurationSeconds: null,
},
]) {
expect(
generationInputsOrNull({
fields: [{ title: '用户描述', value: '金币落地' }],
references: [],
soundEffect: invalidSoundEffect,
}),
).toBeNull();
}
});
it('hydrates character animation sequence fields from the project resource', () => {
const layer: CanvasLayer = {
id: 'layer-action',
src/components/image-editor/ImageCanvasEditorModel.ts
+82 -1
View File
@@ -1,3 +1,8 @@
import {
EDITOR_SOUND_EFFECT_MODEL,
SOUND_EFFECT_DURATION_MAX_SECONDS,
SOUND_EFFECT_DURATION_MIN_SECONDS,
} from '../../../packages/shared/src/contracts/editorAudio';
import type {
EditorAssetGenerationInputs,
EditorAssetLibrarySnapshot,
@@ -21,6 +26,7 @@ import type {
PerfectPixelOperationSnapshot,
SnapCandidate,
} from './ImageCanvasEditorTypes';
import { validateSoundEffectPrompt } from './ImageCanvasSoundEffectPromptModel';
export const EDITOR_ASSET_FOLDERS: EditorAssetFolder[] = [
{
@@ -2170,6 +2176,7 @@ export function generationInputsOrNull(
const snapshot = value as {
fields?: unknown;
references?: unknown;
soundEffect?: unknown;
};
const fields = Array.isArray(snapshot.fields)
? snapshot.fields.flatMap((field) => {
@@ -2206,7 +2213,81 @@ export function generationInputsOrNull(
})
: [];
return fields.length || references.length ? { fields, references } : null;
const hasSoundEffect = Object.prototype.hasOwnProperty.call(
snapshot,
'soundEffect',
);
const soundEffect = hasSoundEffect
? soundEffectGenerationMetadataOrNull(snapshot.soundEffect)
: null;
if (hasSoundEffect && !soundEffect) {
return null;
}
return fields.length || references.length || soundEffect
? {
fields,
references,
...(soundEffect ? { soundEffect } : {}),
}
: null;
}
function soundEffectGenerationMetadataOrNull(
value: unknown,
): NonNullable<CanvasGenerationInputs['soundEffect']> | null {
if (!isSnapshotRecord(value)) {
return null;
}
const userPrompt =
typeof value.userPrompt === 'string' ? value.userPrompt : '';
const actualPrompt =
typeof value.actualPrompt === 'string' ? value.actualPrompt : '';
const userPromptValidation = validateSoundEffectPrompt(userPrompt);
const actualPromptValidation = validateSoundEffectPrompt(actualPrompt);
if (
value.schemaVersion !== 2 ||
value.model !== EDITOR_SOUND_EFFECT_MODEL ||
(value.durationMode !== 'auto' && value.durationMode !== 'manual') ||
typeof value.loop !== 'boolean' ||
!userPromptValidation.ok ||
userPromptValidation.prompt !== userPrompt ||
!actualPromptValidation.ok ||
actualPromptValidation.prompt !== actualPrompt ||
typeof value.actualDurationSeconds !== 'number' ||
!Number.isFinite(value.actualDurationSeconds) ||
value.actualDurationSeconds <= 0 ||
value.actualDurationSeconds > 600
) {
return null;
}
let requestedDurationSeconds: number | null;
if (value.durationMode === 'auto') {
if (value.requestedDurationSeconds !== null) {
return null;
}
requestedDurationSeconds = null;
} else {
if (
typeof value.requestedDurationSeconds !== 'number' ||
!Number.isFinite(value.requestedDurationSeconds) ||
value.requestedDurationSeconds < SOUND_EFFECT_DURATION_MIN_SECONDS ||
value.requestedDurationSeconds > SOUND_EFFECT_DURATION_MAX_SECONDS
) {
return null;
}
requestedDurationSeconds = value.requestedDurationSeconds;
}
return {
schemaVersion: 2,
userPrompt,
actualPrompt,
model: EDITOR_SOUND_EFFECT_MODEL,
durationMode: value.durationMode,
requestedDurationSeconds,
actualDurationSeconds: value.actualDurationSeconds,
loop: value.loop,
};
}
export function canvasAssetKindOrNull(value: unknown): CanvasAssetKind | null {
src/components/image-editor/ImageCanvasEditorTypes.ts
+2
View File
@@ -1,3 +1,4 @@
import type { EditorSoundEffectGenerationMetadataV2 } from '../../../packages/shared/src/contracts/editorAudio';
import type {
EditorAssetSnapshot,
EditorCharacterAnimationFrameCount,
@@ -83,6 +84,7 @@ export type CanvasGenerationInputReference = {
export type CanvasGenerationInputs = {
fields: CanvasGenerationInputField[];
references: CanvasGenerationInputReference[];
soundEffect?: EditorSoundEffectGenerationMetadataV2;
};
export type CanvasLayer = {
src/components/image-editor/ImageCanvasGenerationLayerModel.test.ts
+1
View File
@@ -462,6 +462,7 @@ describe('ImageCanvasGenerationLayerModel', () => {
prompt: '金币掉落叮当声',
actualPrompt: '金币掉落叮当声',
model: 'audio1.0',
provider: 'VectorEngine',
taskId: 'sound-task-1',
priceMudPoints: 10,
audioKind: 'sound-effect',
src/components/image-editor/ImageCanvasGenerationSubmissionModel.test.ts
+14 -33
View File
@@ -1026,15 +1026,16 @@ describe('ImageCanvasGenerationSubmissionModel', () => {
normalizedPrompt: '金币掉落叮当声',
input: {
prompt: '金币掉落叮当声',
model: 'audio1.0',
model: 'eleven_text_to_sound_v2',
duration: 7,
loop: false,
},
result: {
title: '游戏音效 4',
generationInputs: {
fields: [
{ title: 'prompt', value: '金币掉落叮当声' },
{ title: 'model', value: 'audio1.0' },
{ title: 'model', value: 'eleven_text_to_sound_v2' },
{ title: '时长', value: '7秒' },
],
references: [],
@@ -1043,38 +1044,18 @@ describe('ImageCanvasGenerationSubmissionModel', () => {
});
});
it('uses the game sound effect fallback for an all-whitespace prompt', () => {
const plan = buildImageGenerationSubmissionPlan({
dialog: {
mode: 'audio-sound-effect',
prompt: ' \t\r\n ',
status: 'idle',
},
layers: [],
nextGeneratedIndex: 6,
});
expect(plan).toEqual({
kind: 'audio',
audioKind: 'sound-effect',
normalizedPrompt: '游戏音效',
input: {
prompt: '游戏音效',
model: 'audio1.0',
duration: 5,
},
result: {
title: '游戏音效 6',
generationInputs: {
fields: [
{ title: 'prompt', value: '游戏音效' },
{ title: 'model', value: 'audio1.0' },
{ title: '时长', value: '5秒' },
],
references: [],
it('rejects an all-whitespace sound effect prompt without a fallback', () => {
expect(() =>
buildImageGenerationSubmissionPlan({
dialog: {
mode: 'audio-sound-effect',
prompt: ' \t\r\n ',
status: 'idle',
},
},
});
layers: [],
nextGeneratedIndex: 6,
}),
).toThrow('音效描述不能为空');
});
it('builds game background music audio submission plans with instrumental fixed to true', () => {
src/components/image-editor/ImageCanvasGenerationSubmissionModel.ts
+18 -7
View File
@@ -1,3 +1,4 @@
import { EDITOR_SOUND_EFFECT_MODEL } from '../../../packages/shared/src/contracts/editorAudio';
import type {
EditorBackgroundMusicGenerationInput,
EditorCharacterAnimationGenerationInput,
@@ -32,7 +33,6 @@ import {
DEFAULT_EDITOR_BGFILTER_SEG_MODEL,
DEFAULT_EDITOR_GENERATION_BACKGROUND_COLOR,
DEFAULT_SOUND_EFFECT_DURATION_SECONDS,
DEFAULT_SOUND_EFFECT_MODEL,
DEFAULT_SPEC_FORM_VALUES,
DEFAULT_VIDEO_ASPECT_RATIO,
DEFAULT_VIDEO_DURATION_SECONDS,
@@ -50,6 +50,7 @@ import {
SPEC_TYPE_LABEL,
} from './ImageCanvasGenerationModel';
import { getPublicationMaterialsWorkflow } from './ImageCanvasPublicationMaterialsModel';
import { validateSoundEffectPrompt } from './ImageCanvasSoundEffectPromptModel';
type ImageGenerationSubmissionOptions = {
dialog: GenerateDialogState;
@@ -135,9 +136,6 @@ function getDialogDefaultPrompt(mode: GenerateDialogState['mode']) {
if (mode === 'edit') {
return '修改当前图片';
}
if (mode === 'audio-sound-effect') {
return '游戏音效';
}
return 'AI 生成图片';
}
@@ -252,8 +250,20 @@ export function buildImageGenerationSubmissionPlan({
};
}
const normalizedPrompt =
dialog.prompt.trim() || getDialogDefaultPrompt(dialog.mode);
const soundEffectPrompt =
dialog.mode === 'audio-sound-effect'
? validateSoundEffectPrompt(dialog.prompt)
: null;
if (soundEffectPrompt && !soundEffectPrompt.ok) {
throw new Error(
soundEffectPrompt.reason === 'empty'
? '音效描述不能为空'
: '音效描述不能超过 2048 个字符',
);
}
const normalizedPrompt = soundEffectPrompt
? soundEffectPrompt.prompt
: dialog.prompt.trim() || getDialogDefaultPrompt(dialog.mode);
if (dialog.mode === 'edit') {
const sourceLayer = layers.find(
@@ -533,7 +543,7 @@ export function buildImageGenerationSubmissionPlan({
}
if (dialog.mode === 'audio-sound-effect') {
const soundModel = dialog.soundModel ?? DEFAULT_SOUND_EFFECT_MODEL;
const soundModel = EDITOR_SOUND_EFFECT_MODEL;
const durationSeconds =
typeof dialog.soundDurationSeconds === 'number'
? Math.min(10, Math.max(2, Math.round(dialog.soundDurationSeconds)))
@@ -546,6 +556,7 @@ export function buildImageGenerationSubmissionPlan({
prompt: normalizedPrompt,
model: soundModel,
duration: durationSeconds,
loop: false,
},
result: {
title: resolveGenerationAssetLabel(
src/components/image-editor/ImageCanvasLayerCommandModel.test.ts
+28
View File
@@ -190,6 +190,34 @@ describe('ImageCanvasLayerCommandModel', () => {
expect(removeCanvasLayers(layers, ['first'])).toEqual([layers[1]]);
});
it('deep-clones authoritative SFX metadata with generation inputs', () => {
const layer = createLayer({
id: 'sound-effect',
generationInputs: {
fields: [{ title: '用户描述', value: '金币落地' }],
references: [],
soundEffect: {
schemaVersion: 2,
userPrompt: '金币落地',
actualPrompt: 'A coin landing sound',
model: 'eleven_text_to_sound_v2',
durationMode: 'auto',
requestedDurationSeconds: null,
actualDurationSeconds: 4.8,
loop: false,
},
},
});
const clipboard = createCanvasLayerClipboard([layer], [layer.id], 'copy');
const cloned = clipboard?.layers[0];
expect(cloned?.generationInputs).toEqual(layer.generationInputs);
expect(cloned?.generationInputs).not.toBe(layer.generationInputs);
expect(cloned?.generationInputs?.soundEffect).not.toBe(
layer.generationInputs?.soundEffect,
);
});
it('moves layer z-indexes with the same commands as the context menu', () => {
const layers = [
createLayer({ id: 'bottom', zIndex: 1 }),
src/components/image-editor/ImageCanvasLayerCommandModel.ts
+4
View File
@@ -24,10 +24,14 @@ function cloneLayer(layer: CanvasLayer): CanvasLayer {
...layer,
generationInputs: layer.generationInputs
? {
...layer.generationInputs,
fields: layer.generationInputs.fields.map((field) => ({ ...field })),
references: layer.generationInputs.references.map((reference) => ({
...reference,
})),
...(layer.generationInputs.soundEffect
? { soundEffect: { ...layer.generationInputs.soundEffect } }
: {}),
}
: layer.generationInputs,
};
src/components/image-editor/ImageCanvasSoundEffectPresetModel.test.ts
+222
View File
@@ -0,0 +1,222 @@
import { describe, expect, it } from 'vitest';
import {
appendSoundEffectPromptPreset,
SOUND_EFFECT_PROMPT_PRESET_GROUPS,
SOUND_EFFECT_PROMPT_PRESETS,
} from './ImageCanvasSoundEffectPresetModel';
describe('ImageCanvasSoundEffectPresetModel', () => {
it('freezes 40 event presets and 12 requirements with unique identities', () => {
expect(SOUND_EFFECT_PROMPT_PRESETS).toHaveLength(52);
expect(
SOUND_EFFECT_PROMPT_PRESETS.filter(
(preset) => preset.category === 'event',
),
).toHaveLength(40);
expect(
SOUND_EFFECT_PROMPT_PRESETS.filter(
(preset) => preset.category === 'requirement',
),
).toHaveLength(12);
expect(new Set(SOUND_EFFECT_PROMPT_PRESETS.map(({ id }) => id)).size).toBe(
52,
);
expect(
new Set(SOUND_EFFECT_PROMPT_PRESETS.map(({ label }) => label)).size,
).toBe(52);
expect(
new Set(SOUND_EFFECT_PROMPT_PRESETS.map(({ group }) => group)),
).toEqual(new Set(SOUND_EFFECT_PROMPT_PRESET_GROUPS));
});
it('locks the visible labels and prompt text in authoritative order', () => {
expect(
SOUND_EFFECT_PROMPT_PRESETS.map(({ category, group, label, prompt }) => [
category,
group,
label,
prompt,
]),
).toEqual([
['event', 'ui-operation', '轻触按钮', '柔和的按钮点击声'],
['event', 'ui-operation', '确认操作', '明亮的确认提示音'],
['event', 'ui-operation', '返回取消', '轻微下降的取消提示音'],
['event', 'ui-operation', '页面切换', '快速掠过的界面切换声'],
['event', 'ui-operation', '通知提醒', '清晰柔和的通知提示音'],
['event', 'ui-operation', '操作错误', '短促克制的错误提示音'],
['event', 'pickup-reward', '金币拾取', '金币拾取时清脆的金属叮当声'],
['event', 'pickup-reward', '道具拾取', '拾取道具时轻快的提示音'],
['event', 'pickup-reward', '获得奖励', '奖励出现时明亮的提示音'],
[
'event',
'pickup-reward',
'宝箱开启',
'金属锁扣弹开，随后响起明亮的奖励提示音',
],
[
'event',
'pickup-reward',
'解锁内容',
'锁定状态解除，随后响起解锁提示音',
],
['event', 'pickup-reward', '稀有掉落', '稀有物品出现时闪耀的奖励提示音'],
[
'event',
'growth-result',
'物品合成',
'两件物品融合，随后响起明亮的完成提示音',
],
[
'event',
'growth-result',
'角色升级',
'能量快速上升，随后响起明亮的升级提示音',
],
[
'event',
'growth-result',
'任务完成',
'任务完成提示音，随后响起简短的奖励音符',
],
[
'event',
'growth-result',
'成就达成',
'明亮的成就提示音，随后响起简短的庆祝音符',
],
[
'event',
'growth-result',
'挑战胜利',
'明亮的胜利提示音，随后响起短暂的庆祝音符',
],
['event', 'growth-result', '挑战失败', '低沉的失败提示音'],
['event', 'character-combat', '角色跳跃', '角色轻盈跳起的声音'],
['event', 'character-combat', '角色落地', '角色落地时轻微的撞击声'],
['event', 'character-combat', '轻度受击', '轻微撞击的受击声'],
['event', 'character-combat', '重度受击', '沉重有力的撞击声'],
['event', 'character-combat', '攻击挥动', '武器快速挥过空气的呼啸声'],
['event', 'character-combat', '攻击命中', '武器击中目标的清晰撞击声'],
['event', 'character-combat', '格挡成功', '武器碰撞，随后被挡开的金属声'],
['event', 'character-combat', '物体破碎', '物体撞击地面后快速破碎的声音'],
['event', 'skill-status', '技能蓄力', '能量逐渐聚集的低沉嗡鸣声'],
[
'event',
'skill-status',
'魔法释放',
'柔和的魔法能量扩散，带有圆润空灵的闪光声',
],
[
'event',
'skill-status',
'治疗恢复',
'柔和能量扩散，带有温暖圆润的提示音',
],
['event', 'skill-status', '护盾生成', '能量向外展开，形成稳定的护盾声'],
[
'event',
'skill-status',
'瞬间移动',
'能量快速收缩，随后以短促的空气抽离声消失',
],
[
'event',
'skill-status',
'冰冻技能',
'冰霜能量扩散，随后响起清脆的冻结声',
],
['event', 'skill-status', '火焰技能', '火焰迅速喷发，带有短促的燃烧声'],
[
'event',
'skill-status',
'状态强化',
'能量逐渐上升，形成稳定明亮的提示音',
],
[
'event',
'mechanism-scene-interaction',
'门开启',
'门锁解除，随后厚重的木门缓慢打开',
],
[
'event',
'mechanism-scene-interaction',
'机关启动',
'机关解锁，随后齿轮开始转动',
],
[
'event',
'mechanism-scene-interaction',
'拉杆触发',
'拉杆被扳动，随后远处机关启动',
],
[
'event',
'mechanism-scene-interaction',
'石块移动',
'大型石块缓慢移动时低沉的摩擦声',
],
[
'event',
'mechanism-scene-interaction',
'传送门开启',
'能量旋转聚集，随后响起持续、空灵的传送门展开声',
],
[
'event',
'mechanism-scene-interaction',
'倒计时警告',
'逐渐加快的倒计时提示音',
],
['requirement', 'style-direction', '休闲可爱', '轻快可爱的卡通风格'],
['requirement', 'style-direction', '复古街机', '复古街机风格'],
['requirement', 'style-direction', '科幻电子', '干净的科幻电子音色'],
['requirement', 'style-direction', '奇幻魔法', '柔和梦幻的魔法音色'],
['requirement', 'style-direction', '写实自然', '自然真实的声音质感'],
['requirement', 'style-direction', '卡通夸张', '夸张鲜明的卡通风格'],
['requirement', 'feedback-requirement', '轻柔反馈', '轻柔克制'],
['requirement', 'feedback-requirement', '有力反馈', '更有力的撞击感'],
['requirement', 'feedback-requirement', '短促反馈', '短促的单次声音'],
['requirement', 'feedback-requirement', '两段递进', '由弱到强的两段变化'],
[
'requirement',
'feedback-requirement',
'干净突出',
'主体声音清晰，减少杂音',
],
[
'requirement',
'feedback-requirement',
'柔和不刺耳',
'圆润柔和，避免尖锐高频',
],
]);
});
it('canonicalizes and appends with the frozen Unicode punctuation rule', () => {
const preset = SOUND_EFFECT_PROMPT_PRESETS[40];
expect(appendSoundEffectPromptPreset('\u0085\u2003', preset)).toBe(
preset.prompt,
);
expect(appendSoundEffectPromptPreset('金币落地', preset)).toBe(
`金币落地，${preset.prompt}`,
);
expect(appendSoundEffectPromptPreset('金币落地！\u0085', preset)).toBe(
`金币落地！${preset.prompt}`,
);
});
it('allows duplicates and preserves overlong text without truncation', () => {
const preset = SOUND_EFFECT_PROMPT_PRESETS[0];
const once = appendSoundEffectPromptPreset('按钮', preset);
expect(appendSoundEffectPromptPreset(once, preset)).toBe(
`按钮，${preset.prompt}，${preset.prompt}`,
);
const overlong = '声'.repeat(2048);
const appended = appendSoundEffectPromptPreset(overlong, preset);
expect(appended).toBe(`${overlong}，${preset.prompt}`);
expect(Array.from(appended).length).toBeGreaterThan(2048);
});
});
src/components/image-editor/ImageCanvasSoundEffectPresetModel.ts
+423
View File
@@ -0,0 +1,423 @@
import { canonicalizeSoundEffectPrompt } from './ImageCanvasSoundEffectPromptModel';
export type SoundEffectPromptPresetCategory = 'event' | 'requirement';
export type SoundEffectPromptPresetGroup =
| 'ui-operation'
| 'pickup-reward'
| 'growth-result'
| 'character-combat'
| 'skill-status'
| 'mechanism-scene-interaction'
| 'style-direction'
| 'feedback-requirement';
export type SoundEffectPromptPreset = {
id: string;
category: SoundEffectPromptPresetCategory;
group: SoundEffectPromptPresetGroup;
label: string;
prompt: string;
};
const UNICODE_PUNCTUATION_CODE_POINT = /^\p{Punctuation}$/u;
const SOUND_EFFECT_PRESET_SEPARATOR = '，';
export const SOUND_EFFECT_PROMPT_PRESET_GROUPS = [
'ui-operation',
'pickup-reward',
'growth-result',
'character-combat',
'skill-status',
'mechanism-scene-interaction',
'style-direction',
'feedback-requirement',
] as const satisfies readonly SoundEffectPromptPresetGroup[];
/** 权威设计「SFX 生成优化 V2.0」冻结的 40 个事件预设与 12 个补充要求。 */
export const SOUND_EFFECT_PROMPT_PRESETS = [
{
id: 'button-tap',
category: 'event',
group: 'ui-operation',
label: '轻触按钮',
prompt: '柔和的按钮点击声',
},
{
id: 'operation-confirm',
category: 'event',
group: 'ui-operation',
label: '确认操作',
prompt: '明亮的确认提示音',
},
{
id: 'back-cancel',
category: 'event',
group: 'ui-operation',
label: '返回取消',
prompt: '轻微下降的取消提示音',
},
{
id: 'page-transition',
category: 'event',
group: 'ui-operation',
label: '页面切换',
prompt: '快速掠过的界面切换声',
},
{
id: 'notification',
category: 'event',
group: 'ui-operation',
label: '通知提醒',
prompt: '清晰柔和的通知提示音',
},
{
id: 'operation-error',
category: 'event',
group: 'ui-operation',
label: '操作错误',
prompt: '短促克制的错误提示音',
},
{
id: 'coin-pickup',
category: 'event',
group: 'pickup-reward',
label: '金币拾取',
prompt: '金币拾取时清脆的金属叮当声',
},
{
id: 'item-pickup',
category: 'event',
group: 'pickup-reward',
label: '道具拾取',
prompt: '拾取道具时轻快的提示音',
},
{
id: 'reward-received',
category: 'event',
group: 'pickup-reward',
label: '获得奖励',
prompt: '奖励出现时明亮的提示音',
},
{
id: 'chest-open',
category: 'event',
group: 'pickup-reward',
label: '宝箱开启',
prompt: '金属锁扣弹开，随后响起明亮的奖励提示音',
},
{
id: 'content-unlock',
category: 'event',
group: 'pickup-reward',
label: '解锁内容',
prompt: '锁定状态解除，随后响起解锁提示音',
},
{
id: 'rare-drop',
category: 'event',
group: 'pickup-reward',
label: '稀有掉落',
prompt: '稀有物品出现时闪耀的奖励提示音',
},
{
id: 'item-craft',
category: 'event',
group: 'growth-result',
label: '物品合成',
prompt: '两件物品融合，随后响起明亮的完成提示音',
},
{
id: 'character-level-up',
category: 'event',
group: 'growth-result',
label: '角色升级',
prompt: '能量快速上升，随后响起明亮的升级提示音',
},
{
id: 'quest-complete',
category: 'event',
group: 'growth-result',
label: '任务完成',
prompt: '任务完成提示音，随后响起简短的奖励音符',
},
{
id: 'achievement-unlocked',
category: 'event',
group: 'growth-result',
label: '成就达成',
prompt: '明亮的成就提示音，随后响起简短的庆祝音符',
},
{
id: 'challenge-victory',
category: 'event',
group: 'growth-result',
label: '挑战胜利',
prompt: '明亮的胜利提示音，随后响起短暂的庆祝音符',
},
{
id: 'challenge-defeat',
category: 'event',
group: 'growth-result',
label: '挑战失败',
prompt: '低沉的失败提示音',
},
{
id: 'character-jump',
category: 'event',
group: 'character-combat',
label: '角色跳跃',
prompt: '角色轻盈跳起的声音',
},
{
id: 'character-land',
category: 'event',
group: 'character-combat',
label: '角色落地',
prompt: '角色落地时轻微的撞击声',
},
{
id: 'light-hit',
category: 'event',
group: 'character-combat',
label: '轻度受击',
prompt: '轻微撞击的受击声',
},
{
id: 'heavy-hit',
category: 'event',
group: 'character-combat',
label: '重度受击',
prompt: '沉重有力的撞击声',
},
{
id: 'attack-swing',
category: 'event',
group: 'character-combat',
label: '攻击挥动',
prompt: '武器快速挥过空气的呼啸声',
},
{
id: 'attack-hit',
category: 'event',
group: 'character-combat',
label: '攻击命中',
prompt: '武器击中目标的清晰撞击声',
},
{
id: 'block-success',
category: 'event',
group: 'character-combat',
label: '格挡成功',
prompt: '武器碰撞，随后被挡开的金属声',
},
{
id: 'object-break',
category: 'event',
group: 'character-combat',
label: '物体破碎',
prompt: '物体撞击地面后快速破碎的声音',
},
{
id: 'skill-charge',
category: 'event',
group: 'skill-status',
label: '技能蓄力',
prompt: '能量逐渐聚集的低沉嗡鸣声',
},
{
id: 'magic-cast',
category: 'event',
group: 'skill-status',
label: '魔法释放',
prompt: '柔和的魔法能量扩散，带有圆润空灵的闪光声',
},
{
id: 'healing',
category: 'event',
group: 'skill-status',
label: '治疗恢复',
prompt: '柔和能量扩散，带有温暖圆润的提示音',
},
{
id: 'shield-create',
category: 'event',
group: 'skill-status',
label: '护盾生成',
prompt: '能量向外展开，形成稳定的护盾声',
},
{
id: 'teleport',
category: 'event',
group: 'skill-status',
label: '瞬间移动',
prompt: '能量快速收缩，随后以短促的空气抽离声消失',
},
{
id: 'ice-skill',
category: 'event',
group: 'skill-status',
label: '冰冻技能',
prompt: '冰霜能量扩散，随后响起清脆的冻结声',
},
{
id: 'fire-skill',
category: 'event',
group: 'skill-status',
label: '火焰技能',
prompt: '火焰迅速喷发，带有短促的燃烧声',
},
{
id: 'status-buff',
category: 'event',
group: 'skill-status',
label: '状态强化',
prompt: '能量逐渐上升，形成稳定明亮的提示音',
},
{
id: 'door-open',
category: 'event',
group: 'mechanism-scene-interaction',
label: '门开启',
prompt: '门锁解除，随后厚重的木门缓慢打开',
},
{
id: 'mechanism-start',
category: 'event',
group: 'mechanism-scene-interaction',
label: '机关启动',
prompt: '机关解锁，随后齿轮开始转动',
},
{
id: 'lever-trigger',
category: 'event',
group: 'mechanism-scene-interaction',
label: '拉杆触发',
prompt: '拉杆被扳动，随后远处机关启动',
},
{
id: 'stone-move',
category: 'event',
group: 'mechanism-scene-interaction',
label: '石块移动',
prompt: '大型石块缓慢移动时低沉的摩擦声',
},
{
id: 'portal-open',
category: 'event',
group: 'mechanism-scene-interaction',
label: '传送门开启',
prompt: '能量旋转聚集，随后响起持续、空灵的传送门展开声',
},
{
id: 'countdown-warning',
category: 'event',
group: 'mechanism-scene-interaction',
label: '倒计时警告',
prompt: '逐渐加快的倒计时提示音',
},
{
id: 'casual-cute',
category: 'requirement',
group: 'style-direction',
label: '休闲可爱',
prompt: '轻快可爱的卡通风格',
},
{
id: 'retro-arcade',
category: 'requirement',
group: 'style-direction',
label: '复古街机',
prompt: '复古街机风格',
},
{
id: 'sci-fi-electronic',
category: 'requirement',
group: 'style-direction',
label: '科幻电子',
prompt: '干净的科幻电子音色',
},
{
id: 'fantasy-magic',
category: 'requirement',
group: 'style-direction',
label: '奇幻魔法',
prompt: '柔和梦幻的魔法音色',
},
{
id: 'realistic-natural',
category: 'requirement',
group: 'style-direction',
label: '写实自然',
prompt: '自然真实的声音质感',
},
{
id: 'cartoon-exaggerated',
category: 'requirement',
group: 'style-direction',
label: '卡通夸张',
prompt: '夸张鲜明的卡通风格',
},
{
id: 'gentle-feedback',
category: 'requirement',
group: 'feedback-requirement',
label: '轻柔反馈',
prompt: '轻柔克制',
},
{
id: 'strong-feedback',
category: 'requirement',
group: 'feedback-requirement',
label: '有力反馈',
prompt: '更有力的撞击感',
},
{
id: 'short-feedback',
category: 'requirement',
group: 'feedback-requirement',
label: '短促反馈',
prompt: '短促的单次声音',
},
{
id: 'two-stage-rise',
category: 'requirement',
group: 'feedback-requirement',
label: '两段递进',
prompt: '由弱到强的两段变化',
},
{
id: 'clean-prominent',
category: 'requirement',
group: 'feedback-requirement',
label: '干净突出',
prompt: '主体声音清晰，减少杂音',
},
{
id: 'soft-not-harsh',
category: 'requirement',
group: 'feedback-requirement',
label: '柔和不刺耳',
prompt: '圆润柔和，避免尖锐高频',
},
] as const satisfies readonly SoundEffectPromptPreset[];
function readLastCodePoint(value: string) {
const codePoints = Array.from(value);
return codePoints[codePoints.length - 1];
}
export function appendSoundEffectPromptPreset(
currentPrompt: string,
preset: SoundEffectPromptPreset,
) {
const canonicalPrompt = canonicalizeSoundEffectPrompt(currentPrompt);
if (!canonicalPrompt) {
return preset.prompt;
}
const lastCodePoint = readLastCodePoint(canonicalPrompt);
if (lastCodePoint && UNICODE_PUNCTUATION_CODE_POINT.test(lastCodePoint)) {
return `${canonicalPrompt}${preset.prompt}`;
}
return `${canonicalPrompt}${SOUND_EFFECT_PRESET_SEPARATOR}${preset.prompt}`;
}
src/components/image-editor/ImageCanvasSoundEffectPromptModel.test.ts
+60
View File
@@ -0,0 +1,60 @@
import { describe, expect, it } from 'vitest';
import soundEffectPromptCanonicalizationCases from '../../../packages/shared/test-fixtures/sound-effect-prompt-canonicalization.json';
import {
canonicalizeSoundEffectPrompt,
countSoundEffectPromptCodePoints,
SOUND_EFFECT_PROMPT_MAX_CODE_POINTS,
validateSoundEffectPrompt,
} from './ImageCanvasSoundEffectPromptModel';
describe('ImageCanvasSoundEffectPromptModel', () => {
it('matches the shared Unicode canonicalization fixtures', () => {
for (const fixture of soundEffectPromptCanonicalizationCases) {
expect(canonicalizeSoundEffectPrompt(fixture.input), fixture.name).toBe(
fixture.prompt,
);
expect(
countSoundEffectPromptCodePoints(fixture.input),
fixture.name,
).toBe(fixture.charCount);
expect(
canonicalizeSoundEffectPrompt(
canonicalizeSoundEffectPrompt(fixture.input),
),
fixture.name,
).toBe(fixture.prompt);
}
});
it('accepts 2048 code points and rejects 2049 without truncating', () => {
const maximum = `\u0085${'😀'.repeat(
SOUND_EFFECT_PROMPT_MAX_CODE_POINTS,
)}\u2003`;
const accepted = validateSoundEffectPrompt(maximum);
expect(accepted).toEqual({
ok: true,
prompt: '😀'.repeat(SOUND_EFFECT_PROMPT_MAX_CODE_POINTS),
charCount: SOUND_EFFECT_PROMPT_MAX_CODE_POINTS,
});
const overlong = '声'.repeat(SOUND_EFFECT_PROMPT_MAX_CODE_POINTS + 1);
expect(validateSoundEffectPrompt(overlong)).toEqual({
ok: false,
prompt: overlong,
charCount: SOUND_EFFECT_PROMPT_MAX_CODE_POINTS + 1,
reason: 'too-long',
});
});
it('rejects empty canonical prompts without adding a default description', () => {
for (const prompt of ['', ' \t\r\n', '\u0085\u2003\u00a0']) {
expect(validateSoundEffectPrompt(prompt)).toEqual({
ok: false,
prompt: '',
charCount: 0,
reason: 'empty',
});
}
});
});
src/components/image-editor/ImageCanvasSoundEffectPromptModel.ts
+55
View File
@@ -0,0 +1,55 @@
export const SOUND_EFFECT_PROMPT_MAX_CODE_POINTS = 2048;
const LEADING_UNICODE_WHITE_SPACE = /^\p{White_Space}+/u;
const TRAILING_UNICODE_WHITE_SPACE = /\p{White_Space}+$/u;
export type SoundEffectPromptValidationResult =
| {
ok: true;
prompt: string;
charCount: number;
}
| {
ok: false;
prompt: string;
charCount: number;
reason: 'empty' | 'too-long';
};
/**
* 只删除首尾 Unicode `White_Space`。内部空白、组合字符、ZWJ、U+200B、U+FEFF
* 及其它 code point 均原样保留；不能替换为语义不同的原生 `trim()`。
*/
export function canonicalizeSoundEffectPrompt(prompt: string) {
return prompt
.replace(LEADING_UNICODE_WHITE_SPACE, '')
.replace(TRAILING_UNICODE_WHITE_SPACE, '');
}
export function countSoundEffectPromptCodePoints(prompt: string) {
return Array.from(canonicalizeSoundEffectPrompt(prompt)).length;
}
export function validateSoundEffectPrompt(
prompt: string,
): SoundEffectPromptValidationResult {
const canonicalPrompt = canonicalizeSoundEffectPrompt(prompt);
const charCount = Array.from(canonicalPrompt).length;
if (charCount === 0) {
return {
ok: false,
prompt: canonicalPrompt,
charCount,
reason: 'empty',
};
}
if (charCount > SOUND_EFFECT_PROMPT_MAX_CODE_POINTS) {
return {
ok: false,
prompt: canonicalPrompt,
charCount,
reason: 'too-long',
};
}
return { ok: true, prompt: canonicalPrompt, charCount };
}
src/services/image-editor/editorProjectClient.test.ts
+77 -15
View File
@@ -1752,9 +1752,9 @@ describe('editorProjectClient', () => {
height: 120,
sourceType: 'generated',
prompt: '金币掉落叮当声',
actualPrompt: '金币掉落叮当声',
model: 'audio1.0',
provider: 'VectorEngine',
actualPrompt: 'A bright coin pickup chime',
model: 'eleven_text_to_sound_v2',
provider: 'elevenlabs',
taskId: 'sound-task-1',
audioKind: 'sound-effect',
asset: {
@@ -1771,8 +1771,9 @@ describe('editorProjectClient', () => {
const result = await generateEditorSoundEffect({
prompt: '金币掉落叮当声',
model: 'audio1.0',
duration: 7,
model: 'eleven_text_to_sound_v2',
duration: 7.5,
loop: true,
assetFolderId: 'project',
assetLabel: '游戏音效 1',
});
@@ -1786,8 +1787,9 @@ describe('editorProjectClient', () => {
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
prompt: '金币掉落叮当声',
model: 'audio1.0',
duration: 7,
model: 'eleven_text_to_sound_v2',
duration: 7.5,
loop: true,
assetFolderId: 'project',
assetLabel: '游戏音效 1',
}),
@@ -1800,24 +1802,23 @@ describe('editorProjectClient', () => {
);
});
it('sends editor sound effect duration to Vidu instead of legacy type and tempo', async () => {
it('canonicalizes omitted duration to null and omitted loop to false', async () => {
requestJsonMock.mockResolvedValueOnce({
audioSrc: '/generated-character-drafts/editor-audios/sfx-null.mp3',
width: 420,
height: 120,
sourceType: 'generated',
prompt: '按钮确认短促音',
actualPrompt: '按钮确认短促音',
model: 'audio1.0',
provider: 'VectorEngine',
actualPrompt: 'A short confirmation click',
model: 'eleven_text_to_sound_v2',
provider: 'elevenlabs',
taskId: 'sound-task-null',
audioKind: 'sound-effect',
});
await generateEditorSoundEffect({
prompt: '按钮确认短促音',
model: 'audio1.0',
duration: 5,
model: 'eleven_text_to_sound_v2',
});
expect(requestJsonMock).toHaveBeenCalledWith(
@@ -1825,8 +1826,9 @@ describe('editorProjectClient', () => {
expect.objectContaining({
body: JSON.stringify({
prompt: '按钮确认短促音',
model: 'audio1.0',
duration: 5,
model: 'eleven_text_to_sound_v2',
duration: null,
loop: false,
}),
}),
'生成游戏音效失败',
@@ -1834,6 +1836,66 @@ describe('editorProjectClient', () => {
);
});
it('accepts the 0.5 and 30 second sound effect duration boundaries', async () => {
requestJsonMock.mockResolvedValue({});
for (const duration of [0.5, 30]) {
await generateEditorSoundEffect({
prompt: '按钮确认短促音',
model: 'eleven_text_to_sound_v2',
duration,
});
}
expect(requestJsonMock).toHaveBeenNthCalledWith(
1,
'/api/editor/audios/sound-effects/generations',
expect.objectContaining({
body: JSON.stringify({
prompt: '按钮确认短促音',
model: 'eleven_text_to_sound_v2',
duration: 0.5,
loop: false,
}),
}),
'生成游戏音效失败',
expect.any(Object),
);
expect(requestJsonMock).toHaveBeenNthCalledWith(
2,
'/api/editor/audios/sound-effects/generations',
expect.objectContaining({
body: JSON.stringify({
prompt: '按钮确认短促音',
model: 'eleven_text_to_sound_v2',
duration: 30,
loop: false,
}),
}),
'生成游戏音效失败',
expect.any(Object),
);
});
it('rejects invalid sound effect durations before the request', async () => {
for (const duration of [
0.49,
30.01,
Number.NaN,
Number.POSITIVE_INFINITY,
]) {
await expect(
generateEditorSoundEffect({
prompt: '按钮确认短促音',
model: 'eleven_text_to_sound_v2',
duration,
}),
).rejects.toThrow('游戏音效时长必须在 0.5-30 秒之间');
}
expect(requestJsonMock).not.toHaveBeenCalled();
});
it('generates editor background music through the backend BFF', async () => {
const representativeComplexPromptCase =
backgroundMusicPromptCanonicalizationCases.find(
src/services/image-editor/editorProjectClient.ts
+29 -14
View File
@@ -1,6 +1,10 @@
import type {
BackgroundMusicPromptAssistRequest,
BackgroundMusicPromptAssistResponse,
import {
type BackgroundMusicPromptAssistRequest,
type BackgroundMusicPromptAssistResponse,
type EditorSoundEffectGenerationMetadataV2,
type EditorSoundEffectGenerationRequest,
SOUND_EFFECT_DURATION_MAX_SECONDS,
SOUND_EFFECT_DURATION_MIN_SECONDS,
} from '../../../packages/shared/src/contracts/editorAudio';
import type { ExternalGenerationJobStatusRecord } from '../../../packages/shared/src/contracts/externalGeneration';
import { requestJson } from '../apiClient';
@@ -125,6 +129,7 @@ export type EditorAssetGenerationInputReference = {
export type EditorAssetGenerationInputs = {
fields: EditorAssetGenerationInputField[];
references: EditorAssetGenerationInputReference[];
soundEffect?: EditorSoundEffectGenerationMetadataV2;
[key: string]: unknown;
};
@@ -547,16 +552,14 @@ export type EditorVideoGenerationResult = {
queueState?: ExternalGenerationJobStatusRecord | null;
};
export type EditorSoundEffectGenerationInput = {
prompt: string;
model: 'audio1.0';
duration: number;
projectId?: string | null;
canvasCompletion?: EditorCanvasGenerationCompletionInput | null;
generationInputs?: EditorAssetGenerationInputs | null;
assetFolderId?: string | null;
assetLabel?: string | null;
};
export type EditorSoundEffectGenerationInput =
EditorSoundEffectGenerationRequest & {
projectId?: string | null;
canvasCompletion?: EditorCanvasGenerationCompletionInput | null;
generationInputs?: EditorAssetGenerationInputs | null;
assetFolderId?: string | null;
assetLabel?: string | null;
};
export type EditorBackgroundMusicGenerationInput = {
gptDescriptionPrompt: string;
@@ -582,10 +585,12 @@ export type EditorAudioGenerationResult = {
prompt: string;
actualPrompt?: string | null;
model: string;
provider: string;
taskId: string;
priceMudPoints: number;
audioKind: 'sound-effect' | 'background-music';
durationSeconds?: number | null;
loop?: boolean | null;
resource?: EditorProjectResourceSnapshot | null;
asset?: EditorAssetSnapshot | null;
project?: EditorProjectSnapshot | null;
@@ -1353,12 +1358,22 @@ export async function generateEditorVideo(input: EditorVideoGenerationInput) {
export async function generateEditorSoundEffect(
input: EditorSoundEffectGenerationInput,
) {
const duration = input.duration ?? null;
if (
duration !== null &&
(!Number.isFinite(duration) ||
duration < SOUND_EFFECT_DURATION_MIN_SECONDS ||
duration > SOUND_EFFECT_DURATION_MAX_SECONDS)
) {
throw new Error('游戏音效时长必须在 0.5-30 秒之间');
}
return requestJson<EditorAudioGenerationResponse>(
EDITOR_SOUND_EFFECT_GENERATION_API,
jsonRequest('POST', {
prompt: input.prompt,
model: input.model,
duration: input.duration,
duration,
loop: input.loop ?? false,
...(input.projectId ? { projectId: input.projectId } : {}),
...(input.canvasCompletion
? { canvasCompletion: input.canvasCompletion }