diff --git a/.codex/skills/genarrative-external-editor-api/references/api-operations.md b/.codex/skills/genarrative-external-editor-api/references/api-operations.md index 1597a2a12..193626892 100644 --- a/.codex/skills/genarrative-external-editor-api/references/api-operations.md +++ b/.codex/skills/genarrative-external-editor-api/references/api-operations.md @@ -48,7 +48,7 @@ Every generation row requires a stable `Idempotency-Key` header and returns HTTP | UI asset extraction | `/api/external/v1/editor/ui-designs/assets/extractions` | `sourceImageSrc`, `aspectRatio`, `imageSize` | `screenColor`, `model`, `referenceImageSrcs`, `projectId`, `assetFolderId`, `spritesheetLabel`, `canvasCompletion` | | Character animation | `/api/external/v1/editor/character-animations/generations` | `sourceLayerId`, `sourceImageSrc`, `sourceWidth`, `sourceHeight`, `promptText`, `resolution`, `ratio`, `frameCount`, `durationSeconds`, `model` | `projectId`, `sourceResourceId`, `assetFolderId`, `assetLabel`, `canvasCompletion` | | Video generation | `/api/external/v1/editor/videos/generations` | `prompt`, `model`, `aspectRatio`, `durationSeconds`, `resolution`, `mode`, `sound` | `referenceImageSrcs`, `referenceVideoSrcs`, `referenceAudioSrcs`, `webSearchEnabled`, `projectId`, `assetFolderId`, `assetLabel`, `canvasCompletion` | -| Sound effect | `/api/external/v1/editor/audios/sound-effects/generations` | `prompt`, `duration` | `model`, `projectId`, `assetFolderId`, `assetLabel`, `canvasCompletion`, `generationInputs` | +| Sound effect | `/api/external/v1/editor/audios/sound-effects/generations` | `prompt` | `model`, `duration`, `loop`, `projectId`, `assetFolderId`, `assetLabel`, `canvasCompletion`, `generationInputs` | | Background music | `/api/external/v1/editor/audios/background-music/generations` | `gptDescriptionPrompt`, `makeInstrumental` | `projectId`, `assetFolderId`, `assetLabel`, `canvasCompletion`, `generationInputs` | Poll all eight through: @@ -95,6 +95,7 @@ Use OpenAPI as the final authority; these common values are a routing aid: - Video `aspectRatio`: `16:9`, `9:16`, `1:1`, `4:3`, `3:4`, `21:9`. - Video `resolution`: `480p`, `720p`, `1080p`; `mode`: `std`; `sound`: `on` or `off`. - Character animation uses `model: "seedance2.0-fast"`; `resolution`: `480p` or `720p`; `frameCount`: `32`, `40`, or `48`; `durationSeconds`: `4`, `5`, or `6`; `ratio`: `same`, `1:1`, `4:3`, `16:9`, `9:16`, or `3:4`. +- Sound effect uses canonical model `eleven_text_to_sound_v2`; omit `duration` or send `null` for automatic duration, otherwise send a finite `0.5-30` number. `loop` defaults to `false` and remains independent from Prompt text. - UI extraction uses `aspectRatio: "1:1"`; use `imageSize: "1K"` for normal/small extraction and `2K` for dense designs. Do not hard-code this list as a replacement client schema. In particular, the top-level image `style` field is intentionally extensible; see `requests-and-outputs.md` for its fallback behavior. diff --git a/.codex/skills/genarrative-external-editor-api/references/requests-and-outputs.md b/.codex/skills/genarrative-external-editor-api/references/requests-and-outputs.md index 1f3d04e3c..c37c77cc2 100644 --- a/.codex/skills/genarrative-external-editor-api/references/requests-and-outputs.md +++ b/.codex/skills/genarrative-external-editor-api/references/requests-and-outputs.md @@ -193,6 +193,7 @@ Do not guess dimensions or pass a temporary signed read URL. See `authentication The completed `result` may contain stable artifact fields such as: - `objectKey`, media type, dimensions, or task ID. +- Sound-effect `durationSeconds` is the probed MP3 duration and `loop` is the frozen request boolean; neither is inferred from Prompt text. - `resource`, `resourceId`, or equivalent canvas reference. - `asset`, `assetId`, or equivalent library reference. - `spritesheetResource`, `spritesheetAsset`, and stable spritesheet metadata. diff --git a/.codex/skills/genarrative-external-editor-api/scripts/genarrative_external_api.py b/.codex/skills/genarrative-external-editor-api/scripts/genarrative_external_api.py index bce78e20e..8cdb21e49 100644 --- a/.codex/skills/genarrative-external-editor-api/scripts/genarrative_external_api.py +++ b/.codex/skills/genarrative-external-editor-api/scripts/genarrative_external_api.py @@ -649,13 +649,19 @@ class GenarrativeExternalClient: idempotency_key=idempotency_key, ) - def generate_sound_effect(self, prompt: str, duration: int, **fields: Any) -> Any: + def generate_sound_effect( + self, + prompt: str, + duration: float | None = None, + loop: bool = False, + **fields: Any, + ) -> Any: self._apply_canvas_session_fields(fields, prompt, 360, 120) prompt = self._apply_art_spec(fields, prompt) idempotency_key = fields.pop("idempotencyKey", None) return self.submit_and_wait_generation( "/api/external/v1/editor/audios/sound-effects/generations", - {"prompt": prompt, "duration": duration, **fields}, + {"prompt": prompt, "duration": duration, "loop": loop, **fields}, idempotency_key=idempotency_key, ) diff --git a/docs/openapi/genarrative-external-v1.openapi.json b/docs/openapi/genarrative-external-v1.openapi.json index 659c13a6f..76ffb4e23 100644 --- a/docs/openapi/genarrative-external-v1.openapi.json +++ b/docs/openapi/genarrative-external-v1.openapi.json @@ -4449,25 +4449,41 @@ "EditorSoundEffectGenerationRequest": { "type": "object", "required": [ - "prompt", - "duration" + "prompt" ], "properties": { "prompt": { "type": "string", - "minLength": 1 + "minLength": 1, + "maxLength": 2048, + "description": "用户原始语言音效描述。服务端只删除首尾 Unicode White_Space,并按 Unicode code point 校验 1-2048。" }, "model": { "type": [ "string", "null" ], - "default": "audio1.0" + "default": "eleven_text_to_sound_v2", + "description": "省略、null、空串、纯 Unicode White_Space 或首尾空白包围的 eleven_text_to_sound_v2 均 canonicalize 为 eleven_text_to_sound_v2;audio1.0 和其它非空值返回 400。" }, "duration": { - "type": "integer", - "minimum": 2, - "maximum": 10 + "anyOf": [ + { + "type": "number", + "minimum": 0.5, + "maximum": 30 + }, + { + "type": "null" + } + ], + "default": null, + "description": "null 或省略表示自动时长;有限数值表示手动时长。服务端不按 UI 0.1 秒步进取整。" + }, + "loop": { + "type": "boolean", + "default": false, + "description": "独立 Loop 参数;服务端不从 Prompt 推断、同步或校验。" }, "projectId": { "type": [ @@ -4629,6 +4645,22 @@ "background-music" ] }, + "durationSeconds": { + "type": [ + "number", + "null" + ], + "exclusiveMinimum": 0, + "maximum": 600, + "description": "SFX V2 为 MP3 探测所得实际时长;不是请求时长。BGM 或历史结果可省略。" + }, + "loop": { + "type": [ + "boolean", + "null" + ], + "description": "SFX V2 返回冻结并发送给 provider 的 Loop;BGM 或历史结果可省略。" + }, "project": { "anyOf": [ { diff --git a/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md b/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md index fe7a6592b..62fa519cc 100644 --- a/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md +++ b/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md @@ -2,7 +2,7 @@ 日期:`2026-08-06` -状态:`T1、T2、T3、T4 已完成,可以继续进入 T5;T2 翻译 service、T3 ElevenLabs adapter 与 T4 前端冻结参数均待 T5 正式接线,当前仍不是可发布切点` +状态:`T1–T5 已完成,可以继续进入 T6;T6 测试、灰度与发布门禁完成前仍不可发布` 开发分支:`feat/sound_opt` @@ -177,6 +177,12 @@ T4 没有修改 Worker、provider 调用、正式请求的 nullable duration / L - 实现新模型定价键、历史 Vidu 只读 / 重绘兼容、中英 Prompt + Loop 详情和真实时长。 - 同批更新 External v1 Rust DTO / handler / OpenAPI / Idempotency-Key 重放 / compact result;任一字段不一致时 T5 不完成。 +实施记录(`2026-08-07`):T5 已完成。站内与 External SFX 请求在定价、预扣和 enqueue 前统一 canonicalize 为固定模型、canonical userPrompt、nullable 小数时长与 Loop;正式浏览器 POST 不再配置 unsafe retry,queue payload 不包含提前翻译的 actualPrompt。Worker 在既有冻结计费上下文内执行 Luna 英文化、单次 ElevenLabs POST、MP3 校验与实际时长探测、OSS、项目资源 / 账号素材 / 画布完成态写回,并使用 queue job ID 或 inline 预生成的平台 ID 作为 Task ID。服务端重建 `generation_inputs_json`,客户端自报的实际英文 Prompt、实际时长、模型与 Loop 不进入权威 metadata。 + +新定价键 `eleven_text_to_sound_v2` 已加入默认 JSON、api-server 与 SpacetimeDB 值校验,旧 `audio1.0` 键继续保留;历史 SpacetimeDB 定价快照仅缺新键时由受控本地定价补齐读取,下一次后台保存写回完整矩阵,不修改 schema。详情展示中英 Prompt、实际时长、Loop、生成模型与完整平台 Task ID;SFX V2 重绘恢复 userPrompt、duration mode / requested duration 和 Loop,自动时长不会把实际输出时长误作下一次手动值。 + +External v1 Rust handler、共享 DTO、OpenAPI、compact result 与 Agent Skill 已同步 nullable `0.5-30` 时长、Loop、固定模型和实际 `durationSeconds`;接受的 model 形态生成同一 canonical queue payload,旧 / 未知模型在 enqueue 前返回 `400`。External compact 继续隐藏 provider 与 Prompt,只保留稳定资源引用、实际时长和 Loop。T5 定向 Rust、TypeScript、External/OpenAPI、Agent、定价与 SpacetimeDB WASM build 已通过,未执行真实 LLM、ElevenLabs 或其它付费请求;完整失败矩阵、端到端与发布 smoke 继续归属 T6。 + ### T6:测试、文档、灰度和发布门禁 - 汇总 T1–T5 分层测试,增加 mock LLM + mock ElevenLabs + mock OSS 失败矩阵、端到端等值、刷新 / 重绘、计费退款、无重试、External 幂等和 BGM 回归。 diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 750acd5eb..c3d9e9b3d 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -127,6 +127,7 @@ - 影响范围:AI 游戏创作 `runtime_driver/task_start.rs`、`task_queue.rs`、自主构建 continuation 合同、Supervisor 进度卡与相应 Rust/AppSurface 回归;不改变 manifest DAG、Agent catalog、Provider 路由或项目产物合同。 - 验证方式:不预占 child locks,真实一次调度三项首波任务,并在有界时间内证明每个逻辑 Run 至少写入 running/`turn.started`;重复调度不得新增逻辑 Run。前端固定时钟覆盖正常运行、子 Agent 新活动、疑似停滞、各类合法等待与 terminal 冻结。 - 关联文档:`docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md`、`docs/project-memory/shared-memory/pitfalls.md`。 + ## 2026-07-31 图集切片按需编码并批量确认持久化 - 背景:`2026-07-29 图集切片必须受前置容量和有界 CPU 保护` 收口了连通域数量与 CPU 并发,但切片仍在一次循环里全部裁剪并编码,最多 64 份 PNG 字节连同整张 RGBA 同时驻留内存;持久化又按切片逐个调用 procedure,N 片至少 2N 次写入外加一次 cohort 完成,任一片失败都会留下已确认的部分记录。手动拆分入口另有一处重复鉴权:`get_editor_project` 已经取回并定位了来源资源,随后仍走 `parse_editor_reference_image` 按注册 ID 再解析一次,触发全账号项目与素材库扫描。 @@ -139,6 +140,7 @@ - 验证方式:`platform-image` 覆盖 prepare 不编码且 `Send + Sync`、并发编码多个 index 结果不变、累计裁剪像素在编码前拒绝;`api-server` 覆盖切片记录 ID 稳定且按 owner / index 分区、自动路径保留处理超时告警码、上传超时释放内存许可;`spacetime-module` 覆盖批次校验的完整 cohort、重复 objectKey、来源资源同 owner 同 project、部分 cohort 拒绝与重放只在内容一致时复用。 - 关联文档:`docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md`、`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`、本文件 `2026-07-29 图集切片必须受前置容量和有界 CPU 保护`。 - 补记说明:本条为事后补写,记录提交 `cf1a02312` 已落地的行为,不改变其任何决策。 + ## 2026-07-31 AI 游戏创作资源依赖图采用 Rust 只读拓扑与前端派生 SVG > 状态:其中资源卡 Pointer Move 拖动预览与局部更新验收已由 2026-08-03 mentor 最新决定暂缓;只读拓扑、SVG 派生展示、搜索与选择高亮合同继续生效。 @@ -6047,6 +6049,7 @@ - 占位删除与重试:completion 必须读取当前权威 dialog;若删除已先持久化,只跳过画布 layer / dialog 写回,不得使用请求中的旧 placeholder 复活图层,已经成功持久化的 project resource / 账号素材允许保留。若回包时本地占位已删除,前端不得应用完成快照或写历史;现有布局 CAS 没有 deletion tombstone,因此 completion 先提交、删除保存后冲突的极端竞态仍按权威快照收口,绝对“删除意图胜出”留待 targeted delete / tombstone 方案。该路由是 unsafe POST,客户端不得配置 `EDITOR_REQUEST_RETRY_OPTIONS`;请求字节可能已发出后不因 transport 异常或 `408 / 425 / 429 / 502 / 503 / 504` 自动重放,Bearer 中间件在 handler 前拒绝请求后的既有认证恢复继续保留。结果未知时先 GET 权威项目 / 素材快照,由用户显式决定是否再次执行。 - 历史边界:成功加入画布时写一条 `perfect-pixel` 历史,中文标签为“完美像素”,并纳入新增结果保护;撤销不得让派生 PNG 消失。像素处理失败或 completion 因占位删除未落画布时不写该历史。 - 关联文档:`docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md`、`docs/【图片画布】撤销范围与操作提示方案-2026-07-17.md`、`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`、`docs/【编辑器】图片画布结构化持久化与迁移回滚方案-2026-07-19.md`。 + ## 2026-07-31 game-chat 每条输出入聊天、试玩后收束与平台图集引用 - 背景:game-chat 的 ready response 之前只作为 transient stream 展示,专业 Agent 的 `final-reply` 只进入各自私有 conversation,刷新或事件 / 轮询重放时项目聊天可能丢失这些输出;自主构建完成后仍可能继续进入发布任务;配置 External Editor API 时,原型 HTML 也可能不实际使用平台生成的 Canvas 美术资源。 @@ -6295,6 +6298,7 @@ - 影响范围:`/editor/canvas` 的 `audio-background-music` 面板撤销按钮与其定向测试;不改变单层交换快照语义、canonicalization、提交锁、Suno 契约或后端 Prompt 助手,也不修改状态模型字段,`temporaryPromptSnapshot` 已在公开 dialog 状态中且只在 `completing` / `simplifying` 期间非空。本条不适用于 SFX,V1.0 不改动 SFX 的一键优化与撤销行为。 - 验证方式:按矩阵逐行覆盖初始隐藏、首次与再次 AI 处理期间显示并禁用、成功启用、失败隐藏、手动编辑后仍启用、点击预设隐藏、`submitting` 有无快照的两种表现、解除锁定后恢复,以及连续撤销互换保持启用;并断言处理期间按钮仍在可访问树中且为真实禁用态。 - 关联文档:`docs/【编辑器】画板音乐生成入口设计-2026-06-18.md`。 + ## 2026-08-03 完美像素对账判据改看 dialog 收口状态,网关合成响应归入未知结果 - 缺陷一(对账把真成功判成失败):对账用「同 ID 的 generation-dialog 是否还在权威快照里」判定成败,而服务端成功回填时**保留**该 dialog 并就地改写——`apply_editor_canvas_generation_items` 置 `status: "idle"`、`composerOpen: false`、写入 `generatedLayerId`、清掉 `errorMessage`,该行为另有服务端测试断言 `dialog["generatedLayerId"]` 钉住。所以响应丢失但服务端其实已完成时,判据反向:用户被告知「画布未收到完美像素结果,请确认素材库」,而结果早已在画布上,重做一遍就造出第二份;这条分支还刻意不套用快照,本地也看不到那个新图层。 @@ -6468,6 +6472,7 @@ - 权威性与剩余风险:preflight 不创建锁、reservation 或新表记录;最终 `persist_editor_pixel_art_result_and_return` 仍在同一事务内重复目录、布局、幂等 identity 和 revision 校验。preflight 通过后若目录或画布并发漂移,最终事务仍可能在 PUT 后拒绝并留下无引用 OSS object;彻底消除该 TOCTOU 需要 durable reservation / journal 或事务协调,不在本 PR 的最小修复边界内。 - 契约影响:只新增 SpacetimeDB procedure ABI 与生成 bindings;没有表字段、index、migration、HTTP DTO、路由、状态码、OpenAPI 或 shared-contracts 变化。 - 关联文档:`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`、`docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md`。 + ## 2026-08-04 AI 游戏项目 manifest 存储与工作台实时投影 - 存储决策:`.agent/manifest.json` 的版本追加不可变约束由同目录持久专用锁保护,读取旧状态、校验版本前缀、安装临时文件和安装后回读必须处于同一临界区;进程内 Mutex 不能替代跨进程文件锁。 @@ -6527,6 +6532,7 @@ - classic script 分析单元把 inline 与无 `defer / async` 的本地 external 正文按 `game/index.html` 标签顺序交错组成 parser-blocking 段,再把 classic external `defer` 按文档顺序放到解析完成后的 deferred 段;不得把 defer-before-inline 误投影为外链先执行。classic external `async` 的下载完成顺序不可静态证明,当前静态门直接失败关闭。带 `src` 标签的 inline body 继续忽略;外部文件仍执行可信普通文件、`game/` 边界、文件数与累计体积门禁,重复标签按浏览器出现次数保留求值位置。 - Canvas 尺寸、可见性、元素绑定和 stylesheet 选择器扫描只消费浏览器可渲染标记;`template / textarea / noscript / title / style / xmp / iframe / noembed / plaintext` 内的 Canvas、标签和样式诱饵全部跳过。活动顶层 stylesheet 与可见标记分开提取,既允许真实 CSS 参与隐藏/尺寸判断,也不把 CSS raw-text 中的伪标签当作 DOM。 - ESM 组合单元按 dependency 初始化先于 importer 顶层求值排列。import reference 的 span replacement 仍基于原 importer 完成,随后把已闭包的 dependency projection 放在 importer 前并对最终单元重跑 parser、semantic、单元 `2 MiB` 与累计投影 `32 MiB` 门禁;循环模块继续按 `(origin module, original root binding)` canonical identity 去重并要求有界固定点收敛。 + ## 2026-08-04 JavaScript 延迟状态与复合调用边 - 受控异步 callback 的 alias 读取按完整 enclosing invocation 链延迟到各层函数同步收尾,最外层再延迟到当前 job 末尾;callback 写入仍不在注册点同步提交。conditional / assignment expression callee 分别在 test / RHS 求值后建立调用边,`new` 同时执行普通 function constructor 及 alias。 @@ -6727,6 +6733,7 @@ - 业务隔离:共享组件不等于共享规则。SFX 继续使用 Vidu `audio1.0`、2–10 秒、默认 5 秒、现有 Prompt 回退、1500 字限制、价格和提交链路;BGM 继续使用 canonical Prompt、200 字生成限制、30 个预设、AI 补全 / 简化、单层撤销、提交锁和 Suno。BGM 按 dialog ID 写回,SFX 继续走现有 `setGenerateDialog`,两条路径不得互换。 - 非目标:本次只规划视图归并,不实现 SFX V2 的 ElevenLabs、中译英、自动时长、30 秒、Loop、一键优化或预设,不修改任何后端、External v1、Schema、计费或需求原文,也不新建配置驱动的 composer 框架。 - 实施状态:已恢复共享音频 composer,独立完整 BGM composer 及其测试文件已删除,原覆盖完整迁入总 composer。Prompt / 预设 / controller / 总 composer `121/121`、surface 与 submission workflow `72/72` 通过,typecheck、变更文件 ESLint、Prettier、编码检查和差异检查通过;没有修改后端、契约或需求原文,也没有实现 SFX V2 独有功能。 + ## 2026-08-06 SFX 生成优化 V2.0 T0 设计与迁移口径 - 权威入口:SFX V2 的可编码规则已完整融合到 `docs/【编辑器】画板音乐生成入口设计-2026-06-18.md`。实现、审查、测试和发布只以该 tracked 权威设计、本条决策和共享实施计划为依据,不依赖团队通过 Git 无法取得的本地资料。 @@ -6759,3 +6766,11 @@ - 预设视图:BGM 预设跑马灯抽出无业务语义的音频内核,保留单一可访问控件队列、无缝滚动、hover、触摸、页面可见性和 reduced-motion 行为;BGM / SFX 各自保留 wrapper、预设模型和业务 class。SFX wrapper 展示固定 `40 + 12` 预设,不保存展开、滚动或 hover 状态。 - 参数与布局:SFX 首次打开为手动 `5s`、Loop false;手动 slider 为 `0.5-30s`、步进 `0.1s`,自动模式禁用 slider 但保留最近手动值。layout 恢复 `soundDurationMode / soundDurationSeconds / soundLoop`,历史 Vidu dialog 与改造入口统一打开固定 `eleven_text_to_sound_v2` / `ElevenLabs` 面板。前端显示新模型 `5` 泥点兜底,正式价格仍以后端 T5 入队冻结值为真相。 - 锁与阶段边界:优化、提交或既有生成态只锁当前 SFX dialog 的输入、预设、滚动、参数、撤销和生成。提交 claim 同步冻结 canonical Prompt、时长模式、最近手动值与 Loop;T4 不改变正式请求的 nullable duration / Loop 映射,不接 Worker 翻译或 ElevenLabs,不修改服务端动态定价、计费、OSS、持久化详情、External v1、OpenAPI 或 SpacetimeDB schema。T4 必须与 T5 同一发布列车,不能单独发布。 + +## 2026-08-07 SFX 生成优化 V2.0 T5 正式生成与 External v1 + +- 正式执行链:站内与 External 请求在定价、预扣和 enqueue 前统一收敛为 canonical userPrompt、`model = eleven_text_to_sound_v2`、nullable 小数 duration 与 Loop;队列载荷不包含 actualPrompt。Worker 在既有冻结计费上下文内顺序执行 Luna 英文化、单次 ElevenLabs POST、MP3 校验 / 实际时长探测、OSS 和项目资源 / 账号素材 / 画布完成态写回,任一失败进入既有退款边界。queue 使用 job ID,inline 在 provider 前生成平台 Task ID,不伪造 provider task ID。 +- 权威结果:服务端只保留 Agent 身份关联字段并重建 SFX V2 `generation_inputs_json`,统一写入 userPrompt、actualPrompt、固定模型、duration mode、请求 / 实际时长和 Loop;客户端自报的实际英文 Prompt、实际时长、模型和 Loop 均被覆盖。信息弹窗展示中英 Prompt、实际时长、Loop、模型和完整平台 Task ID;重绘优先恢复 V2 metadata,自动模式恢复默认最近手动值 `5s`,不把实际输出时长当作手动请求值。历史 Vidu 素材仍按旧字段只读,并以新模型重绘。 +- 定价兼容:默认配置、api-server 与 SpacetimeDB 值校验同时要求保留 `audio1.0` 和新增 `eleven_text_to_sound_v2`。已存在的 SpacetimeDB 定价快照仅缺新键时,api-server 从当前受控默认 / override 补入该键后读取;其它缺失模型仍失败。该兼容不修改 schema、不在读取时写库,下一次后台保存自然持久化完整矩阵;队列计费、响应和资产成本继续使用入队冻结价格。 +- External v1:Rust DTO / handler、OpenAPI、幂等 canonical payload、compact result 与仓库 Agent Skill 同批演进。model 的省略 / null / 空串 / 纯 Unicode White_Space / 包围空白新模型 / 显式新模型统一入队;旧模型和未知非空值在 enqueue 前返回 `400`。完成结果增加实际 `durationSeconds` 与 Loop,继续隐藏 provider、userPrompt 和 actualPrompt,只暴露稳定结果引用。 +- 阶段状态:T1–T5 已完成,可以进入 T6;T6 仍需汇总 mock LLM / ElevenLabs / OSS 失败矩阵、计费退款、端到端等值、BGM 回归、API smoke、旧 Vidu 队列 drain 和发布 / 回滚门禁。T5 未执行真实 LLM、ElevenLabs 或其它付费请求,且没有 SpacetimeDB schema、migration 或 bindings 变更。 diff --git a/docs/【编辑器】模型定价配置管理方案-2026-06-22.md b/docs/【编辑器】模型定价配置管理方案-2026-06-22.md index 1a13c31b7..53943f418 100644 --- a/docs/【编辑器】模型定价配置管理方案-2026-06-22.md +++ b/docs/【编辑器】模型定价配置管理方案-2026-06-22.md @@ -35,6 +35,7 @@ "prices": { "480p": 10, "720p": 20, "1080p": 40 } }, "audio1.0": { "unit": "perGeneration", "price": 5 }, + "eleven_text_to_sound_v2": { "unit": "perGeneration", "price": 5 }, "chirp-v5": { "unit": "perGeneration", "price": 12 } } } @@ -46,6 +47,7 @@ - `price`:单一价格,适合音效、背景音乐等单次生成模型。 - `prices`:档位价格,图片模型按尺寸档位配置,视频模型按分辨率配置。 - 生图模型必须补齐支持尺寸:`gemini-3.1-flash-image-preview` 配 `0.5K / 1K / 2K`,`gpt-image-2` 配 `1K / 2K`。 +- 新编辑器 SFX 只读取 `eleven_text_to_sound_v2`;`audio1.0` 继续保留为历史 Vidu 配置兼容键,两者均按次独立配置。 后端保存前校验当前正式模型、必要尺寸和必要分辨率都存在且大于 0。 @@ -61,6 +63,8 @@ SpacetimeDB 模块会在事务内重复执行同等强度的校验,并拒绝 所有会调用外部生成 provider 的编辑器生成请求都必须由后端计算价格,前端请求不提交价格字段;同步执行按当前运行时配置进入 `execute_billable_asset_operation_with_cost` 预扣泥点,预扣失败不得继续调用上游。外部生成队列在入队时把价格写入 `external_generation_job.price_mud_points`,worker 必须用该冻结价格完成扣费、退款、响应和资产成本持久化,配置更新不得改变已入队任务金额。普通图片、规范、角色、UI 设计、宣发素材、快速编辑 / 图片修改、图标 spritesheet、UI 设计图提取素材、视频、角色动作、音效和背景音乐均遵循该规则。背景色决策(gpt-5-mini)本身也是一次上游调用,同样必须在预扣泥点之后发起:预扣前只做颜色无关的算价 / 校验(动画用默认色占位算价),决策放进 billable 闭包,余额不足则决策不跑、决策失败走失败退款。需要向前端展示实际扣费时,由后端在响应中返回 `priceMudPoints`。 +SFX V2 上线前已经存在的 SpacetimeDB 定价快照可能只有 `audio1.0`。读取这类历史快照时,`api-server` 只允许从当前受控默认配置或本地 override 补入缺失的 `eleven_text_to_sound_v2` 条目,使旧快照可继续读取;其它必需模型缺失仍失败。该兼容不修改 schema,也不在读取时写数据库;下一次后台保存完整定价矩阵时自然持久化新键。发布前仍应确认运行时配置中的新键和价格已经批准。 + ## 运行时身份首次授权 模型定价 writer、外部生成队列和钱包调用都以真实 SpacetimeDB `ctx.sender()` 校验运行时服务 identity。原始 bootstrap secret 固定为 64 位十六进制;首次授权使用与当前 `spacetime_module.wasm` 构建时注入 SHA-256 摘要对应的原始值,模块收到原始值后重新计算 SHA-256 并做常量时间比较,WASM 只嵌入摘要、不嵌入原文。bootstrap secret 只能在配置表为空时建立首个受信身份,表存在后不能重复使用。queue 和钱包 runtime guard 只接受精确 `writer_identity`,迁移操作员身份不自动获得在线生成或钱包权限;因此当前生产 API、worker 和 controller 必须继承同一份 runtime token。非 HTTP 角色只做 queue procedure 鉴权预检,不具备 seed 或轮换身份的职责。migration operator 与 runtime writer 必须互斥:任何已登记 operator 都不能成为 writer,当前 writer 也不能被授权为 operator;一旦已有 operator,bootstrap secret 不得再新增或接管 operator。 diff --git a/docs/【编辑器】画布Agent对话面板-2026-07-03.md b/docs/【编辑器】画布Agent对话面板-2026-07-03.md index 01aad43ae..e3bc14106 100644 --- a/docs/【编辑器】画布Agent对话面板-2026-07-03.md +++ b/docs/【编辑器】画布Agent对话面板-2026-07-03.md @@ -96,7 +96,7 @@ - `EditorAgentToolCall.args` 的正式持久化契约是**校验后的规范参数 JSON**,不是 LLM 返回的原始 JSON。api-server 收到工具调用后,必须先按已注册的 ToolArgs 反序列化、补齐字段默认值、删除未进入 ToolArgs 的未知 / 退役字段、执行工具参数校验,再重新序列化并写入 `args`;校验失败的调用不得持久化为待确认消息。所有有明确默认值的工具标量参数在强类型 ToolArgs 中必须使用非 `Option` 字段:调用方省略字段或把顶层字段显式传为 `null` 时,统一在 ToolArgs 反序列化前视为未提供,由 Serde 补齐默认值,并把具体默认值写入规范 `args`;没有默认值的必填字段显式传为 `null` 时同样按缺失处理.(for compatibility) 后续计价、确认展示和 job payload 不得再次使用 `unwrap_or` 补同一默认值。LLM 原始参数只作为本次规范化的瞬时输入,不作为执行或审计真相;确认、取消、任务回填与后续上下文统一读取同一条消息中的规范 `args`。图片参数继续只保存由真实 data key 计算出的 opaque SHA-256 `imageId`;不得为了前端预览把 `args` 中的图片 ID 改写成 `objectKey`、URL 或展示对象,也不得由前端重组或回传一份新的执行参数。 - api-server 内画布 Agent 工具统一实现 object-safe `EditorAgentTool: ToolDyn`。`validate_args`、计价、确认展示、worker job 构建、`format_execute_message` 和结果媒体投影都使用统一 JSON 边界;每个具体工具实现负责把 JSON 反序列化为自己的强类型 Args / 结果,并把校验与完成消息格式化转发到 `platform-editor-agent` 中既有的 typed `validate_args` / `format_execute_message`,不得在调用方复制工具规则。`editor_agent_tool(toolName, context)` 是唯一按工具名分派的位置,规划、确认和任务回填只调用返回的 dyn tool;新增工具必须补齐同一个 trait 实现和该工厂分支。LLM builder 的 `.tool(...)` 注册列表仍是独立显式清单,不属于本次动态分派。framework runner 必须在 `ToolCallOutput` 中保留工具返回的结构化 output;runner 写入 LLM memory 与 api-server 使用规范参数持久化 system text 时统一调用公开的 `format_tool_call_message`,不得丢弃 `TOOL_CALL_PENDING_MESSAGE` 后自行拼另一套“等待确认”输出。 - `EditorAgentToolCall.displayArgs` 是必填、只读的用户确认展示投影,与 `args` 分离: - - `stringArgs` 保存提示词、比例、清晰度、模型、时长等可展示参数的稳定名称、用户可见标题和值;前端渲染模型字段时复用图片编辑器公共展示名映射,`gemini-3.1-flash-image-preview` 显示为 `nanobanana2`、`audio1.0` 显示为 `Vidu`、`chirp-v5` 显示为 `Suno`,视频模型显示现有产品标签,不得改写后端参数真相; + - `stringArgs` 保存提示词、比例、清晰度、模型、时长等可展示参数的稳定名称、用户可见标题和值;前端渲染模型字段时复用图片编辑器公共展示名映射,`gemini-3.1-flash-image-preview` 显示为 `nanobanana2`、`eleven_text_to_sound_v2` 显示为 `ElevenLabs`、历史 `audio1.0` 显示为 `Vidu`、`chirp-v5` 显示为 `Suno`,视频模型显示现有产品标签,不得改写后端参数真相; - `imageArgs` 按“目标图片 / 参考图片”等参数分组,每个 `refs` 项包含与规范参数对应的 `imageId`,以及后端从已校验会话上下文解析出的 `objectKey`、`imageSrc`、可选 `thumbnailSrc` / `label` / `width` / `height`。 - `extras.priceMudPoints` 保存创建待确认消息时按后端运行时模型定价快照计算的预计泥点消耗;前端统一展示为“预计消耗 N泥点”,不自行计算价格。 - `displayArgs` 只能由 api-server 按已注册 tool 白名单,基于已经通过 ToolArgs 校验的 `args` 和当前请求开始时从 OSS 会话文档一次性构建的 `EditorToolContext` 生成;该 context 必须按 opaque `ImageId` 同时保存执行所需的 `dataKey` 与展示所需的图片地址、Object Key、缩略图、label、宽高,参数校验、确认展示和 job payload 统一查同一份 context。不能信任 LLM 自报的展示地址、标题或素材元数据。展示投影不参与确认执行,确认接口仍只读取同一条持久化 tool call 的 `args`,避免“看到的素材”和“实际执行的素材”分叉。 diff --git a/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md b/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md index f848d8bce..e0feeb152 100644 --- a/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md +++ b/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md @@ -12,7 +12,7 @@ 音效与背景音乐继续使用同一个音频 composer,并由组件内的 `isSoundEffect = dialog.mode === 'audio-sound-effect'` 隔离行为。共享视图不等于共享业务规则:BGM 继续使用 Suno、200 字 canonical Prompt、30 个预设、AI 补全 / 简化、单层撤销和方案 A 提交锁;SFX V2 固定使用 ElevenLabs `eleven_text_to_sound_v2`、52 个预设、一键优化、自动中译英、自动 / 手动时长和 Loop。两条路径的 Prompt 模型、controller、预设 wrapper、锁和提交契约必须分别维护,不得交叉复用业务状态。 -SFX V2 已完成产品与技术口径冻结,T0 已通过,T1、T2、T3、T4 已完成,可以继续进入 T5;这不表示功能已上线。T2 已落地登录态一键优化 BFF 和仅生成流水线内部可见的 Worker 翻译 service,T3 已落地 ElevenLabs 直接二进制 adapter,T4 已落地 dialog-scoped 前端 controller、52 预设、优化 / 撤销、自动 / 手动时长与 Loop UI。翻译到正式 Worker、ElevenLabs、计费、持久化和 External v1 的接线仍属于 T5。在正式切换完成前,当前正式生成代码仍是 Vidu SFX V1 行为,T4 前端也不得单独发布。历史 Vidu 素材继续只读展示;重绘时使用历史用户 Prompt 打开 SFX V2 面板,新任务统一走 ElevenLabs,不回退 Vidu。 +SFX V2 已完成产品与技术口径冻结,T0 已通过,T1–T5 已完成,可以继续进入 T6 测试、灰度与发布门禁;这不表示功能已上线。登录态一键优化、Worker 翻译、ElevenLabs 直接二进制 adapter、dialog-scoped 前端交互以及正式提交 / Worker / 计费 / OSS / 权威 metadata / External v1 已完成接线。历史 Vidu 素材继续只读展示;重绘时使用历史用户 Prompt 打开 SFX V2 面板,新任务统一走 ElevenLabs,不回退 Vidu。T6 完成前仍不得发布。 ## 入口与交互 diff --git a/server-rs/crates/api-server/config/editor-generation-pricing.default.json b/server-rs/crates/api-server/config/editor-generation-pricing.default.json index 3e08db484..18d18f1b7 100644 --- a/server-rs/crates/api-server/config/editor-generation-pricing.default.json +++ b/server-rs/crates/api-server/config/editor-generation-pricing.default.json @@ -67,6 +67,10 @@ "unit": "perGeneration", "price": 5 }, + "eleven_text_to_sound_v2": { + "unit": "perGeneration", + "price": 5 + }, "chirp-v5": { "unit": "perGeneration", "price": 12 diff --git a/server-rs/crates/api-server/src/app.rs b/server-rs/crates/api-server/src/app.rs index a2d70713f..73bbdea2b 100644 --- a/server-rs/crates/api-server/src/app.rs +++ b/server-rs/crates/api-server/src/app.rs @@ -5194,6 +5194,10 @@ mod tests { payload["models"]["audio1.0"]["price"], Value::Number(5.into()) ); + assert_eq!( + payload["models"]["eleven_text_to_sound_v2"]["price"], + Value::Number(5.into()) + ); assert_eq!( payload["models"]["chirp-v5"]["price"], Value::Number(12.into()) @@ -5260,6 +5264,7 @@ mod tests { "prices": { "480p": 11, "720p": 22, "1080p": 44 } }, "audio1.0": { "unit": "perGeneration", "price": 15 }, + "eleven_text_to_sound_v2": { "unit": "perGeneration", "price": 16 }, "chirp-v5": { "unit": "perGeneration", "price": 9 } } }) @@ -5298,6 +5303,10 @@ mod tests { payload["models"]["audio1.0"]["unit"], Value::String("perGeneration".to_string()) ); + assert_eq!( + payload["models"]["eleven_text_to_sound_v2"]["price"], + Value::Number(16.into()) + ); assert_eq!( payload["models"]["seedance2.0"]["prices"]["720p"], Value::Number(26.into()) diff --git a/server-rs/crates/api-server/src/editor_agent/tool.rs b/server-rs/crates/api-server/src/editor_agent/tool.rs index 60fcac80d..31fc50cd4 100644 --- a/server-rs/crates/api-server/src/editor_agent/tool.rs +++ b/server-rs/crates/api-server/src/editor_agent/tool.rs @@ -1509,8 +1509,9 @@ mod tests { "height": 0, "sourceType": "generated", "prompt": "按钮点击声", - "model": "audio1.0", - "provider": "vectorengine", + "actualPrompt": "A short button click", + "model": "eleven_text_to_sound_v2", + "provider": "elevenlabs", "taskId": "task-1", "priceMudPoints": 5, "audioKind": "sound-effect" diff --git a/server-rs/crates/api-server/src/editor_generation_config.rs b/server-rs/crates/api-server/src/editor_generation_config.rs index 5143bae61..ab488b7af 100644 --- a/server-rs/crates/api-server/src/editor_generation_config.rs +++ b/server-rs/crates/api-server/src/editor_generation_config.rs @@ -28,6 +28,8 @@ const EDITOR_VIDEO_MODEL_KLING_3_OMNI: &str = "kling3.0-omni"; const EDITOR_VIDEO_MODEL_VEO_3_1: &str = "veo3.1"; const EDITOR_VIDEO_MODEL_VEO_3_1_FAST: &str = "veo3.1-fast"; pub(crate) const EDITOR_SOUND_EFFECT_MODEL_VIDU: &str = "audio1.0"; +pub(crate) const EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS: &str = + shared_contracts::assets::EDITOR_SOUND_EFFECT_MODEL; pub(crate) const EDITOR_BACKGROUND_MUSIC_MODEL_SUNO: &str = "chirp-v5"; const IMAGE_PRICE_SIZE_0_5K: &str = "0.5K"; @@ -188,11 +190,12 @@ impl EditorGenerationPricingConfig { if let Some(price_mud_points) = current_external_generation_billing_price_mud_points() { return price_mud_points; } - let normalized_model = normalize_non_empty_model(model, EDITOR_SOUND_EFFECT_MODEL_VIDU); + let normalized_model = + normalize_non_empty_model(model, EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS); read_flat_price( &self.models, normalized_model, - EDITOR_SOUND_EFFECT_MODEL_VIDU, + EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS, ) } @@ -235,6 +238,11 @@ impl EditorGenerationPricingConfig { EDITOR_SOUND_EFFECT_MODEL_VIDU, EditorGenerationPricingUnit::PerGeneration, )?; + validate_required_flat_price( + &self.models, + EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS, + EditorGenerationPricingUnit::PerGeneration, + )?; validate_required_flat_price( &self.models, EDITOR_BACKGROUND_MUSIC_MODEL_SUNO, @@ -670,7 +678,8 @@ mod tests { queued_price ); assert_eq!( - config.sound_effect_model_mud_points(Some("audio1.0")), + config + .sound_effect_model_mud_points(Some(EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS)), queued_price ); assert_eq!( @@ -781,7 +790,13 @@ mod tests { #[test] fn editor_audio_generation_price_uses_configured_model_rates() { assert_eq!( - editor_sound_effect_model_generation_mud_points(Some("audio1.0")), + editor_sound_effect_model_generation_mud_points(Some( + EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS + )), + 5 + ); + assert_eq!( + editor_sound_effect_model_generation_mud_points(Some(EDITOR_SOUND_EFFECT_MODEL_VIDU)), 5 ); assert_eq!( @@ -832,6 +847,7 @@ mod tests { "prices": { "480p": 11, "720p": 22, "1080p": 44 } }, "audio1.0": { "unit": "perGeneration", "price": 15 }, + "eleven_text_to_sound_v2": { "unit": "perGeneration", "price": 16 }, "chirp-v5": { "unit": "perGeneration", "price": 9 } } }"#, @@ -854,6 +870,10 @@ mod tests { config.character_animation_model_mud_points(Some("seedance2.0-fast"), "720p", 6), 132 ); + assert_eq!( + config.sound_effect_model_mud_points(Some(EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS)), + 16 + ); assert_eq!(config.sound_effect_model_mud_points(Some("audio1.0")), 15); assert_eq!( config.background_music_model_mud_points(Some("chirp-v5")), @@ -901,6 +921,7 @@ mod tests { "prices": { "480p": 10, "720p": 20, "1080p": 40 } }, "audio1.0": { "unit": "perGeneration", "price": 10 }, + "eleven_text_to_sound_v2": { "unit": "perGeneration", "price": 5 }, "chirp-v5": { "unit": "perGeneration", "price": 5 } } }"#, diff --git a/server-rs/crates/api-server/src/external_editor_api.rs b/server-rs/crates/api-server/src/external_editor_api.rs index 4600201dc..336b8cdb1 100644 --- a/server-rs/crates/api-server/src/external_editor_api.rs +++ b/server-rs/crates/api-server/src/external_editor_api.rs @@ -768,30 +768,16 @@ pub async fn generate_external_editor_video( Ok(external_generation_accepted_response(&request_context, job)) } -/// T1 先冻结 External v1 的 model 输入矩阵;实际在定价、预扣和入队前接线属于 T5。 -#[allow(dead_code)] fn canonicalize_external_editor_sound_effect_model( value: Option<&str>, ) -> Result<&'static str, AppError> { - let normalized = value - .map(|value| value.trim_matches(char::is_whitespace)) - .filter(|value| !value.is_empty()); - match normalized { - None => Ok(shared_contracts::assets::EDITOR_SOUND_EFFECT_MODEL), - Some(value) if value == shared_contracts::assets::EDITOR_SOUND_EFFECT_MODEL => { - Ok(shared_contracts::assets::EDITOR_SOUND_EFFECT_MODEL) - } - Some(_) => Err( - AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ - "provider": EXTERNAL_EDITOR_PROVIDER, - "field": "model", - "message": format!( - "model 只支持 {}", - shared_contracts::assets::EDITOR_SOUND_EFFECT_MODEL - ), - })), - ), - } + shared_contracts::assets::canonicalize_editor_sound_effect_model(value).map_err(|message| { + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": EXTERNAL_EDITOR_PROVIDER, + "field": "model", + "message": message, + })) + }) } pub async fn generate_external_editor_sound_effect( @@ -807,7 +793,10 @@ pub async fn generate_external_editor_sound_effect( require_scope_response(&request_context, &principal, SCOPE_EDITOR_IMAGE_GENERATE)?; let idempotency_key = require_idempotency_key(&headers) .map_err(|error| error.into_response_with_context(Some(&request_context)))?; - let Json(payload) = parse_external_generation_json_payload(&request_context, payload)?; + let Json(mut payload) = parse_external_generation_json_payload(&request_context, payload)?; + let model = canonicalize_external_editor_sound_effect_model(payload.model.as_deref()) + .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + payload.model = Some(model.to_string()); let job = enqueue_editor_sound_effect_generation_for_owner( &state, &request_context, @@ -1719,6 +1708,38 @@ mod tests { .get("/api/external/v1/editor/audios/sound-effects/generations") .is_some() ); + let sound_request = &parsed["components"]["schemas"]["EditorSoundEffectGenerationRequest"]; + assert_eq!(sound_request["required"], json!(["prompt"])); + assert_eq!( + sound_request["properties"]["prompt"]["maxLength"], + json!(2048) + ); + assert_eq!( + sound_request["properties"]["model"]["default"], + json!(shared_contracts::assets::EDITOR_SOUND_EFFECT_MODEL) + ); + assert_eq!( + sound_request["properties"]["duration"]["anyOf"][0]["type"], + json!("number") + ); + assert_eq!( + sound_request["properties"]["duration"]["anyOf"][0]["minimum"], + json!(0.5) + ); + assert_eq!( + sound_request["properties"]["duration"]["anyOf"][0]["maximum"], + json!(30) + ); + assert_eq!(sound_request["properties"]["loop"]["default"], json!(false)); + let audio_response = &parsed["components"]["schemas"]["EditorAudioGenerationResponse"]; + assert_eq!( + audio_response["properties"]["durationSeconds"]["maximum"], + json!(600) + ); + assert_eq!( + audio_response["properties"]["loop"]["type"], + json!(["boolean", "null"]) + ); assert!( parsed["paths"] .get("/api/external/v1/editor/audios/background-music/generations") diff --git a/server-rs/crates/api-server/src/external_generation_worker.rs b/server-rs/crates/api-server/src/external_generation_worker.rs index e1174fdc7..af859745a 100644 --- a/server-rs/crates/api-server/src/external_generation_worker.rs +++ b/server-rs/crates/api-server/src/external_generation_worker.rs @@ -1007,6 +1007,7 @@ async fn process_external_generation_job_once( request_context, job.owner_user_id.clone(), Ok(Json(payload)), + Some(job.job_id.clone()), ) .await { @@ -1337,6 +1338,7 @@ fn compact_external_api_generation_result(result: Value) -> Value { | "model" | "taskId" | "durationSeconds" + | "loop" | "resolution" | "priceMudPoints" | "audioKind" @@ -2087,12 +2089,22 @@ mod tests { }; use shared_contracts::editor_agent::{EditorAgentMessage, EditorAgentToolCallStatus}; - for (tool_name, prompt, audio_kind) in [ - (GenerateSoundEffectTool::NAME, "按钮点击声", "sound-effect"), + for (tool_name, prompt, actual_prompt, audio_kind, model, provider) in [ + ( + GenerateSoundEffectTool::NAME, + "按钮点击声", + "A short button click", + "sound-effect", + "eleven_text_to_sound_v2", + "elevenlabs", + ), ( GenerateBackgroundMusicTool::NAME, "森林背景音乐", + "森林背景音乐", "background-music", + "chirp-v5", + "vectorengine", ), ] { let mut job = external_generation_job_record_fixture(Some("lease-1")); @@ -2107,18 +2119,21 @@ mod tests { "height": 120, "sourceType": "generated", "prompt": prompt, - "model": "audio1.0", - "provider": "vectorengine", + "actualPrompt": actual_prompt, + "model": model, + "provider": provider, "taskId": "task-1", "priceMudPoints": 5, "audioKind": audio_kind, + "durationSeconds": if audio_kind == "sound-effect" { json!(5.25) } else { Value::Null }, + "loop": if audio_kind == "sound-effect" { json!(false) } else { Value::Null }, }); let payload: Value = serde_json::from_str(&editor_generation_result_payload_json(&job, &response)) .expect("worker compact payload should serialize"); assert_eq!( payload["editor-agent-tool-call-result"]["provider"], - json!("vectorengine") + json!(provider) ); let mut message: EditorAgentMessage = serde_json::from_value(json!({ "id": 1, @@ -2352,6 +2367,8 @@ mod tests { "assetObjectId": "asset-object-main", "width": 1024, "height": 1024, + "durationSeconds": 7.42, + "loop": true, "provider": "internal-provider-must-not-persist", "resource": { "resourceId": "resource-main", @@ -2430,6 +2447,8 @@ mod tests { json!("users/user-1/generated/main.png") ); assert_eq!(result["assetObjectId"], json!("asset-object-main")); + assert_eq!(result["durationSeconds"], json!(7.42)); + assert_eq!(result["loop"], json!(true)); assert_eq!(result["resource"]["resourceId"], json!("resource-main")); assert_eq!( result["resource"]["sourceResourceId"], diff --git a/server-rs/crates/api-server/src/state.rs b/server-rs/crates/api-server/src/state.rs index af376db99..d8d541170 100644 --- a/server-rs/crates/api-server/src/state.rs +++ b/server-rs/crates/api-server/src/state.rs @@ -36,8 +36,9 @@ use tracing::{info, warn}; use crate::config::AppConfig; use crate::editor_generation_config::{ - EditorGenerationModelPricing, EditorGenerationPricingConfig, EditorGenerationPricingError, - EditorGenerationPricingStore, EditorGenerationPricingUnit, + EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS, EditorGenerationModelPricing, + EditorGenerationPricingConfig, EditorGenerationPricingError, EditorGenerationPricingStore, + EditorGenerationPricingUnit, }; use crate::tracking_outbox::TrackingOutbox; use crate::wallet_refund_outbox::WalletRefundOutbox; @@ -368,6 +369,7 @@ fn editor_generation_pricing_to_records( fn editor_generation_pricing_from_record( record: EditorGenerationPricingConfigRecord, + legacy_fallback: &EditorGenerationPricingConfig, ) -> Result { let mut models = BTreeMap::new(); for pricing in record.models { @@ -407,6 +409,18 @@ fn editor_generation_pricing_from_record( ))); } } + if !models.contains_key(EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS) { + let pricing = legacy_fallback + .models + .get(EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS) + .cloned() + .ok_or_else(|| { + EditorGenerationPricingError::Invalid(format!( + "本地模型定价配置缺少模型 {EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS}" + )) + })?; + models.insert(EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS.to_string(), pricing); + } let config = EditorGenerationPricingConfig { models }; config.validate()?; Ok(config) @@ -621,7 +635,8 @@ impl AppState { .await { Ok(Some(record)) => { - let pricing = editor_generation_pricing_from_record(record)?; + let legacy_fallback = self.editor_generation_pricing_store.snapshot()?; + let pricing = editor_generation_pricing_from_record(record, &legacy_fallback)?; self.editor_generation_pricing_store .replace(pricing.clone())?; Ok(pricing) @@ -661,7 +676,7 @@ impl AppState { )) .await .map_err(|error| EditorGenerationPricingError::Persistence(error.to_string()))?; - let pricing = editor_generation_pricing_from_record(record)?; + let pricing = editor_generation_pricing_from_record(record, &next)?; self.editor_generation_pricing_store .replace(pricing.clone())?; Ok(pricing) @@ -693,7 +708,7 @@ impl AppState { ) .await .map_err(|error| EditorGenerationPricingError::Persistence(error.to_string()))?; - let pricing = editor_generation_pricing_from_record(record)?; + let pricing = editor_generation_pricing_from_record(record, &fallback)?; self.editor_generation_pricing_store .replace(pricing.clone())?; Ok(pricing) @@ -2318,12 +2333,39 @@ mod tests { updated_at_micros: 1, }; - let actual = editor_generation_pricing_from_record(record) + let actual = editor_generation_pricing_from_record(record, &expected) .expect("typed pricing record should map back"); assert_eq!(actual, expected); } + #[test] + fn editor_generation_pricing_typed_record_backfills_legacy_sfx_model() { + let fallback = crate::editor_generation_config::parse_editor_generation_pricing_json( + crate::editor_generation_config::EDITOR_GENERATION_PRICING_DEFAULT_JSON, + "test default pricing", + ) + .expect("default pricing should parse"); + let mut models = + editor_generation_pricing_to_records(&fallback).expect("pricing should map to records"); + models.retain(|pricing| pricing.model != EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS); + let record = EditorGenerationPricingConfigRecord { + config_id: "global".to_string(), + models, + updated_by_admin_user_id: Some("admin:test".to_string()), + updated_at: "2026-08-07T00:00:00Z".to_string(), + updated_at_micros: 1, + }; + + let actual = editor_generation_pricing_from_record(record, &fallback) + .expect("legacy pricing should receive only the new SFX model fallback"); + + assert_eq!( + actual.models.get(EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS), + fallback.models.get(EDITOR_SOUND_EFFECT_MODEL_ELEVENLABS) + ); + } + #[test] fn editor_generation_pricing_upsert_input_uses_runtime_service_bootstrap_secret() { let mut config = AppConfig::default(); @@ -2366,8 +2408,8 @@ mod tests { updated_at_micros: 1, }; - let error = - editor_generation_pricing_from_record(record).expect_err("duplicate model should fail"); + let error = editor_generation_pricing_from_record(record, &config) + .expect_err("duplicate model should fail"); assert!(error.to_string().contains("重复模型")); } diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation.rs index d276a408d..a86eca089 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation.rs @@ -4,8 +4,6 @@ mod generation; mod persist; mod publish; mod settings; -// T2 交付内部翻译 service;T5 才在正式音效 Worker 流水线调用并移除该暂时豁免。 -#[allow(dead_code)] mod sound_effect_translation; mod types; diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs index b85a60baf..b55755a0b 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs @@ -9,10 +9,12 @@ use platform_oss::LegacyAssetPrefix; use serde_json::Value; use serde_json::json; use shared_contracts::assets; +use shared_kernel::build_prefixed_uuid_id; use spacetime_client::ExternalGenerationJobRecord; use crate::{ api_response::json_success_body, + asset_billing::execute_billable_asset_operation_with_cost, auth::AuthenticatedAccessToken, editor_generation_config::EditorGenerationPricingConfig, editor_generation_queue::{ @@ -23,7 +25,7 @@ use crate::{ editor_project::{ EditorCanvasGeneratedLayerInput, PersistEditorGeneratedAssetRequest, build_editor_canvas_generated_layer_item, complete_editor_canvas_generation_with_items, - persist_editor_generated_media_asset, with_editor_media_duration_generation_input, + persist_editor_generated_media_asset, }, http_error::AppError, request_context::RequestContext, @@ -32,19 +34,22 @@ use crate::{ use super::{ errors::{map_platform_audio_error, parse_json_payload, vector_engine_bad_gateway}, - publish::{wait_for_generated_audio_asset, wait_for_generated_audio_asset_with_task_kind}, - settings::require_vector_engine_audio_settings, + persist::persist_generated_audio_asset, + publish::{build_audio_billing_asset_id, wait_for_generated_audio_asset}, + settings::{require_elevenlabs_audio_settings, require_vector_engine_audio_settings}, + sound_effect_translation::translate_sound_effect_prompt_for_worker, types::{AudioAssetBindingTarget, AudioAssetSlot}, }; const EDITOR_AUDIO_WIDTH: u32 = 420; const EDITOR_AUDIO_HEIGHT: u32 = 120; -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub(super) struct NormalizedEditorSoundEffectRequest { pub(super) prompt: String, pub(super) model: String, - pub(super) duration: u8, + pub(super) duration: Option, + pub(super) loop_enabled: bool, pub(super) price_mud_points: u32, } @@ -61,6 +66,12 @@ struct PreparedEditorBackgroundMusicQueueJob { price_mud_points: u32, } +#[derive(Debug)] +struct PreparedEditorSoundEffectQueueJob { + payload: assets::EditorSoundEffectGenerateRequest, + price_mud_points: u32, +} + #[derive(Debug)] struct CanonicalEditorBackgroundMusicSubmissionPayload { payload: assets::EditorBackgroundMusicGenerateRequest, @@ -98,37 +109,46 @@ pub(super) fn normalize_editor_sound_effect_request_with_pricing( payload: assets::EditorSoundEffectGenerateRequest, pricing: &EditorGenerationPricingConfig, ) -> Result { - let model = normalize_editor_sound_model(payload.model.as_deref())?; + let payload = canonicalize_editor_sound_effect_submission_payload(payload)?; + let model = payload + .model + .clone() + .expect("canonical SFX payload must contain model"); let expected_price_mud_points = pricing.sound_effect_model_mud_points(Some(model.as_str())); - let prompt = platform_audio::validate_sound_effect_prompt(&payload.prompt) - .map_err(map_platform_audio_error)?; Ok(NormalizedEditorSoundEffectRequest { - prompt: prompt.prompt.to_string(), + prompt: payload.prompt, model, - duration: normalize_editor_sound_duration(payload.duration)?, + duration: payload.duration, + loop_enabled: payload.loop_enabled, price_mud_points: expected_price_mud_points, }) } -fn normalize_editor_sound_model(value: Option<&str>) -> Result { - let model = platform_audio::normalize_optional_text(value) - .unwrap_or_else(|| platform_audio::VIDU_AUDIO_MODEL.to_string()); - if model == platform_audio::VIDU_AUDIO_MODEL { - return Ok(model); - } - Err(editor_audio_bad_request( - "编辑器音效暂只支持 Vidu audio1.0 模型", - )) +fn canonicalize_editor_sound_effect_submission_payload( + mut payload: assets::EditorSoundEffectGenerateRequest, +) -> Result { + let model = assets::canonicalize_editor_sound_effect_model(payload.model.as_deref()) + .map_err(editor_audio_bad_request)?; + let prompt = platform_audio::validate_sound_effect_prompt(&payload.prompt) + .map_err(map_platform_audio_error)?; + let duration = platform_audio::validate_sound_effect_duration_seconds(payload.duration) + .map_err(map_platform_audio_error)?; + payload.prompt = prompt.prompt.to_string(); + payload.model = Some(model.to_string()); + payload.duration = duration; + Ok(payload) } -fn normalize_editor_sound_duration(value: Option) -> Result { - let value = value.unwrap_or(f64::from( - platform_audio::DEFAULT_SOUND_EFFECT_DURATION_SECONDS, - )); - if value.is_finite() && value.fract() == 0.0 && (2.0..=10.0).contains(&value) { - return Ok(value as u8); - } - Err(editor_audio_bad_request("音效 duration 必须在 2-10 秒之间")) +fn prepare_editor_sound_effect_queue_job( + payload: assets::EditorSoundEffectGenerateRequest, + pricing: &EditorGenerationPricingConfig, +) -> Result { + let payload = canonicalize_editor_sound_effect_submission_payload(payload)?; + let price_mud_points = pricing.sound_effect_model_mud_points(payload.model.as_deref()); + Ok(PreparedEditorSoundEffectQueueJob { + payload, + price_mud_points, + }) } #[cfg(any())] @@ -208,8 +228,14 @@ pub async fn generate_editor_sound_effect( }, )); } - generate_editor_sound_effect_for_owner(state, request_context, owner_user_id, Ok(Json(payload))) - .await + generate_editor_sound_effect_for_owner( + state, + request_context, + owner_user_id, + Ok(Json(payload)), + None, + ) + .await } pub(crate) async fn enqueue_editor_sound_effect_generation_for_owner( @@ -227,10 +253,12 @@ pub(crate) async fn enqueue_editor_sound_effect_generation_for_owner( })) .into_response_with_context(Some(request_context)) })?; - let normalized = normalize_editor_sound_effect_request_with_pricing(payload.clone(), &pricing) + let prepared = prepare_editor_sound_effect_queue_job(payload, &pricing) .map_err(|error| error.into_response_with_context(Some(request_context)))?; - let source_entity_id = - editor_generation_source_entity_id(payload.project_id.as_deref(), "editor-sound-effect"); + let source_entity_id = editor_generation_source_entity_id( + prepared.payload.project_id.as_deref(), + "editor-sound-effect", + ); enqueue_editor_generation_job_for_caller( state, request_context, @@ -238,8 +266,8 @@ pub(crate) async fn enqueue_editor_sound_effect_generation_for_owner( EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND, source_entity_id, "图片画布生成音效", - u64::from(normalized.price_mud_points), - &payload, + u64::from(prepared.price_mud_points), + &prepared.payload, external_idempotency_key, ) .await @@ -251,13 +279,9 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( request_context: RequestContext, owner_user_id: String, payload: Result, JsonRejection>, + platform_task_id: Option, ) -> Result, Response> { let Json(payload) = parse_json_payload(&request_context, payload)?; - let project_id = payload.project_id.clone(); - let canvas_completion = payload.canvas_completion.clone(); - let generation_inputs = payload.generation_inputs.clone(); - let asset_folder_id = payload.asset_folder_id.clone(); - let asset_label = payload.asset_label.clone(); let pricing = state .editor_generation_pricing() .await @@ -268,156 +292,247 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( })) }) .map_err(|error| error.into_response_with_context(Some(&request_context)))?; - let normalized = normalize_editor_sound_effect_request_with_pricing(payload, &pricing) + let normalized = normalize_editor_sound_effect_request_with_pricing(payload.clone(), &pricing) .map_err(|error| error.into_response_with_context(Some(&request_context)))?; - let generation_inputs = with_editor_media_duration_generation_input( - generation_inputs, - u32::from(normalized.duration), - ); - let settings = require_vector_engine_audio_settings(&state) + let settings = require_elevenlabs_audio_settings(&state) .map_err(|error| error.into_response_with_context(Some(&request_context)))?; - let http_client = platform_audio::build_vector_engine_audio_http_client(&settings) + let http_client = platform_audio::build_elevenlabs_audio_http_client(&settings) .map_err(map_platform_audio_error) .map_err(|error| error.into_response_with_context(Some(&request_context)))?; - let task = platform_audio::submit_editor_sound_effect_task( - &http_client, - &settings, - platform_audio::EditorSoundEffectTaskRequest { - prompt: normalized.prompt.clone(), - duration: normalized.duration, - seed: None, - model: Some(normalized.model.clone()), - }, - ) - .await - .map_err(map_platform_audio_error) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; - let generated = wait_for_generated_audio_asset_with_task_kind( + let llm_client = state + .editor_agent_llm_client() + .ok_or_else(|| { + AppError::from_status(StatusCode::SERVICE_UNAVAILABLE).with_details(json!({ + "provider": "editor-sound-effect-translation", + "reason": "editor_llm_unavailable", + "message": "音效 Prompt 英文化服务未配置", + })) + }) + .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + let task_id = platform_task_id + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| build_prefixed_uuid_id("task-")); + let target = build_editor_audio_target( + task_id.as_str(), + "sound_effect", + "editor_sound_effect", + Some(u64::from(normalized.price_mud_points)), + ); + let billing_asset_id = + build_audio_billing_asset_id(task_id.as_str(), AudioAssetSlot::SoundEffect, &target); + let response = execute_billable_asset_operation_with_cost( &state, owner_user_id.as_str(), - task.task_id.clone(), - AudioAssetSlot::SoundEffect, - task.kind, - build_editor_audio_target( - &task.task_id, - "sound_effect", - "editor_sound_effect", - Some(u64::from(normalized.price_mud_points)), - ), - ) - .await - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; - let audio_src = generated - .audio_src - .ok_or_else(|| vector_engine_bad_gateway("音效生成完成但缺少播放地址")) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; - let (resource, asset) = persist_editor_generated_media_asset( - &state, - PersistEditorGeneratedAssetRequest { - project_id: project_id.clone(), - owner_user_id: owner_user_id.clone(), - folder_id: asset_folder_id, - label: asset_label - .and_then(|value| { - let trimmed = value.trim().to_string(); - (!trimmed.is_empty()).then_some(trimmed) - }) - .unwrap_or_else(|| "生成音效".to_string()), - image_src: audio_src.clone(), - object_key: generated.object_key.clone(), - asset_object_id: generated.asset_object_id.clone(), - width: EDITOR_AUDIO_WIDTH, - height: EDITOR_AUDIO_HEIGHT, - prompt: normalized.prompt.clone(), - actual_prompt: Some(normalized.prompt.clone()), - model: normalized.model.clone(), - provider: generated.provider.clone(), - task_id: generated.task_id.clone(), - source_resource_id: None, - asset_kind: Some("sound-effect".to_string()), - generation_inputs: generation_inputs.clone(), - thumbnail_src: None, - generation_cost_mud_points: u64::from(normalized.price_mud_points), - image_sequence_frames: None, - image_sequence_duration_ms: None, - }, - ) - .await - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; - let completed_project = if let Some(completion) = canvas_completion.as_ref() { - let layer_id = format!("layer-editor-sound-effect-{}", generated.task_id); - let resource_id = resource - .as_ref() - .map(|resource| resource.resource_id.clone()) - .unwrap_or_else(|| format!("local-resource-editor-sound-effect-{}", generated.task_id)); - let item = build_editor_canvas_generated_layer_item( - completion, - EditorCanvasGeneratedLayerInput { - layer_id: layer_id.clone(), - resource_id, - title: completion.title.trim().to_string(), - src: audio_src.clone(), - thumbnail_src: None, - original_width: EDITOR_AUDIO_WIDTH, - original_height: EDITOR_AUDIO_HEIGHT, - display_width: completion.placeholder.width, - display_height: completion.placeholder.height, - source_type: "generated".to_string(), - prompt: Some(normalized.prompt.clone()), - actual_prompt: Some(normalized.prompt.clone()), - model: Some(normalized.model.clone()), - provider: Some(generated.provider.clone()), - task_id: Some(generated.task_id.clone()), - object_key: generated.object_key.clone(), - asset_object_id: generated.asset_object_id.clone(), - source_resource_id: None, - asset_kind: Some("sound-effect".to_string()), - generation_inputs, - }, - ); - complete_editor_canvas_generation_with_items( - &state, - owner_user_id.as_str(), - project_id.as_deref(), - canvas_completion.as_ref(), - vec![item], - Some(layer_id), - ) - .await - .map_err(|error| error.into_response_with_context(Some(&request_context)))? - } else { - None - }; - let completed_project = - completed_project.and_then(|project| serde_json::to_value(project).ok()); - let resource = resource.and_then(|resource| serde_json::to_value(resource).ok()); - let asset = asset.and_then(|asset| serde_json::to_value(asset).ok()); + target.asset_kind.as_str(), + billing_asset_id.as_str(), + u64::from(normalized.price_mud_points), + async { + let actual_prompt = + translate_sound_effect_prompt_for_worker(llm_client, normalized.prompt.as_str()) + .await + .map_err(|error| { + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": "editor-sound-effect-translation", + "reason": error.reason_code(), + "message": error.to_string(), + })) + })?; + let generated = platform_audio::generate_elevenlabs_sound_effect( + &http_client, + &settings, + platform_audio::ElevenLabsSoundEffectRequest { + text: actual_prompt.clone(), + duration_seconds: normalized.duration, + loop_enabled: normalized.loop_enabled, + }, + ) + .await + .map_err(map_platform_audio_error)?; + let duration_mode = if normalized.duration.is_some() { + assets::EditorSoundEffectDurationMode::Manual + } else { + assets::EditorSoundEffectDurationMode::Auto + }; + let metadata = assets::EditorSoundEffectGenerationMetadataV2::try_new( + normalized.prompt.clone(), + actual_prompt.clone(), + duration_mode, + normalized.duration, + generated.duration_seconds, + normalized.loop_enabled, + ) + .map_err(|message| { + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": platform_audio::ELEVENLABS_PROVIDER, + "message": message, + })) + })?; + let generation_inputs = build_editor_sound_effect_generation_inputs( + payload.generation_inputs.as_ref(), + &metadata, + )?; + let persisted_audio = persist_generated_audio_asset( + &state, + state.editor_oss_http_client(), + owner_user_id.as_str(), + task_id.as_str(), + platform_audio::GeneratedAudioPersistSource { + provider: platform_audio::ELEVENLABS_PROVIDER.to_string(), + file_stem: "sound-effect".to_string(), + }, + target.clone(), + generated.audio, + ) + .await?; + let audio_src = persisted_audio.audio_src; + let object_key = Some(persisted_audio.object_key); + let asset_object_id = Some(persisted_audio.asset_object_id); + let project_id = payload.project_id.clone(); + let canvas_completion = payload.canvas_completion.clone(); + let (resource, asset) = persist_editor_generated_media_asset( + &state, + PersistEditorGeneratedAssetRequest { + project_id: project_id.clone(), + owner_user_id: owner_user_id.clone(), + folder_id: payload.asset_folder_id.clone(), + label: payload + .asset_label + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| "生成音效".to_string()), + image_src: audio_src.clone(), + object_key: object_key.clone(), + asset_object_id: asset_object_id.clone(), + width: EDITOR_AUDIO_WIDTH, + height: EDITOR_AUDIO_HEIGHT, + prompt: normalized.prompt.clone(), + actual_prompt: Some(actual_prompt.clone()), + model: normalized.model.clone(), + provider: platform_audio::ELEVENLABS_PROVIDER.to_string(), + task_id: task_id.clone(), + source_resource_id: None, + asset_kind: Some("sound-effect".to_string()), + generation_inputs: Some(generation_inputs.clone()), + thumbnail_src: None, + generation_cost_mud_points: u64::from(normalized.price_mud_points), + image_sequence_frames: None, + image_sequence_duration_ms: None, + }, + ) + .await?; + let completed_project = if let Some(completion) = canvas_completion.as_ref() { + let layer_id = format!("layer-editor-sound-effect-{task_id}"); + let resource_id = resource + .as_ref() + .map(|resource| resource.resource_id.clone()) + .unwrap_or_else(|| format!("local-resource-editor-sound-effect-{task_id}")); + let item = build_editor_canvas_generated_layer_item( + completion, + EditorCanvasGeneratedLayerInput { + layer_id: layer_id.clone(), + resource_id, + title: completion.title.trim().to_string(), + src: audio_src.clone(), + thumbnail_src: None, + original_width: EDITOR_AUDIO_WIDTH, + original_height: EDITOR_AUDIO_HEIGHT, + display_width: completion.placeholder.width, + display_height: completion.placeholder.height, + source_type: "generated".to_string(), + prompt: Some(normalized.prompt.clone()), + actual_prompt: Some(actual_prompt.clone()), + model: Some(normalized.model.clone()), + provider: Some(platform_audio::ELEVENLABS_PROVIDER.to_string()), + task_id: Some(task_id.clone()), + object_key: object_key.clone(), + asset_object_id: asset_object_id.clone(), + source_resource_id: None, + asset_kind: Some("sound-effect".to_string()), + generation_inputs: Some(generation_inputs), + }, + ); + complete_editor_canvas_generation_with_items( + &state, + owner_user_id.as_str(), + project_id.as_deref(), + canvas_completion.as_ref(), + vec![item], + Some(layer_id), + ) + .await? + } else { + None + }; + let completed_project = + completed_project.and_then(|project| serde_json::to_value(project).ok()); + let resource = resource.and_then(|resource| serde_json::to_value(resource).ok()); + let asset = asset.and_then(|asset| serde_json::to_value(asset).ok()); - Ok(json_success_body( - Some(&request_context), - assets::EditorAudioGenerateResponse { - ok: true, - audio_src, - object_key: generated.object_key, - asset_object_id: generated.asset_object_id, - width: EDITOR_AUDIO_WIDTH, - height: EDITOR_AUDIO_HEIGHT, - source_type: "generated".to_string(), - prompt: normalized.prompt.clone(), - actual_prompt: Some(normalized.prompt), - model: normalized.model, - provider: generated.provider, - task_id: generated.task_id, - price_mud_points: normalized.price_mud_points, - audio_kind: "sound-effect".to_string(), - duration_seconds: None, - loop_enabled: None, - project: completed_project, - resource, - asset, - queue_state: None, + Ok(assets::EditorAudioGenerateResponse { + ok: true, + audio_src, + object_key, + asset_object_id, + width: EDITOR_AUDIO_WIDTH, + height: EDITOR_AUDIO_HEIGHT, + source_type: "generated".to_string(), + prompt: normalized.prompt.clone(), + actual_prompt: Some(actual_prompt), + model: normalized.model.clone(), + provider: platform_audio::ELEVENLABS_PROVIDER.to_string(), + task_id: task_id.clone(), + price_mud_points: normalized.price_mud_points, + audio_kind: "sound-effect".to_string(), + duration_seconds: Some(generated.duration_seconds), + loop_enabled: Some(normalized.loop_enabled), + project: completed_project, + resource, + asset, + queue_state: None, + }) }, - )) + ) + .await + .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + + Ok(json_success_body(Some(&request_context), response)) +} + +fn build_editor_sound_effect_generation_inputs( + original: Option<&Value>, + metadata: &assets::EditorSoundEffectGenerationMetadataV2, +) -> Result { + let mut result = serde_json::Map::new(); + if let Some(original) = original.and_then(Value::as_object) { + for key in ["source", "conversationId", "toolCallMessageId"] { + if let Some(value) = original.get(key) { + result.insert(key.to_string(), value.clone()); + } + } + } + result.insert( + "fields".to_string(), + json!([ + { "title": "用户描述", "value": metadata.user_prompt() }, + { "title": "实际英文提示词", "value": metadata.actual_prompt() }, + { "title": "时长", "value": format!("{}秒", metadata.actual_duration_seconds()) }, + { "title": "Loop", "value": if metadata.loop_enabled() { "开启" } else { "关闭" } }, + ]), + ); + result.insert("references".to_string(), json!([])); + result.insert( + "soundEffect".to_string(), + serde_json::to_value(metadata).map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ + "provider": "editor-sound-effect-metadata", + "message": format!("构造音效生成元数据失败:{error}"), + })) + })?, + ); + Ok(Value::Object(result)) } pub async fn generate_editor_background_music( @@ -753,13 +868,15 @@ fn editor_audio_bad_request(message: impl Into) -> AppError { #[cfg(test)] mod tests { use serde::Deserialize; - use serde_json::Value; + use serde_json::{Value, json}; use shared_contracts::assets; use super::{ CanonicalEditorBackgroundMusicSubmissionPayload, NormalizedEditorBackgroundMusicRequest, - build_editor_background_music_generate_response, normalize_editor_background_music_request, + build_editor_background_music_generate_response, + build_editor_sound_effect_generation_inputs, normalize_editor_background_music_request, normalize_editor_sound_effect_request, prepare_editor_background_music_queue_job, + prepare_editor_sound_effect_queue_job, }; use crate::editor_generation_queue::serialize_editor_generation_job_payload_for_test; @@ -802,13 +919,14 @@ mod tests { fn sound_effect_payload( prompt: impl Into, model: Option<&str>, - duration: u8, + duration: Option, + loop_enabled: bool, ) -> assets::EditorSoundEffectGenerateRequest { assets::EditorSoundEffectGenerateRequest { prompt: prompt.into(), model: model.map(str::to_string), - duration: Some(f64::from(duration)), - loop_enabled: false, + duration, + loop_enabled, project_id: Some("project-1".to_string()), canvas_completion: None, generation_inputs: None, @@ -940,32 +1058,37 @@ mod tests { let normalized = normalize_editor_sound_effect_request(sound_effect_payload( " 金币掉落叮当声 ", None, - 7, + Some(7.5), + true, )) - .expect("default Vidu sound effect request should normalize"); + .expect("default ElevenLabs sound effect request should normalize"); assert_eq!(normalized.prompt, "金币掉落叮当声"); - assert_eq!(normalized.model, platform_audio::VIDU_AUDIO_MODEL); - assert_eq!(normalized.duration, 7); + assert_eq!(normalized.model, assets::EDITOR_SOUND_EFFECT_MODEL); + assert_eq!(normalized.duration, Some(7.5)); + assert!(normalized.loop_enabled); assert_eq!( normalized.price_mud_points, - pricing.sound_effect_model_mud_points(Some(platform_audio::VIDU_AUDIO_MODEL)) + pricing.sound_effect_model_mud_points(Some(assets::EDITOR_SOUND_EFFECT_MODEL)) ); let explicit_model = normalize_editor_sound_effect_request(sound_effect_payload( "按钮确认短促音", - Some(" audio1.0 "), - 5, + Some(" eleven_text_to_sound_v2 "), + None, + false, )) - .expect("the only supported Vidu model should be accepted"); - assert_eq!(explicit_model.model, platform_audio::VIDU_AUDIO_MODEL); + .expect("the only supported ElevenLabs model should be accepted"); + assert_eq!(explicit_model.model, assets::EDITOR_SOUND_EFFECT_MODEL); + assert_eq!(explicit_model.duration, None); let error = normalize_editor_sound_effect_request(sound_effect_payload( "按钮确认短促音", - Some(platform_audio::SUNO_DEFAULT_MODEL), - 5, + Some(platform_audio::VIDU_AUDIO_MODEL), + Some(5.0), + false, )) - .expect_err("Suno text-to-sound should remain unavailable for editor SFX"); + .expect_err("legacy Vidu model should be rejected for new editor SFX"); assert_eq!(error.status_code(), axum::http::StatusCode::BAD_REQUEST); } @@ -975,34 +1098,134 @@ mod tests { let normalized = normalize_editor_sound_effect_request(sound_effect_payload( format!(" {max_prompt} "), None, - 2, + Some(0.5), + false, )) - .expect("2048 canonical code points and the 2-second boundary should pass"); + .expect("2048 canonical code points and the 0.5-second boundary should pass"); assert_eq!(normalized.prompt, max_prompt); - assert_eq!(normalized.duration, 2); + assert_eq!(normalized.duration, Some(0.5)); - let normalized = - normalize_editor_sound_effect_request(sound_effect_payload("环境循环音", None, 10)) - .expect("the 10-second boundary should pass"); - assert_eq!(normalized.duration, 10); + let normalized = normalize_editor_sound_effect_request(sound_effect_payload( + "环境循环音", + None, + Some(30.0), + true, + )) + .expect("the 30-second boundary should pass"); + assert_eq!(normalized.duration, Some(30.0)); + assert!(normalized.loop_enabled); + + let precise = normalize_editor_sound_effect_request(sound_effect_payload( + "精确时长音效", + None, + Some(1.234_567_89), + false, + )) + .expect("server should preserve provider-supported decimal precision"); + assert_eq!(precise.duration, Some(1.234_567_89)); let overlong_prompt = "声".repeat(platform_audio::SOUND_EFFECT_PROMPT_MAX_CODE_POINTS + 1); - let error = - normalize_editor_sound_effect_request(sound_effect_payload(overlong_prompt, None, 5)) - .expect_err("2049 canonical code points should fail"); + let error = normalize_editor_sound_effect_request(sound_effect_payload( + overlong_prompt, + None, + Some(5.0), + false, + )) + .expect_err("2049 canonical code points should fail"); assert_eq!(error.status_code(), axum::http::StatusCode::BAD_REQUEST); - for duration in [1, 11] { + for duration in [0.49, 30.01] { let error = normalize_editor_sound_effect_request(sound_effect_payload( "环境循环音", None, - duration, + Some(duration), + false, )) - .expect_err("duration outside the existing 2-10 range should fail"); + .expect_err("duration outside 0.5-30 seconds should fail"); assert_eq!(error.status_code(), axum::http::StatusCode::BAD_REQUEST); } } + #[test] + fn editor_sound_effect_queue_payload_is_canonical_before_idempotency() { + let pricing = + crate::editor_generation_config::load_editor_generation_pricing_from_paths(None) + .expect("default model pricing should be valid"); + let accepted_models = [ + None, + Some(""), + Some(" \u{0085} "), + Some(" eleven_text_to_sound_v2 "), + ]; + let mut payloads = Vec::new(); + for model in accepted_models { + let prepared = prepare_editor_sound_effect_queue_job( + sound_effect_payload("\u{0085}按钮点击声\u{2003}", model, None, false), + &pricing, + ) + .expect("accepted model form should prepare"); + let serialized = serialize_editor_generation_job_payload_for_test(&prepared.payload) + .expect("canonical queue payload should serialize"); + let payload: Value = serde_json::from_str(&serialized).expect("payload should be JSON"); + assert_eq!(payload["prompt"], "按钮点击声"); + assert_eq!(payload["model"], assets::EDITOR_SOUND_EFFECT_MODEL); + assert_eq!(payload["duration"], Value::Null); + assert_eq!(payload["loop"], false); + payloads.push(serialized); + } + assert!(payloads.windows(2).all(|pair| pair[0] == pair[1])); + + for model in ["audio1.0", "eleven_text_to_sound_v3"] { + let error = prepare_editor_sound_effect_queue_job( + sound_effect_payload("按钮点击声", Some(model), Some(5.0), false), + &pricing, + ) + .expect_err("legacy and unknown model must fail before enqueue"); + assert_eq!(error.status_code(), axum::http::StatusCode::BAD_REQUEST); + } + } + + #[test] + fn editor_sound_effect_generation_inputs_replace_untrusted_authoritative_fields() { + let metadata = assets::EditorSoundEffectGenerationMetadataV2::try_new( + "金币落地".to_string(), + "A bright coin landing chime".to_string(), + assets::EditorSoundEffectDurationMode::Auto, + None, + 7.42, + true, + ) + .expect("metadata should build"); + let generation_inputs = build_editor_sound_effect_generation_inputs( + Some(&json!({ + "source": "editor-agent", + "conversationId": "conversation-1", + "toolCallMessageId": 7, + "fields": [{ "title": "实际英文提示词", "value": "伪造值" }], + "soundEffect": { "actualPrompt": "forged" }, + "unknown": "drop-me" + })), + &metadata, + ) + .expect("server metadata should serialize"); + + assert_eq!(generation_inputs["source"], "editor-agent"); + assert_eq!(generation_inputs["conversationId"], "conversation-1"); + assert_eq!(generation_inputs["toolCallMessageId"], 7); + assert_eq!(generation_inputs["fields"][0]["value"], "金币落地"); + assert_eq!( + generation_inputs["fields"][1]["value"], + "A bright coin landing chime" + ); + assert_eq!(generation_inputs["fields"][2]["value"], "7.42秒"); + assert_eq!(generation_inputs["fields"][3]["value"], "开启"); + assert_eq!( + generation_inputs["soundEffect"]["actualPrompt"], + "A bright coin landing chime" + ); + assert!(generation_inputs.get("unknown").is_none()); + } + #[test] fn external_background_music_queue_preparation_preserves_the_raw_payload() { let pricing = diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/persist.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/persist.rs index 8de160dd8..b992e9b65 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation/persist.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/persist.rs @@ -14,7 +14,7 @@ use crate::{http_error::AppError, platform_errors::map_oss_error, state::AppStat use super::{ clock::current_utc_micros, errors::{map_asset_field_error, map_spacetime_error}, - types::{AudioAssetBindingTarget, AudioAssetSlot}, + types::AudioAssetBindingTarget, }; #[derive(Clone, Debug)] @@ -29,8 +29,7 @@ pub(super) async fn persist_generated_audio_asset( http_client: &reqwest::Client, owner_user_id: &str, task_id: &str, - _slot: AudioAssetSlot, - task_kind: platform_audio::AudioTaskKind, + source: GeneratedAudioPersistSource, target: AudioAssetBindingTarget, audio: DownloadedAudio, ) -> Result { @@ -46,7 +45,7 @@ pub(super) async fn persist_generated_audio_asset( platform_audio::prepare_generated_audio_put_request(GeneratedAudioPersistInput { owner_user_id: owner_user_id.to_string(), task_id: task_id.to_string(), - source: GeneratedAudioPersistSource::from_task_kind(task_kind), + source, target: GeneratedAudioPersistTarget { entity_kind: target.entity_kind.clone(), entity_id: target.entity_id.clone(), diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/publish.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/publish.rs index 5b50e846c..bfe4ec168 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation/publish.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/publish.rs @@ -104,8 +104,7 @@ pub(super) async fn publish_generated_audio_asset_with_task_kind( &http_client, owner_user_id, &task_id, - slot, - task_kind, + platform_audio::GeneratedAudioPersistSource::from_task_kind(task_kind), target.clone(), audio, ) diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/settings.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/settings.rs index 4bf401048..00e828dc5 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation/settings.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/settings.rs @@ -43,7 +43,6 @@ pub(super) fn require_vector_engine_audio_settings( }) } -#[allow(dead_code)] pub(super) fn require_elevenlabs_audio_settings( state: &AppState, ) -> Result { diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs index 8831789e5..79daa4032 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs @@ -1,6 +1,6 @@ use crate::agent::prompt::PENDING_USER_CONFIRMATION_MESSAGE; use crate::framework::tool::{Tool, ToolFailure}; -use platform_audio::VIDU_AUDIO_MODEL; +use platform_audio::ELEVENLABS_SOUND_EFFECT_MODEL; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use shared_contracts::assets::EditorAudioGenerateResponse; @@ -21,7 +21,7 @@ impl Display for GenerateSoundEffectError { match self { Self::InvalidModel(model) => write!( f, - "{model} is not a valid sound effect model; only {VIDU_AUDIO_MODEL} is supported" + "{model} is not a valid sound effect model; only {ELEVENLABS_SOUND_EFFECT_MODEL} is supported" ), Self::InvalidDuration(duration) => write!( f, @@ -49,7 +49,7 @@ pub struct GenerateSoundEffectToolArgs { } fn default_sound_effect_model() -> String { - VIDU_AUDIO_MODEL.to_string() + ELEVENLABS_SOUND_EFFECT_MODEL.to_string() } fn default_sound_effect_duration() -> u8 { @@ -76,7 +76,7 @@ impl Tool for GenerateSoundEffectTool { "type": "object", "properties": { "prompt": { "type": "string", "description": "音效内容、材质、节奏和情绪描述。" }, - "model": { "type": "string", "enum": [VIDU_AUDIO_MODEL], "default": VIDU_AUDIO_MODEL, "description": "音效模型。" }, + "model": { "type": "string", "enum": [ELEVENLABS_SOUND_EFFECT_MODEL], "default": ELEVENLABS_SOUND_EFFECT_MODEL, "description": "音效固定使用 ElevenLabs。" }, "duration": { "type": "integer", "enum": GenerateSoundEffectTool::SUPPORTED_DURATIONS, "default": GenerateSoundEffectTool::DEFAULT_DURATION, "description": "音效时长(秒)。" }, }, "required": ["prompt"], @@ -106,7 +106,7 @@ impl Tool for GenerateSoundEffectTool { } impl GenerateSoundEffectTool { - pub const DEFAULT_MODEL: &'static str = VIDU_AUDIO_MODEL; + pub const DEFAULT_MODEL: &'static str = ELEVENLABS_SOUND_EFFECT_MODEL; pub const DEFAULT_DURATION: u8 = 5; pub const SUPPORTED_DURATIONS: &'static [u8] = &[2, 3, 4, 5, 6, 7, 8, 9, 10]; @@ -117,7 +117,7 @@ impl GenerateSoundEffectTool { if args.prompt.trim().is_empty() { return Err(GenerateSoundEffectError::PromptNotProvided); } - if args.model != VIDU_AUDIO_MODEL { + if args.model != ELEVENLABS_SOUND_EFFECT_MODEL { return Err(GenerateSoundEffectError::InvalidModel(args.model.clone())); } if !Self::SUPPORTED_DURATIONS.contains(&args.duration) { diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs index 1c26f27a7..a9205f524 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs @@ -25,7 +25,7 @@ mod tests { use super::generate_ui_design::{GenerateUiDesignTool, GenerateUiDesignToolArgs}; use super::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; use crate::framework::tool::Tool; - use platform_audio::{SUNO_DEFAULT_MODEL, VIDU_AUDIO_MODEL}; + use platform_audio::{ELEVENLABS_SOUND_EFFECT_MODEL, SUNO_DEFAULT_MODEL}; use platform_image::{GPT_IMAGE_2_MODEL, NANOBANANA_2_MODEL}; use serde_json::json; @@ -84,7 +84,7 @@ mod tests { assert_eq!(video.duration_seconds, 4); assert_eq!(video.resolution, "720p"); assert_eq!(video.sound, "on"); - assert_eq!(sound.model, VIDU_AUDIO_MODEL); + assert_eq!(sound.model, ELEVENLABS_SOUND_EFFECT_MODEL); assert_eq!(sound.duration, 5); assert_eq!(music.model, SUNO_DEFAULT_MODEL); } diff --git a/server-rs/crates/shared-contracts/src/assets.rs b/server-rs/crates/shared-contracts/src/assets.rs index 953bc4bd9..c1dd65e71 100644 --- a/server-rs/crates/shared-contracts/src/assets.rs +++ b/server-rs/crates/shared-contracts/src/assets.rs @@ -515,6 +515,19 @@ pub struct EditorIconSpritesheetGenerateResponse { pub const EDITOR_SOUND_EFFECT_MODEL: &str = "eleven_text_to_sound_v2"; +pub fn canonicalize_editor_sound_effect_model( + value: Option<&str>, +) -> Result<&'static str, &'static str> { + let normalized = value + .map(|value| value.trim_matches(char::is_whitespace)) + .filter(|value| !value.is_empty()); + match normalized { + None => Ok(EDITOR_SOUND_EFFECT_MODEL), + Some(EDITOR_SOUND_EFFECT_MODEL) => Ok(EDITOR_SOUND_EFFECT_MODEL), + Some(_) => Err("model 只支持 eleven_text_to_sound_v2"), + } +} + #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub enum EditorSoundEffectDurationMode { @@ -652,7 +665,7 @@ pub struct EditorSoundEffectGenerateRequest { pub prompt: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub model: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(default)] pub duration: Option, #[serde(default, rename = "loop")] pub loop_enabled: bool, @@ -1582,7 +1595,7 @@ mod tests { }) .expect("sound request with unset model should serialize"); assert!(unset_model_payload.get("model").is_none()); - assert!(unset_model_payload.get("duration").is_none()); + assert_eq!(unset_model_payload["duration"], json!(null)); assert_eq!(unset_model_payload["loop"], json!(false)); for nullable_payload in [ @@ -1667,6 +1680,36 @@ mod tests { ); } + #[test] + fn editor_sound_effect_model_canonicalizer_freezes_external_v1_inputs() { + for accepted in [ + None, + Some(""), + Some(" \t\r\n"), + Some("\u{0085}\u{2003}\u{00a0}"), + Some(EDITOR_SOUND_EFFECT_MODEL), + Some("\u{0085} eleven_text_to_sound_v2 \u{2003}"), + ] { + assert_eq!( + canonicalize_editor_sound_effect_model(accepted) + .expect("accepted model form should canonicalize"), + EDITOR_SOUND_EFFECT_MODEL, + "accepted={accepted:?}" + ); + } + + for rejected in [ + "audio1.0", + "AUDIO1.0", + "Eleven_text_to_sound_v2", + "eleven_text_to_sound_v3", + "\u{200b}", + "\u{feff}", + ] { + assert!(canonicalize_editor_sound_effect_model(Some(rejected)).is_err()); + } + } + #[test] fn editor_sound_effect_v2_metadata_uses_the_frozen_camel_case_shape() { let metadata = EditorSoundEffectGenerationMetadataV2::try_new( diff --git a/server-rs/crates/spacetime-module/src/editor_project_storage.rs b/server-rs/crates/spacetime-module/src/editor_project_storage.rs index 0058006b0..70b8daee6 100644 --- a/server-rs/crates/spacetime-module/src/editor_project_storage.rs +++ b/server-rs/crates/spacetime-module/src/editor_project_storage.rs @@ -65,7 +65,8 @@ const EDITOR_LEGACY_GREEN_SCREEN_SOURCE_ASSET_KIND: &str = "editor_green_screen_ const EDITOR_MANUAL_ATLAS_SPLIT_TASK_PREFIX: &str = "editor-atlas-split-"; const EDITOR_GENERATION_IMAGE_MODEL_NANOBANANA2: &str = "gemini-3.1-flash-image-preview"; const EDITOR_GENERATION_IMAGE_MODEL_GPT_IMAGE_2: &str = "gpt-image-2"; -const EDITOR_GENERATION_SOUND_EFFECT_MODEL: &str = "audio1.0"; +const EDITOR_GENERATION_SOUND_EFFECT_MODEL_VIDU: &str = "audio1.0"; +const EDITOR_GENERATION_SOUND_EFFECT_MODEL_ELEVENLABS: &str = "eleven_text_to_sound_v2"; const EDITOR_GENERATION_BACKGROUND_MUSIC_MODEL: &str = "chirp-v5"; const EDITOR_GENERATION_VIDEO_MODELS: [&str; 6] = [ "seedance2.0-fast", @@ -6156,7 +6157,12 @@ fn validate_required_editor_generation_pricing( } validate_required_editor_generation_flat_price( &models_by_name, - EDITOR_GENERATION_SOUND_EFFECT_MODEL, + EDITOR_GENERATION_SOUND_EFFECT_MODEL_VIDU, + EDITOR_GENERATION_PRICING_UNIT_PER_GENERATION, + )?; + validate_required_editor_generation_flat_price( + &models_by_name, + EDITOR_GENERATION_SOUND_EFFECT_MODEL_ELEVENLABS, EDITOR_GENERATION_PRICING_UNIT_PER_GENERATION, )?; validate_required_editor_generation_flat_price( @@ -15244,7 +15250,8 @@ mod tests { EDITOR_GENERATION_PRICING_UNIT_PER_GENERATION, &EDITOR_GENERATION_GPT_IMAGE_2_TIERS, ), - flat_pricing(EDITOR_GENERATION_SOUND_EFFECT_MODEL), + flat_pricing(EDITOR_GENERATION_SOUND_EFFECT_MODEL_VIDU), + flat_pricing(EDITOR_GENERATION_SOUND_EFFECT_MODEL_ELEVENLABS), flat_pricing(EDITOR_GENERATION_BACKGROUND_MUSIC_MODEL), ]; models.extend(EDITOR_GENERATION_VIDEO_MODELS.map(|model| { @@ -15385,8 +15392,11 @@ mod tests { normalize_editor_generation_pricing_models(valid_editor_generation_pricing_models()) .expect("complete pricing matrix should normalize"); - assert_eq!(normalized.len(), 10); - assert_eq!(normalized[0].model, EDITOR_GENERATION_SOUND_EFFECT_MODEL); + assert_eq!(normalized.len(), 11); + assert_eq!( + normalized[0].model, + EDITOR_GENERATION_SOUND_EFFECT_MODEL_VIDU + ); } #[test] diff --git a/src/components/image-editor/ImageCanvasGenerationDialogModel.test.ts b/src/components/image-editor/ImageCanvasGenerationDialogModel.test.ts index 070b0ad95..368e66af2 100644 --- a/src/components/image-editor/ImageCanvasGenerationDialogModel.test.ts +++ b/src/components/image-editor/ImageCanvasGenerationDialogModel.test.ts @@ -376,6 +376,46 @@ describe('ImageCanvasGenerationDialogModel', () => { }); }); + it('restores authoritative SFX V2 prompt, duration mode, and loop on redraw', () => { + const draft = createAudioRedrawGenerationDialogDraft( + createLayer({ + id: 'layer-sfx-v2', + mediaType: 'audio', + assetKind: 'sound-effect', + prompt: '不应覆盖权威用户描述', + generationInputs: { + fields: [ + { title: '用户描述', value: '金币落地' }, + { title: '实际英文提示词', value: 'A bright coin landing chime' }, + { title: '时长', value: '7.42秒' }, + { title: 'Loop', value: '开启' }, + ], + references: [], + soundEffect: { + schemaVersion: 2, + userPrompt: '金币落地', + actualPrompt: 'A bright coin landing chime', + model: 'eleven_text_to_sound_v2', + durationMode: 'auto', + requestedDurationSeconds: null, + actualDurationSeconds: 7.42, + loop: true, + }, + }, + }), + ); + + expect(draft).toMatchObject({ + mode: 'audio-sound-effect', + sourceLayerId: 'layer-sfx-v2', + prompt: '金币落地', + soundModel: 'eleven_text_to_sound_v2', + soundDurationMode: 'auto', + soundDurationSeconds: 5, + soundLoop: true, + }); + }); + it('creates edit, quick-edit, and character animation panel drafts', () => { const sourceLayer = createLayer({ prompt: '【系统内置】完整拼接提示词:原图提示,自动画质优化。', diff --git a/src/components/image-editor/ImageCanvasGenerationDialogModel.ts b/src/components/image-editor/ImageCanvasGenerationDialogModel.ts index 0fbc876b6..956b475c0 100644 --- a/src/components/image-editor/ImageCanvasGenerationDialogModel.ts +++ b/src/components/image-editor/ImageCanvasGenerationDialogModel.ts @@ -472,6 +472,7 @@ const USER_PROMPT_INPUT_TITLES = new Set( '快速编辑提示词', '重绘提示词', '动作描述', + '用户描述', ].map((title) => title.toLowerCase()), ); @@ -508,6 +509,14 @@ function resolveUserGenerationPromptSnapshot(sourceLayer: CanvasLayer) { } function resolveAudioRedrawPrompt(sourceLayer: CanvasLayer) { + const soundEffect = sourceLayer.generationInputs?.soundEffect; + if ( + sourceLayer.assetKind === 'sound-effect' && + soundEffect?.schemaVersion === 2 && + soundEffect.userPrompt.trim() + ) { + return soundEffect.userPrompt; + } return resolveUserGenerationPromptSnapshot(sourceLayer); } @@ -534,6 +543,17 @@ function resolveAudioRedrawSoundModel( } function resolveAudioRedrawSoundDuration(sourceLayer: CanvasLayer) { + const soundEffect = sourceLayer.generationInputs?.soundEffect; + if (soundEffect?.schemaVersion === 2) { + if ( + soundEffect.durationMode === 'manual' && + typeof soundEffect.requestedDurationSeconds === 'number' && + Number.isFinite(soundEffect.requestedDurationSeconds) + ) { + return soundEffect.requestedDurationSeconds; + } + return DEFAULT_SOUND_EFFECT_DURATION_SECONDS; + } const fieldValue = findGenerationInputFieldValue(sourceLayer, [ '时长', 'duration', @@ -582,9 +602,16 @@ export function createAudioRedrawGenerationDialogDraft( return { ...baseDraft, soundModel: resolveAudioRedrawSoundModel(sourceLayer), - soundDurationMode: 'manual', + soundDurationMode: + sourceLayer.generationInputs?.soundEffect?.schemaVersion === 2 && + sourceLayer.generationInputs.soundEffect.durationMode === 'auto' + ? 'auto' + : 'manual', soundDurationSeconds: resolveAudioRedrawSoundDuration(sourceLayer), - soundLoop: false, + soundLoop: + sourceLayer.generationInputs?.soundEffect?.schemaVersion === 2 + ? sourceLayer.generationInputs.soundEffect.loop + : false, }; } diff --git a/src/components/image-editor/ImageCanvasGenerationLayerModel.ts b/src/components/image-editor/ImageCanvasGenerationLayerModel.ts index cdc1f7518..ee0806117 100644 --- a/src/components/image-editor/ImageCanvasGenerationLayerModel.ts +++ b/src/components/image-editor/ImageCanvasGenerationLayerModel.ts @@ -512,7 +512,8 @@ export function createAudioResultLayer({ objectKey: resource?.objectKey ?? generated.objectKey, assetObjectId: resource?.assetObjectId ?? generated.assetObjectId, sourceAssetId: asset?.assetId, - generationInputs: resource?.generationInputs ?? generationInputs, + generationInputs: + resource?.generationInputs ?? asset?.generationInputs ?? generationInputs, generatedAssetSnapshot: asset ?? undefined, }; } diff --git a/src/components/image-editor/ImageCanvasGenerationModel.test.ts b/src/components/image-editor/ImageCanvasGenerationModel.test.ts index 54522808f..c71601aff 100644 --- a/src/components/image-editor/ImageCanvasGenerationModel.test.ts +++ b/src/components/image-editor/ImageCanvasGenerationModel.test.ts @@ -541,12 +541,18 @@ describe('ImageCanvasGenerationModel', () => { 'Quick Edit Generator', ); expect( - buildSoundEffectGenerationInputs('金币掉落叮当声', 'audio1.0', 7), + buildSoundEffectGenerationInputs( + '金币掉落叮当声', + 'eleven_text_to_sound_v2', + 7, + true, + ), ).toEqual({ fields: [ - { title: 'prompt', value: '金币掉落叮当声' }, - { title: 'model', value: 'audio1.0' }, - { title: '时长', value: '7秒' }, + { title: '用户描述', value: '金币掉落叮当声' }, + { title: 'model', value: 'eleven_text_to_sound_v2' }, + { title: '请求时长', value: '7秒' }, + { title: 'Loop', value: '开启' }, ], references: [], }); diff --git a/src/components/image-editor/ImageCanvasGenerationModel.ts b/src/components/image-editor/ImageCanvasGenerationModel.ts index e6814225b..a47fa33d9 100644 --- a/src/components/image-editor/ImageCanvasGenerationModel.ts +++ b/src/components/image-editor/ImageCanvasGenerationModel.ts @@ -965,9 +965,7 @@ export function calculateEditorSoundEffectPrice( model: string | null | undefined, ) { const normalizedModel = (model ?? DEFAULT_SOUND_EFFECT_MODEL).trim(); - // T4 只切前端固定模型展示;新模型的动态定价键由 T5 同批接入。中间态继续以 - // 历史 Vidu 的 5 泥点前端兜底展示,服务端价格始终是正式真相。 - return readModelFlatPrice(normalizedModel, SOUND_EFFECT_MODEL_VIDU); + return readModelFlatPrice(normalizedModel, DEFAULT_SOUND_EFFECT_MODEL); } export function calculateEditorBackgroundMusicPrice( @@ -1160,13 +1158,18 @@ export function buildVideoGenerationInputs( export function buildSoundEffectGenerationInputs( prompt: string, model: string, - duration: number, + duration: number | null, + loop: boolean, ): CanvasGenerationInputs { return { fields: [ - ...createGenerationInputField('prompt', prompt), + ...createGenerationInputField('用户描述', prompt), ...createGenerationInputField('model', model), - ...createGenerationInputField('时长', `${duration}秒`), + ...createGenerationInputField( + '请求时长', + duration === null ? '自动' : `${duration}秒`, + ), + ...createGenerationInputField('Loop', loop ? '开启' : '关闭'), ], references: [], }; diff --git a/src/components/image-editor/ImageCanvasGenerationSubmissionModel.test.ts b/src/components/image-editor/ImageCanvasGenerationSubmissionModel.test.ts index f7bce4471..1c1cdf458 100644 --- a/src/components/image-editor/ImageCanvasGenerationSubmissionModel.test.ts +++ b/src/components/image-editor/ImageCanvasGenerationSubmissionModel.test.ts @@ -1015,6 +1015,7 @@ describe('ImageCanvasGenerationSubmissionModel', () => { prompt: ' 金币掉落叮当声 ', status: 'idle', soundDurationSeconds: 7, + soundDurationMode: 'manual', }, layers: [], nextGeneratedIndex: 4, @@ -1034,9 +1035,10 @@ describe('ImageCanvasGenerationSubmissionModel', () => { title: '游戏音效 4', generationInputs: { fields: [ - { title: 'prompt', value: '金币掉落叮当声' }, + { title: '用户描述', value: '金币掉落叮当声' }, { title: 'model', value: 'eleven_text_to_sound_v2' }, - { title: '时长', value: '7秒' }, + { title: '请求时长', value: '7秒' }, + { title: 'Loop', value: '关闭' }, ], references: [], }, @@ -1044,6 +1046,48 @@ describe('ImageCanvasGenerationSubmissionModel', () => { }); }); + it('preserves frozen auto duration, loop, and manual decimal precision', () => { + const automatic = buildImageGenerationSubmissionPlan({ + dialog: { + mode: 'audio-sound-effect', + prompt: '环境循环音', + status: 'idle', + soundDurationMode: 'auto', + soundDurationSeconds: 1.23456789, + soundLoop: true, + }, + layers: [], + nextGeneratedIndex: 1, + }); + expect(automatic).toMatchObject({ + input: { duration: null, loop: true }, + result: { + generationInputs: { + fields: expect.arrayContaining([ + { title: '请求时长', value: '自动' }, + { title: 'Loop', value: '开启' }, + ]), + }, + }, + }); + + const manual = buildImageGenerationSubmissionPlan({ + dialog: { + mode: 'audio-sound-effect', + prompt: '精确时长音效', + status: 'idle', + soundDurationMode: 'manual', + soundDurationSeconds: 1.23456789, + soundLoop: false, + }, + layers: [], + nextGeneratedIndex: 2, + }); + expect(manual).toMatchObject({ + input: { duration: 1.23456789, loop: false }, + }); + }); + it('rejects an all-whitespace sound effect prompt without a fallback', () => { expect(() => buildImageGenerationSubmissionPlan({ diff --git a/src/components/image-editor/ImageCanvasGenerationSubmissionModel.ts b/src/components/image-editor/ImageCanvasGenerationSubmissionModel.ts index 8ab4f64d6..5cf030cd0 100644 --- a/src/components/image-editor/ImageCanvasGenerationSubmissionModel.ts +++ b/src/components/image-editor/ImageCanvasGenerationSubmissionModel.ts @@ -545,9 +545,12 @@ export function buildImageGenerationSubmissionPlan({ if (dialog.mode === 'audio-sound-effect') { const soundModel = EDITOR_SOUND_EFFECT_MODEL; const durationSeconds = - typeof dialog.soundDurationSeconds === 'number' - ? Math.min(10, Math.max(2, Math.round(dialog.soundDurationSeconds))) - : DEFAULT_SOUND_EFFECT_DURATION_SECONDS; + dialog.soundDurationMode === 'auto' + ? null + : typeof dialog.soundDurationSeconds === 'number' + ? dialog.soundDurationSeconds + : DEFAULT_SOUND_EFFECT_DURATION_SECONDS; + const loop = dialog.soundLoop === true; return { kind: 'audio', audioKind: 'sound-effect', @@ -556,7 +559,7 @@ export function buildImageGenerationSubmissionPlan({ prompt: normalizedPrompt, model: soundModel, duration: durationSeconds, - loop: false, + loop, }, result: { title: resolveGenerationAssetLabel( @@ -567,6 +570,7 @@ export function buildImageGenerationSubmissionPlan({ normalizedPrompt, soundModel, durationSeconds, + loop, ), }, }; diff --git a/src/components/image-editor/ImageCanvasMetadataModalView.test.tsx b/src/components/image-editor/ImageCanvasMetadataModalView.test.tsx index 37b31d860..dd2c4a1cb 100644 --- a/src/components/image-editor/ImageCanvasMetadataModalView.test.tsx +++ b/src/components/image-editor/ImageCanvasMetadataModalView.test.tsx @@ -260,6 +260,61 @@ describe('ImageCanvasMetadataModalView', () => { expect(within(dialog).queryByText('420 x 120 px')).toBeNull(); }); + it('renders SFX V2 authoritative prompts, actual duration, loop, model, and full task id', () => { + render( + , + ); + + const dialog = screen.getByRole('dialog', { name: '音频信息' }); + for (const text of [ + '用户描述', + '金币落地', + '实际英文提示词', + 'A bright coin landing chime', + '7.42秒', + 'Loop', + '开启', + 'eleven_text_to_sound_v2', + 'task-sfx-v2-1234-abcd', + ]) { + expect(within(dialog).getByText(text)).toBeTruthy(); + } + }); + it('does not render a dialog when no layer is selected', () => { render(); diff --git a/src/components/image-editor/ImageCanvasMetadataModalView.tsx b/src/components/image-editor/ImageCanvasMetadataModalView.tsx index 09f17b62d..779157413 100644 --- a/src/components/image-editor/ImageCanvasMetadataModalView.tsx +++ b/src/components/image-editor/ImageCanvasMetadataModalView.tsx @@ -105,7 +105,11 @@ export function ImageCanvasMetadataModalView({ ) : null}
Task
-
{formatTaskIdForDisplay(layer.taskId)}
+
+ {layer.generationInputs?.soundEffect?.schemaVersion === 2 + ? layer.taskId || '-' + : formatTaskIdForDisplay(layer.taskId)} +
) : null} diff --git a/src/components/image-editor/useImageCanvasGenerationWorkflow.test.tsx b/src/components/image-editor/useImageCanvasGenerationWorkflow.test.tsx index bdb3556da..aaece8ec3 100644 --- a/src/components/image-editor/useImageCanvasGenerationWorkflow.test.tsx +++ b/src/components/image-editor/useImageCanvasGenerationWorkflow.test.tsx @@ -1655,12 +1655,13 @@ describe('useImageCanvasGenerationWorkflow', () => { height: 120, sourceType: 'generated', prompt: '金币跳出音', - actualPrompt: '金币跳出音', - model: 'audio1.0', - provider: 'VectorEngine', + actualPrompt: 'A bright coin pop sound', + model: 'eleven_text_to_sound_v2', + provider: 'elevenlabs', taskId: 'task-sound', audioKind: 'sound-effect', - durationSeconds: 4, + durationSeconds: 7.42, + loop: false, }); render(); @@ -1673,11 +1674,13 @@ describe('useImageCanvasGenerationWorkflow', () => { prompt: '金币跳出音', model: 'eleven_text_to_sound_v2', duration: 8, + loop: false, generationInputs: { fields: [ - { title: 'prompt', value: '金币跳出音' }, + { title: '用户描述', value: '金币跳出音' }, { title: 'model', value: 'eleven_text_to_sound_v2' }, - { title: '时长', value: '8秒' }, + { title: '请求时长', value: '8秒' }, + { title: 'Loop', value: '关闭' }, ], references: [], }, diff --git a/src/services/image-editor/editorProjectClient.test.ts b/src/services/image-editor/editorProjectClient.test.ts index bc41024f3..db7238e08 100644 --- a/src/services/image-editor/editorProjectClient.test.ts +++ b/src/services/image-editor/editorProjectClient.test.ts @@ -106,6 +106,7 @@ describe('editorProjectClient', () => { prices: { '480p': 13, '720p': 26, '1080p': 52 }, }, 'audio1.0': { unit: 'perGeneration', price: 15 }, + eleven_text_to_sound_v2: { unit: 'perGeneration', price: 16 }, 'chirp-v5': { unit: 'perGeneration', price: 9 }, }, }); @@ -113,6 +114,7 @@ describe('editorProjectClient', () => { const pricing = await loadEditorGenerationPricing(); expect(pricing.models['gpt-image-2']?.prices?.['1K']).toBe(29); + expect(pricing.models.eleven_text_to_sound_v2?.price).toBe(16); expect(requestJsonMock).toHaveBeenCalledWith( '/api/editor/generation-pricing', { method: 'GET' }, @@ -1797,10 +1799,7 @@ describe('editorProjectClient', () => { }), }), '生成游戏音效失败', - expect.objectContaining({ - timeoutMs: 1_200_000, - retry: editorRetryOptionsExpectation, - }), + { timeoutMs: 1_200_000 }, ); }); diff --git a/src/services/image-editor/editorProjectClient.ts b/src/services/image-editor/editorProjectClient.ts index 33b6ff077..ca8bb2ed5 100644 --- a/src/services/image-editor/editorProjectClient.ts +++ b/src/services/image-editor/editorProjectClient.ts @@ -1396,7 +1396,6 @@ export async function generateEditorSoundEffect( '生成游戏音效失败', { timeoutMs: 1_200_000, - retry: EDITOR_GENERATION_REQUEST_RETRY_OPTIONS, }, ); }