From cad861a13deeb08721ebbb29b994ec3fc7b292d5 Mon Sep 17 00:00:00 2001 From: Linghong Date: Fri, 7 Aug 2026 09:30:03 +0000 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=20SFX=20V2=20T6=20=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E4=B8=8E=E5=8F=91=E5=B8=83=E9=97=A8=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 抽出单一 SFX Worker 编排并覆盖计费、翻译、ElevenLabs、OSS 与写回 补齐稳定失败分类、退款和零副作用组合测试 修复画布 Agent 的自动时长、小数范围、Loop 与 Prompt 参数契约 同步测试证据、发布门禁和项目共享记忆 --- docs/README.md | 1 + ...计划】SFX生成优化V2.0任务拆解-2026-08-06.md | 10 +- .../shared-memory/decision-log.md | 8 + docs/project-memory/shared-memory/pitfalls.md | 7 + ...】server-rs与SpacetimeDB数据契约-2026-05-15.md | 2 +- ...】SFX生成优化V2.0T6测试与发布门禁-2026-08-07.md | 158 +++++ ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 + .../【编辑器】画布Agent对话面板-2026-07-03.md | 1 + ...编辑器】画板音乐生成入口设计-2026-06-18.md | 5 +- .../api-server/src/editor_agent/tool.rs | 72 +- .../src/vector_engine_audio_generation.rs | 1 + .../vector_engine_audio_generation/errors.rs | 3 + .../generation.rs | 416 +++++++----- .../sound_effect_worker.rs | 619 ++++++++++++++++++ .../crates/platform-audio/src/elevenlabs.rs | 10 +- server-rs/crates/platform-audio/src/error.rs | 22 + .../crates/platform-audio/tests/elevenlabs.rs | 24 +- .../src/agent/tools/generate_sound_effect.rs | 95 ++- .../src/agent/tools/mod.rs | 22 +- 19 files changed, 1224 insertions(+), 254 deletions(-) create mode 100644 docs/【实施记录】SFX生成优化V2.0T6测试与发布门禁-2026-08-07.md create mode 100644 server-rs/crates/api-server/src/vector_engine_audio_generation/sound_effect_worker.rs diff --git a/docs/README.md b/docs/README.md index ab190cd5e..7ab448d2c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,7 @@ - [图片画布编辑器前端拆分计划](./technical/【前端架构】图片画布编辑器前端拆分计划-2026-06-17.md) - [画板音乐生成入口设计](./【编辑器】画板音乐生成入口设计-2026-06-18.md) - [SFX 生成优化 V2.0 任务拆解](./project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md) +- [SFX 生成优化 V2.0 T6 测试与发布门禁](./【实施记录】SFX生成优化V2.0T6测试与发布门禁-2026-08-07.md) - [音频生成 Composer 恢复共享分流方案](./project-memory/plans/【前端重构】音频生成面板恢复共享分流方案-2026-08-06.md) - [BGM 提示词优化 T6 测试与发布门禁](./【实施记录】BGM生成提示词优化T6测试与发布门禁-2026-08-05.md) - [画布 Agent 对话面板](./【编辑器】画布Agent对话面板-2026-07-03.md) diff --git a/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md b/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md index ae3d675b7..40db30504 100644 --- a/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md +++ b/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md @@ -2,7 +2,7 @@ 日期：`2026-08-06` -状态：`T1–T5 已完成，可以继续进入 T6；T6 测试、灰度与发布门禁完成前仍不可发布` +状态：`T1–T6 工程实施已完成；生产配置确认、旧 Vidu 队列 drain、灰度和实际发布仍须按门禁人工执行` 开发分支：`feat/sound_opt` @@ -192,6 +192,10 @@ External v1 Rust handler、共享 DTO、OpenAPI、compact result 与 Agent Skill - 执行定向 TypeScript / Rust / OpenAPI、`npm run typecheck`、`npm run check:encoding`、`git diff --check`、`npm run check:spacetime-schema`、`npm run dev:api-server` + `/healthz`。 - 不将 mock 测试写成真实 provider 验收，不执行未授权付费生成。 +实施记录（`2026-08-07`）：T6 已完成工程侧测试缝、组合失败矩阵、跨入口补齐、稳定失败分类和发布 runbook。正式 SFX Worker 现由同一编排函数串联计费、翻译、ElevenLabs、OSS 和资源 / 素材 / 画布写回；生产 adapter 继续调用原实现，测试 adapter 覆盖自动 / 手动时长 × Loop、余额不足零外部副作用、翻译 / provider / MP3 / OSS / asset confirm-bind / 项目资源 / 账号素材 / 画布写回失败、一次退款和单 job 最多一次 provider POST。ElevenLabs HTTP / 无效音频 / 时长探测分别稳定归类为 `elevenlabs_http_failed / invalid_audio / duration_probe_failed`，OSS 与后续写回归类为 `oss_failed / writeback_failed`；普通用户继续只看到稳定短文案。 + +T6 盘点发现并修复画布 Agent 遗留的 Vidu 参数边界：`generate-sound-effect` 现与站内和 External v1 共用 canonical Prompt、固定模型、`duration = null | 0.5-30` 和 `loop`，显式 `duration: null` 不再被通用 null-default 兼容层错误恢复为手动 `5s`。完整验证和生产门禁记录见 [`docs/【实施记录】SFX生成优化V2.0T6测试与发布门禁-2026-08-07.md`](../../【实施记录】SFX生成优化V2.0T6测试与发布门禁-2026-08-07.md)。本阶段没有调用真实 LLM / ElevenLabs、没有执行付费生成、没有连接生产 SpacetimeDB，也没有执行发布；因此“工程 T6 完成”不等于“生产门禁已放行”。 + ## 依赖和发布列车 ```text @@ -213,7 +217,7 @@ T5 -> T6 | 预设 | 40 + 12、ID / label 唯一、文案等值、逗号追加、重复、清快照、超限保文 | | 优化 | Luna + Medium + completion tokens 总预算 `8192`，Chat wire 只含 `max_completion_tokens=8192`，唯一 JSON、布尔门禁、length 直接失败、content_filter、无 tool call、无候选泄漏、dialog / scope 迟到响应 | | 翻译 | 每轮 Luna + Low + completion tokens 总预算 `8192`，Chat wire 只含 `max_completion_tokens=8192`，中文 / 英文 / 中英混合输入，日文 / 韩文 / 西里尔候选，isEnglish + Script 门禁，2048 / 2049，首轮 length 唯一重试、第二轮 length 最终失败且 provider 0 次 | -| External model | omitted / null / 空串 / 纯空白 / 包围空白新模型 / 显式新模型共用幂等 payload；`audio1.0` / 未知值为 400 + 零副作用 | +| 跨入口 / External model | 登录态、External v1、画布 Agent 共用 canonical SFX queue payload；omitted / null / 空串 / 纯空白 / 包围空白新模型 / 显式新模型共用幂等 payload；`audio1.0` / 未知值为 400 + 零副作用 | | ElevenLabs | auto / manual × Loop false / true，固定 model / influence / format，Key 不泄漏，网络 / HTTP / body 失败均只有一次 POST | | 二进制与时长 | `40 MiB` 接受 / `40 MiB + 1 byte` 拒绝，Content-Length / chunked 超限、空 / HTML / JSON / 损坏 MP3、允许与 fallback MIME；有限正时长、30.5 / 60 / 600s 接受，>600s / NaN / 无穷拒绝 | | 持久化 | prompt / actual_prompt / model / provider / task / actual duration / Loop 权威等值，客户端伪造值失效 | @@ -226,7 +230,7 @@ T5 -> T6 - 不打印值地确认生产 `ELEVENLABS_BASE_URL / ELEVENLABS_API_KEY / ELEVENLABS_REQUEST_TIMEOUT_MS` 均已配置。 - 确认定价 override 包含 `eleven_text_to_sound_v2` 且价格已批准。 -- 只读查询 `editor_sound_effect_generation` 的 queued / running 旧 Vidu payload。非零时先 drain，不得让新 Worker 按 V2 nullable duration / Loop payload 解析旧任务。 +- 只读查询 `external_generation_job` 中 `job_kind = 'editor_sound_effect_generation'` 且 `status IN ('pending', 'running')` 的旧 Vidu payload。非零时先 drain，不得让新 Worker 按 V2 nullable duration / Loop payload 解析旧任务；命令必须显式指定 `--server` / `--server-url`。 - 先部署 api-server / worker，再部署 web；两者之间使用维护窗或暂时关闭 SFX 提交入口。 - External v1 变更提前通知调用方并完成 contract smoke。 diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 8fea34f08..e11e449c8 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -6780,3 +6780,11 @@ - 决策：SFX 的 `userPrompt` 与 `actualPrompt` 从首尾 Unicode `White_Space` 规则改为 ECMAScript `String.trim()` 语义。TypeScript 直接使用 `String.trim()`；Rust 以等值边界字符集合实现，不能使用语义不同的 Rust `str::trim()`。因此首尾 `U+FEFF` 删除、首尾 `U+0085` 保留，内部空白、内部 `U+FEFF`、`U+200B`、组合字符和 ZWJ emoji 继续保持原样。 - 范围：只影响 SFX Prompt 的输入、优化候选、Worker 翻译候选、正式请求、metadata 校验与 ElevenLabs body；BGM Prompt 以及 External v1 `model` 的 Unicode `White_Space` canonicalization 不变。 - 验证：共享 fixture 锁定 ECMAScript 全部首尾删除字符、`U+FEFF` 边界删除与内部保留、`U+0085` 边界保留、内部空白和 `1 / 2048 / 2049` code point；前后端必须共同消费该 fixture。 + +## 2026-08-07 SFX 生成优化 V2.0 T6 Worker 组合门禁 + +- 正式编排：SFX Worker 以同一个内部编排函数串联现有计费、翻译、ElevenLabs、OSS、asset confirm / bind、项目资源 / 账号素材 / 画布写回。生产 adapter 继续调用正式实现，测试 adapter 只替换外部边界；禁止另写与生产分叉的“测试专用业务流程”。 +- 失败与退款：余额不足时 Worker future 不得被 poll，LLM / ElevenLabs / OSS / 写回均为零；预扣后的翻译、provider、MP3、OSS 或写回失败全部一次退款。组合矩阵必须证明每个 job 的 ElevenLabs POST 最多一次、翻译最终失败 provider 为零、成功只扣费一次。 +- 分类：内部稳定 reason code 固定为 `translation_invalid / translation_upstream_failed / translation_budget_exhausted / elevenlabs_http_failed / invalid_audio / duration_probe_failed / oss_failed / writeback_failed`。MIME、空 body 和大小归 `invalid_audio`；MP3 识别、帧读取和时长门禁归 `duration_probe_failed`。普通用户继续只读稳定短文案，不暴露 endpoint、上游正文或凭据。 +- 跨入口：画布 Agent `generate-sound-effect` 与站内 / External v1 共用 canonical Prompt、固定模型、nullable `0.5-30` 小数时长和 Loop；省略 duration 为手动 `5s`，显式 null 为自动。SFX 参数解析必须保留该 null，不能被通用 null-default 兼容层改写。最终仍进入相同 `editor_sound_effect_generation` queue payload，不新增 Agent 专属链路。 +- 发布边界：T6 工程实施和 mock / loopback 门禁不等于真实 provider 或生产验收。发布前关闭 SFX 入队，使用显式 `--server` / `--server-url` 只读查询 `external_generation_job` 中 pending / running 的 `editor_sound_effect_generation`，清零后按 api-server / Worker → Web 顺序部署并灰度；禁止 `--root-dir`、删除任务伪造 drain 或自动回退 Vidu。本次没有 SpacetimeDB schema、migration 或 bindings 变更。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 093f45414..4d2b70eb3 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -4348,3 +4348,10 @@ - 原因：冲突两侧代表不同组件架构，逐行保留看似有用的 JSX 会把一个架构中的局部条件拼进另一个架构。import 排序、格式检查和只覆盖单一 mode 的测试都不能证明这种组合成立。 - 处理：先确定权威组件边界，再按完整调用链解决冲突。图片画布音频入口当前决策是恢复一个共享 `ImageCanvasAudioGenerationComposerView`，由组件内 `isSoundEffect` 分流；BGM/SFX 的 validator、写回、锁和提交契约仍分别保持。不要只补一个常量后继续维持已经废弃的双 composer 边界。 - 验证：同时渲染 `audio-sound-effect` 与 `audio-background-music`，覆盖两个 mode 的正向控件和互斥负向断言、dialog / mode 切换、BGM 稳定 ID 与 controller 缺失的失败关闭，并运行 `ImageCanvasGenerationComposerView.test.tsx` 与 typecheck。 + +## SFX Worker 不能只靠分层单测证明退款和零副作用（2026-08-07） + +- 现象：LLM、ElevenLabs adapter、OSS 和 metadata 各自测试都通过，但无法直接证明余额不足时外部调用为零、翻译失败不会调用 provider、OSS / DB 失败只退款一次，或项目资源 / 素材 / 画布使用同一份权威 metadata。 +- 原因：正式 SFX handler 把计费、翻译、provider、持久化和写回内联在一个 future 中；分层测试只能证明单个 helper，不能证明组合顺序和“失败后不继续”。同时若把 mock 流程另写一遍，它本身又可能与生产逻辑漂移。 +- 处理：抽出单一 Worker 编排函数和计费 / stage adapter。生产 adapter 代理现有正式实现，测试 adapter 逐段记录调用与注入失败；组合矩阵同时断言 charge / refund、LLM / provider / OSS / writeback 计数、稳定 reason code 和权威值等值。ElevenLabs 二进制、MIME、大小、timeout 和 MP3 仍由 loopback adapter 测试负责，组合 mock 不替代协议测试。 +- 验证：自动 / 手动时长 × Loop 四组合成功；余额不足；翻译、HTTP、无效音频、时长探测、OSS PUT / HEAD、asset confirm / bind、项目资源、账号素材和画布写回逐点失败；所有 job provider POST `<= 1`，预扣后失败 refund `= 1`。 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index 0512bd103..e32f7b8b8 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -308,7 +308,7 @@ Responses 的终态载荷既是工具调用的恢复源，也是正文的恢复 - Match3D 1:1 容器 UI：VectorEngine `/v1/images/edits` multipart 参考图。该容器参考图是后端生图协议输入，必须通过 `include_bytes!` 随 `api-server` 编译进二进制，避免 API 单独发布或运行目录缺少 `public/` 时生成失败。 - 敲木鱼敲击物和背景环境图：VectorEngine `/v1/images/edits`，模型固定 `gpt-image-2`。敲击物支持 multipart 多参考图，第一张固定为后端内嵌默认木鱼图，用户上传图只作为新主题参考；prompt 必须要求 `1:1` 单一纯绿色 `#00FF00 / RGB(0,255,0)` 绿幕背景主体图，并禁止黑底、白底、棋盘格和任何实底背景。当前敲击物和返回按钮上传 OSS 前只做服务端绿幕去背后处理，避免泛抠图误伤玉米等主体像素。背景环境图只使用第一步抠图完成后的透明敲击物图作为参考，prompt 必须要求中央主体预留区保持干净，中央 40% 区域禁止出现主题主体、主体局部特写、轮廓影子或重复元素，主题元素只能作为外围氛围，且必须显式声明不继承任何绿色底色、绿幕底色或纯绿色画布。 - Hyper3D / Rodin：只保留后端安全代理和旧数据兼容；Rodin 提交、状态、下载和响应解析归属 `platform-hyper3d`，`api-server/src/hyper3d_generation.rs` 只做路由、配置和错误 envelope 映射；新 Match3D 草稿和批量新增不再生成 GLB。 -- 音频：视觉小说专用音频路由保留；VectorEngine Suno/Vidu provider 协议、任务提交/查询、音频 URL 提取、下载、MIME/extension 归一，以及 ElevenLabs SFX 单次同步二进制请求、`40 MiB` 有界读取、MP3 验证和 `600s` 技术异常上限内的实际时长探测均归属 `platform-audio`。ElevenLabs 直接 adapter 不进入 Suno/Vidu 的 submit + poll 枚举，OSS put 请求准备以显式 provider / file stem 描述来源。`api-server/src/vector_engine_audio_generation.rs` 只做路由、配置、计费、asset object confirm、entity binding 和错误 envelope 映射；拼图、抓大鹅和敲木鱼提示词生成音效入口暂时关闭，通用 `/api/creation/audio/*` 对这些目标返回 `410 Gone`。敲木鱼创作只接收上传 / 录音音频资产；前端选择或录音阶段只在浏览器本地处理待提交音频，统一限制裁切后最长 1 秒、裁掉前后声音过小片段，并用浏览器端近似响度算法平衡到 `-15 LKFS` 后做峰值保护。点击生成时才直传 OSS 并确认 `asset_object`，创作 JSON 只提交轻量 `WoodenFishAudioAsset`，不得继续上传 Data URL 音频；未提供时由 `api-server` 写回内置默认木鱼音 `/wooden-fish/default-hit-sound.mp3`。 +- 音频：视觉小说专用音频路由保留；VectorEngine Suno/Vidu provider 协议、任务提交/查询、音频 URL 提取、下载、MIME/extension 归一，以及 ElevenLabs SFX 单次同步二进制请求、`40 MiB` 有界读取、MP3 验证和 `600s` 技术异常上限内的实际时长探测均归属 `platform-audio`。ElevenLabs 直接 adapter 不进入 Suno/Vidu 的 submit + poll 枚举，OSS put 请求准备以显式 provider / file stem 描述来源。`api-server/src/vector_engine_audio_generation.rs` 只做路由、配置、计费、asset object confirm、entity binding 和错误 envelope 映射；SFX Worker 在该模块内以同一编排函数串联计费、翻译、ElevenLabs、OSS、项目资源 / 账号素材 / 画布写回，生产 adapter 复用正式边界、测试 adapter 只注入 mock。内部失败分类固定为 `translation_invalid / translation_upstream_failed / translation_budget_exhausted / elevenlabs_http_failed / invalid_audio / duration_probe_failed / oss_failed / writeback_failed`，普通用户读取边界继续返回稳定短文案。拼图、抓大鹅和敲木鱼提示词生成音效入口暂时关闭，通用 `/api/creation/audio/*` 对这些目标返回 `410 Gone`。敲木鱼创作只接收上传 / 录音音频资产；前端选择或录音阶段只在浏览器本地处理待提交音频，统一限制裁切后最长 1 秒、裁掉前后声音过小片段，并用浏览器端近似响度算法平衡到 `-15 LKFS` 后做峰值保护。点击生成时才直传 OSS 并确认 `asset_object`，创作 JSON 只提交轻量 `WoodenFishAudioAsset`，不得继续上传 Data URL 音频；未提供时由 `api-server` 写回内置默认木鱼音 `/wooden-fish/default-hit-sound.mp3`。 - OSS：私有 generated path 进入浏览器前必须通过 `/api/assets/read-url` 换签；不要裸请求 `/generated-*`。请求参数的安全语义不能混用：`legacyPublicPath` 是历史公开作品兼容口，只允许 `platform_oss::LEGACY_PUBLIC_PREFIXES` 中的 curated 前缀匿名换签；`objectKey` 是正式对象引用，绝不能复用该前缀旁路，必须查询 `asset_object` 并校验配置 bucket、精确 key、`PublicRead` 或当前 owner。External OpenAPI 的 `/api/external/v1/assets/read-url` 还必须有 `editor:asset` scope，并始终以 API Key 绑定的 `owner_user_id` 执行同一 owner 校验；后台跨账号预览只能走管理员鉴权后的 `/admin/api/assets/read-url`。`/api/assets/read-bytes` 与主站 read-url 共用完全相同的授权，默认仍应由浏览器使用 signed URL 直读，bytes 只作跨域字节读取 fallback。前端如果收到同一 OSS bucket 的完整 `https://*.oss-*.aliyuncs.com/generated-*` 地址，也必须先归一为 legacy path 后走同一换签链路，避免裸连私有 bucket 403 或绕过签名缓存。OSS 签名、读签名、HEAD 和 PUT 的结构化日志由 `platform-oss` 输出，排查资产写入 / 确认失败时优先按 `operation`、`object_key` / `key_prefix`、`status_class`、`error_kind` 和 `elapsed_ms` 下钻。新上传 generated 私有对象默认写入 `Cache-Control: public, max-age=31536000, immutable`；旧对象若缺该头，只能依赖 `ETag` / `Last-Modified` 协商缓存，应通过 OSS 元数据刷新或 CDN 配置补齐，不要恢复 api-server 静态代理。`editor-agent/` 前缀只用于服务端内部读写画布 Agent 会话消息文档，不属于浏览器直传 legacy public prefix；`/api/assets/direct-upload-tickets` 必须拒绝 `legacyPrefix=editor-agent`，内部读取只允许 `editor-agent/{conversationId}.json` 形态。 - 外部 API 失败审计：外部供应商调用未成功时，`api-server` 必须发送 OTLP 失败事件并写入 `tracking_event`。VectorEngine 图片 provider 在 `platform-image` 内输出结构化日志和 `PlatformImageFailureAudit`，覆盖 `request_send`、`response_body`、`upstream_status`、`response_parse`、`missing_image` 和 `image_download` 阶段；编辑器 `screenColor=auto` 的 gpt-5-mini 背景色决策同样必须审计每次已发出的 LLM 调用失败，包括传输 / 超时、上游拒绝、响应体解析、空响应和返回候选外颜色；即使随后降级默认背景色并继续主流程也不得只记 warning。`api-server` 将这些失败映射成 `external_api_call_failure`，`scope_kind = module`、`scope_id = provider`、`module_key = external-api`。metadata 固定包含 provider、endpoint、operation、failureStage、statusCode、statusClass、timeout、retryable、errorMessage、latencyMs、promptChars、referenceImageCount、imageModel、rawExcerpt，以及在调用方可获得上下文时补充的 `userId`（触发者）和 `profileId`（草稿 / 作品 / 场景作用域）。图片生成入口应优先把 owner user id 和 profile id 透传到失败审计，不要只保留 provider 级聚合，否则很难按“谁触发、哪个作品触发”定位问题。普通调用入库优先复用 tracking outbox，outbox 不可写或保护阈值拒绝时回退同步写 SpacetimeDB；不得新增前端兜底或在 SpacetimeDB reducer 内做外部 I/O。`bgfilter-worker` 是受限资源例外：它使用共享 tracking outbox 基础目录下独立的 `bgfilter-worker/` 子目录，provider 失败审计在 spawn 前受进程级 `1024` 硬上限保护并由 shutdown tracker 跟踪；满载、outbox 缺失、保护阈值拒绝或写盘失败时直接丢弃并观测，不回退同步直写 SpacetimeDB。优雅退出先排空已获准任务的 enqueue，再封存并尽力 flush；进程被强杀时只有已 enqueue 记录可在下次启动重放。 - 外部生成运行记录：所有外部生成编排的完成态统一写入 `tracking_event`，`event_key = external_generation_run`，`scope_kind = module`，`scope_id = provider`，`module_key = external-generation`。metadata 固定包含 `runId`、`provider`、`operation`、`requestLabel`、`requestPayload`、`status`、`success`、`failureReason`、`providerRequestId`、`resultPayload`、`startedAtMicros`、`completedAtMicros` 和 `durationMs`。这类记录只用于运行审计和排障，不再走 `ai_task` 旧表。 diff --git a/docs/【实施记录】SFX生成优化V2.0T6测试与发布门禁-2026-08-07.md b/docs/【实施记录】SFX生成优化V2.0T6测试与发布门禁-2026-08-07.md new file mode 100644 index 000000000..05ee80acd --- /dev/null +++ b/docs/【实施记录】SFX生成优化V2.0T6测试与发布门禁-2026-08-07.md @@ -0,0 +1,158 @@ +# SFX 生成优化 V2.0 T6 测试与发布门禁实施记录 + +日期：`2026-08-07` + +状态：`工程实施完成；生产配置、队列 drain、灰度和实际发布待人工执行` + +## 文档定位 + +本文记录 SFX V2 T6 的实际工程范围、组合测试证据和发布 / 回滚门禁。产品与技术规则以[画板音乐生成入口设计](./【编辑器】画板音乐生成入口设计-2026-06-18.md)为准，分阶段依赖以[SFX V2 任务拆解](./project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md)为准。 + +本文不是生产部署授权。所有自动测试都使用内存 mock、loopback HTTP server 或仓库 fixture；没有调用真实 LLM、ElevenLabs、OSS 或其它付费 provider，不能表述为真实 provider 验收。 + +## 实施基线与边界 + +- 隔离分支：`codex/sfx-v2-t6`。 +- 基线：`feat/sound_opt@69f850ff827fe64ec1c7f7a325e34ecf9c338252`，已包含 master 合并、Chat completion tokens 契约修复、未知请求字段拒绝、稳定用户失败文案和 Worker provider budget。 +- 不修改 SpacetimeDB schema、migration、bindings 或表目录。 +- 不修改 BGM Suno、BGM Prompt 助手、BGM 预设、提交锁和定价语义。 +- 不执行生产配置写入、数据库查询、旧任务 drain、服务部署、流量切换或真实生成。 + +## 正式 Worker 组合测试缝 + +正式 SFX Worker 现在通过同一个内部编排函数按顺序执行： + +```text +计费预扣 + -> Luna 英文化 + -> 单次 ElevenLabs POST + -> MP3 / 实际时长门禁 + -> OSS PUT / HEAD + -> asset object confirm / bind + -> 项目资源 / 账号素材 / 画布完成态写回 +``` + +生产 adapter 仍调用原有统一计费、`LlmClient`、`platform-audio` ElevenLabs adapter、OSS 和编辑器持久化实现，没有复制第二套业务流程。测试 adapter 只替换外部边界，以相同编排函数覆盖： + +- 自动 / 手动时长 × Loop false / true 四种成功组合； +- 余额不足时 LLM、ElevenLabs、OSS 和全部写回均为零； +- `translation_invalid / translation_upstream_failed` 时 ElevenLabs 为零； +- ElevenLabs HTTP、无效 body / MIME / 大小、损坏 MP3 / 时长探测失败； +- OSS PUT、OSS HEAD、asset confirm、asset bind、项目资源、账号素材和画布写回失败； +- 所有预扣后的失败恰好退款一次，成功只扣费一次； +- 每个 job 的 ElevenLabs POST 最多一次； +- `prompt / actualPrompt / model / provider / taskId / actual duration / loop / generationInputs.soundEffect` 在响应、项目资源、账号素材和画布 layer 等值。 + +`platform-audio` 的 loopback HTTP 测试继续负责真实 adapter 形态：固定 endpoint / query / header / body、自动 / 手动时长、Loop、MIME fallback、`40 MiB` 与 chunked 上限、timeout / 429 / 5xx / body 失败无 retry、MP3 probe 和独立 `600s` 上限。组合 mock 不替代这些 adapter 测试。 + +## 稳定失败分类 + +Worker 任务记录和内部观测使用以下稳定 reason code： + +| 阶段 | reason code | +| --- | --- | +| 翻译候选不合格 | `translation_invalid` | +| 翻译 transport / upstream | `translation_upstream_failed` | +| 翻译 / provider 预算耗尽 | `translation_budget_exhausted` / `elevenlabs_http_failed` | +| ElevenLabs HTTP / timeout / body 读取 | `elevenlabs_http_failed` | +| MIME、空 body、大小等音频门禁 | `invalid_audio` | +| MP3 识别、帧读取、有限正时长或 600 秒门禁 | `duration_probe_failed` | +| OSS PUT / HEAD | `oss_failed` | +| asset confirm / bind、项目资源、账号素材、画布写回 | `writeback_failed` | + +reason code 不包含 endpoint、provider 原始正文或凭据。普通用户读取失败任务时仍只看到“音效生成失败，请稍后重试。”；原始诊断继续留在受控 Worker / tracing / 后台边界。 + +## 跨入口补齐 + +T6 盘点发现画布 Agent 的 `generate-sound-effect` 虽已使用 ElevenLabs 模型，但仍保留旧 Vidu 的 `2–10` 整数时长且固定 `loop=false`。本阶段修复为： + +- Prompt 使用与站内请求相同的 ECMAScript `String.trim()` 等值 canonicalization 和 `1–2048` code point 门禁； +- model 固定 `eleven_text_to_sound_v2`； +- `duration` 接受 `null` 或有限 `0.5–30` 小数，缺省仍为手动 `5s`； +- `loop` 为独立布尔值，缺省 false； +- 显式 `duration: null` 在 Agent 参数解析中保留为自动时长，不经过通用“顶层 null 当缺省”兼容层； +- Agent 最终生成与登录态、External v1 进入同一个 `editor_sound_effect_generation` canonical queue payload。 + +External v1 仍要求 `Idempotency-Key`。接受的 model 形态在 enqueue 前收敛为同一 payload；`audio1.0` 和未知模型在计费、LLM 与 provider 前返回 `400`。compact result 继续只暴露稳定资源引用、实际时长和 Loop，不暴露 Prompt 或 provider。 + +## 验证结果 + +| 门禁 | 结果 | +| --- | --- | +| `platform-audio` 分层与 ElevenLabs loopback | `60/60` | +| `platform-editor-agent` | `24/24` | +| api-server SFX Prompt / 翻译 / Worker 组合矩阵 | `32/32` | +| External v1 / OpenAPI / 幂等 | `13/13` | +| external-generation Worker / compact / deadline | `31/31` | +| api-server BGM 回归 | `35/35` | +| SFX / BGM 前端提交、刷新、重绘、metadata | `14` 个文件、`451/451` | +| `cargo check -p api-server --all-targets` | 通过；仅既有 dead-code warning | +| `npm run typecheck` | 通过 | +| `npm run check:spacetime-schema` | `137` 张表通过，确认无 schema diff | +| `npm run check:encoding` | `5248` 个文件通过 | +| `cargo fmt --all -- --check` | 通过 | +| `git diff --check` | 通过 | + +本地运行态 smoke 使用独立临时数据库、临时 data dir 和 `18000–18004` 端口，未复用或修改原工作树正在运行的 `3000 / 3101 / 8082 / 8083` 服务。结果如下： + +| 服务 | 地址 | 门禁 | 结果 | +| --- | --- | --- | --- | +| SpacetimeDB | `http://127.0.0.1:18002` | `GET /v1/ping` | HTTP 200 | +| BgFilter worker | `http://127.0.0.1:18004` | `GET /readyz` | HTTP 200，`ready=true` | +| api-server | `http://127.0.0.1:18001` | `GET /healthz` | HTTP 200，`service=genarrative-api-server` | +| Web | `http://127.0.0.1:18000` | `GET /` | HTTP 200 | +| Admin Web | `http://127.0.0.1:18003/admin/` | `GET /admin/` | HTTP 200 | + +临时进程树停止后，`18000–18004` 五个端口均已释放。该 smoke 只证明本地进程、路由和临时 SpacetimeDB 模块可以启动，不包含真实 ElevenLabs / LLM / OSS 调用，也不代表生产配置或生产队列已验收。 + +## 生产发布门禁 + +以下步骤未在 T6 工程实施中执行。发布人员必须在维护窗内逐项记录时间、目标环境和结果，但不得把 Key、Token、Cookie、完整 provider 错误正文或宿主私密路径写入仓库。 + +### 1. 配置与价格 + +在目标 host 的受控 secret 环境中只检查“是否存在”，不输出值： + +```bash +for name in ELEVENLABS_BASE_URL ELEVENLABS_API_KEY ELEVENLABS_REQUEST_TIMEOUT_MS; do + test -n "$(printenv "$name")" || { echo "missing required setting: $name" >&2; exit 1; } +done +``` + +确认生产定价配置包含 `eleven_text_to_sound_v2`，单位 `perGeneration`，批准价格为 `5` 泥点；禁止只依赖前端兜底价格。 + +### 2. 关闭入队并排空旧任务 + +先进入维护窗或关闭 SFX 提交入口，再对显式目标 server 做只读查询。`` 和 `` 必须由发布环境明确提供；禁止依赖默认 server，禁止使用 `--root-dir`： + +```bash +spacetime sql \ + --server \ + --format json \ + "SELECT job_id, status, request_payload_json FROM external_generation_job WHERE job_kind = 'editor_sound_effect_generation' AND (status = 'pending' OR status = 'running')" +``` + +门禁要求结果为零行。非零时保持 SFX 入队关闭，让当前旧 Worker drain；不得用新 Worker 解析存量 Vidu payload，也不得删除或改写任务来伪造清零。该查询只读，不修改 schema 或数据。 + +### 3. 部署顺序与灰度 + +1. 保持 SFX 入队关闭。 +2. 部署共享 env 已对齐的 api-server / external-generation worker。 +3. 检查 `/healthz`，确认 Worker 可启动且没有配置失败。 +4. 部署 Web。 +5. 先放开小比例 SFX 入队，观察失败分类、退款、provider POST 和完成资源对账，再逐步放量。 +6. External v1 调用方完成 contract smoke 后再结束维护窗。 + +灰度必须对账：job 完成数、退款数、ElevenLabs POST 数、完成资源数和孤儿资源数。单 job provider POST 大于 1、翻译失败仍出现 provider POST、成功 job 出现退款或失败 job 无退款时立即停止放量。 + +## 回滚门禁 + +- 不自动切回 Vidu，也不在失败时静默 fallback。 +- 先停止新 SFX 入队，再等待或人工收口 V2 pending / running job。 +- Web、api-server、Worker 和 External v1 契约协同回滚，禁止只回滚一层。 +- 已成功生成的 ElevenLabs 素材继续按通用音频资产和现有 metadata 只读展示，不做数据迁移回滚。 +- 本阶段没有 SpacetimeDB schema 变更，回滚不得执行表迁移、字段删除或数据重建。 + +## 发布判定 + +T6 工程代码与本地确定性门禁完成后，只能判定“具备进入生产维护窗验证的条件”。只有配置存在、价格批准、旧 SFX 队列为零、部署健康检查通过、灰度对账无异常且 External contract smoke 通过后，生产发布才可放行。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index bdb5710af..1d1588b63 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -185,6 +185,8 @@ spacetime sql "SELECT * FROM runtime_setting LIMIT 1" --server http:/ 编辑器 ElevenLabs 音效生成只从服务端读取 `ELEVENLABS_BASE_URL`、`ELEVENLABS_API_KEY` 和 `ELEVENLABS_REQUEST_TIMEOUT_MS`，timeout 默认 `180000ms`；base URL 或 Key 缺失时失败关闭，不回退 Vidu。生产 API 与 external-generation worker 通过共享 API env 取得同一配置，模板见 `deploy/env/api-server.env.example`；Key 不得进入 Web/Vite 环境、命令参数、日志、fixture 或仓库。普通测试只使用 loopback mock，禁止把真实付费请求作为 T3 自动验收。 +SFX V2 发布必须使用维护窗：先关闭 SFX 入队，再对显式目标执行只读 `spacetime sql --server --format json "SELECT job_id, status, request_payload_json FROM external_generation_job WHERE job_kind = 'editor_sound_effect_generation' AND (status = 'pending' OR status = 'running')"`；结果非零时保持旧 Worker drain，不得删除任务或让新 Worker 解析旧 Vidu payload。禁止依赖默认 server，禁止使用 `--root-dir`。清零后先部署共享 env 已对齐的 api-server / external-generation worker，检查 `/healthz` 和 Worker 启动，再部署 Web 并小流量开放 SFX。灰度对账 job 完成数、退款数、ElevenLabs POST 数、完成资源数和孤儿资源；翻译失败仍调用 provider、单 job provider POST 大于一次、成功退款或失败未退款均应立即停止放量。回滚先停止入队并收口 V2 pending / running job，不自动切回 Vidu，不执行 SpacetimeDB schema 或数据回滚。完整清单见 `docs/【实施记录】SFX生成优化V2.0T6测试与发布门禁-2026-08-07.md`。 + VectorEngine 图片生成 / 编辑在 `request_send` 阶段出现 `timeout`、`connect`、libcurl 35 SSL connect reset、libcurl 56 receive error / `unexpected eof while reading`、recv failure 等临时传输错误，或在 `upstream_status` 阶段收到 408 / 429 / 5xx（例如 Nginx HTML `502 Bad Gateway`）时，`platform-image` 会在一次业务请求总上限 5 次内处理；multipart 图片编辑每次重试都会重新构造 form，避免复用已消费的 body。首个 provider attempt 使用 `gpt-image-2`；明确模型不可用、408 / 非拒绝类 429 / 5xx、响应解析失败或非拒绝类缺图时，下一 attempt 直接切兜底模型 `gpt-image-2-c`，之后只在剩余次数内重试兜底模型。发送 / 连接错误无法确认上游是否已受理，只重试同一首选模型，不切模型；认证、普通参数、安全拒绝、图片下载和 budget 错误同样不切。worker 从 job 开始的同一时钟起点计算绝对 deadline，常规保留最后 `60` 秒给审计、OSS 和终态写回；job 预算小于 `120` 秒时保留一半。VectorEngine 单次 attempt timeout 取配置值和剩余 provider 预算的较小值；退避或模型切换后已没有下一次 attempt 的预算时立即停止。该 deadline 覆盖参考图、provider 请求 / 响应和响应图片下载的整次 provider future，但只在 worker 进程内通过 `RequestContext` 传递；普通 HTTP / `inline` 没有该 deadline，继续保持原有 timeout 和重试行为。日志中 `VectorEngine 首选图片模型失败，切换兼容模型` 会携带 `fallback_from_model` / `fallback_to_model`；即使回退成功，首选模型错误仍写入 `external_api_call_failure`，成功运行摘要的 `recoveredFailureCount` 同时递增。排查生产失败时应同时统计 fallback / retry 日志和最终 audit，避免把一次用户请求内的多次发送误判成多个用户请求。这项收口不修改 lease 续租 / fencing、迟到写回仲裁、attempt 耗尽与原子退款语义。 图片编辑器生成属于持久队列长任务：提交接口返回 job 后，前端通过 `/api/runtime/external-generation/jobs/{jobId}` 与编辑器项目资源状态收敛。生产排查小程序或 WebView `Failed to fetch` 时，若 Nginx access log 为 `499`、`upstream_status=-`，先按提交请求的 `request_id`、job id、worker 日志和 `external_api_call_failure` 对齐真实任务，不把客户端断开直接判定为 provider 失败。 diff --git a/docs/【编辑器】画布Agent对话面板-2026-07-03.md b/docs/【编辑器】画布Agent对话面板-2026-07-03.md index bbb069825..60884a746 100644 --- a/docs/【编辑器】画布Agent对话面板-2026-07-03.md +++ b/docs/【编辑器】画布Agent对话面板-2026-07-03.md @@ -29,6 +29,7 @@ - 用户要求“角色规范图”且语义是角色的规范展板、风格展板或设定板时，仍走 `generate_image`，不要误分流到 `generate_character`；只有实际生成角色立绘、角色主形象或角色视觉资产时才走 `generate_character`。用户要求多个图标素材、图集或 spritesheet 时才走 `generate_icon_spritesheet`。 - 所有生成必须走 `execute_billable_asset_operation_with_cost` 与模型定价配置，禁止绕过定价收口。 - function-calling 的 JSON Schema 必须与参数默认值和运行时校验保持一致，不能只在 description 中提示会被运行时拒绝的组合。`generate-ui-design` 固定 `gpt-image-2`，因此 `image_size` 只暴露 `1K / 2K`；其它可切换图片模型的工具通过共享条件 schema 在显式选择 `gpt-image-2` 时同样把 `image_size` 限制为 `1K / 2K`，省略模型时仍按默认 nanobanana2 允许 `0.5K`。`generate-video` 省略 `model` 时按默认 `seedance2.0-fast` 约束 `resolution` 为 `480p / 720p`，显式选择其它模型时仍使用其现有分辨率范围。运行时强类型校验继续作为最终防线。 +- `generate-sound-effect` 与站内 / External v1 的 SFX V2 契约一致：Prompt 使用 ECMAScript `String.trim()` 等值 canonicalization 且限制 `1–2048` Unicode code points，model 固定 `eleven_text_to_sound_v2`，`duration` 缺省为手动 `5s`、显式 `null` 为自动时长、数值范围为有限 `0.5–30` 小数，`loop` 缺省 false。显式 `duration:null` 必须绕过通用“顶层 null 当缺省”兼容层，不能在 job payload 中变回 `5s`；确认后的 canonical payload 继续进入现有 `editor_sound_effect_generation` Worker，不新增 Agent 专属音频链路。 - 图层操作及其他未注册的画板功能第一期不进入对话工具面，仍走现有面板。 ## 当前分支落地状态 diff --git a/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md b/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md index 3879ed7a2..430fedbb2 100644 --- a/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md +++ b/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md @@ -12,7 +12,7 @@ 音效与背景音乐继续使用同一个音频 composer，并由组件内的 `isSoundEffect = dialog.mode === 'audio-sound-effect'` 隔离行为。共享视图不等于共享业务规则：BGM 继续使用 Suno、200 字 canonical Prompt、30 个预设、AI 补全 / 简化、单层撤销和方案 A 提交锁；SFX V2 固定使用 ElevenLabs `eleven_text_to_sound_v2`、52 个预设、一键优化、自动中译英、自动 / 手动时长和 Loop。两条路径的 Prompt 模型、controller、预设 wrapper、锁和提交契约必须分别维护，不得交叉复用业务状态。 -SFX V2 已完成产品与技术口径冻结，T0 已通过，T1–T5 已完成，可以继续进入 T6 测试、灰度与发布门禁；这不表示功能已上线。登录态一键优化、Worker 翻译、ElevenLabs 直接二进制 adapter、dialog-scoped 前端交互以及正式提交 / Worker / 计费 / OSS / 权威 metadata / External v1 已完成接线。历史 Vidu 素材继续只读展示；重绘时使用历史用户 Prompt 打开 SFX V2 面板，新任务统一走 ElevenLabs，不回退 Vidu。T6 完成前仍不得发布。 +SFX V2 已完成产品与技术口径冻结及 T0–T6 工程实施；这不表示功能已上线。登录态一键优化、Worker 翻译、ElevenLabs 直接二进制 adapter、dialog-scoped 前端交互、正式提交 / Worker / 计费 / OSS / 权威 metadata / External v1、组合失败矩阵和发布 runbook 已完成。历史 Vidu 素材继续只读展示；重绘时使用历史用户 Prompt 打开 SFX V2 面板，新任务统一走 ElevenLabs，不回退 Vidu。生产配置确认、旧 Vidu 队列 drain、灰度和实际发布仍须按 T6 门禁人工执行。 ## 入口与交互 @@ -146,6 +146,8 @@ SFX V2 已完成产品与技术口径冻结，T0 已通过，T1–T5 已完成 - 使用纯 Rust MP3 探测获得实际时长，持久化和响应的 `durationSeconds` 必须来自 MP3 而不是请求时长。实际时长必须是有限正数且不大于独立技术异常上限 `600s`；`600s` 允许，任何大于 `600s` 的结果拒绝。该上限不由请求最大 `30s` 推导，也不要求实际时长接近请求值；通过 MP3、MIME、字节和时长门禁的 `30.5s-600s` 结果均可接受。 - ElevenLabs 无 provider task ID。queue 模式使用 `external_generation_job.job_id` 作为平台 operation / `taskId`，inline 兼容模式在 provider 调用前生成平台 task ID，不得伪造 ElevenLabs task ID。provider 固定 `elevenlabs`，model 固定 `eleven_text_to_sound_v2`。 - 预扣成功后才允许翻译和 provider 调用。翻译、provider、二进制验证、时长探测、OSS、资源 / 素材 / 画布写回任一失败都进入现有失败退款边界。 +- 正式 Worker 使用同一内部编排函数串联计费、翻译、ElevenLabs、OSS 和项目资源 / 账号素材 / 画布写回。生产 adapter 必须继续调用现有正式实现；测试 adapter 只替换外部边界，用于证明余额不足零外部副作用、失败一次退款、单 job 最多一次 provider POST 和权威 metadata 等值，不得复制第二套业务流程或把 mock 结果表述成真实 provider 验收。 +- Worker 内部失败分类固定为 `translation_invalid / translation_upstream_failed / translation_budget_exhausted / elevenlabs_http_failed / invalid_audio / duration_probe_failed / oss_failed / writeback_failed`。这些 code 只用于任务记录、日志、指标和后台排障；普通用户失败文案保持稳定短文案，不透出 endpoint、provider 原始正文或凭据。 ### 权威元数据、详情、计费与外部契约 @@ -166,6 +168,7 @@ SFX V2 已完成产品与技术口径冻结，T0 已通过，T1–T5 已完成 - 所有接受形态在定价、预扣和 enqueue 前收敛为同一 canonical model，因而 omitted / null / 空白 / 显式新模型不得产生不同幂等 payload。`audio1.0` 与未知模型必须在入队前失败，并由测试证明零入队、零预扣、零 LLM 和零 provider。成功响应增加可选 `durationSeconds` 和 `loop`；Rust DTO、`docs/openapi/genarrative-external-v1.openapi.json`、`202 / poll / final response`、Idempotency-Key 重放测试和 compact result 必须同批保持一致。 - ElevenLabs 配置只允许从服务端 `ELEVENLABS_BASE_URL`、`ELEVENLABS_API_KEY`、`ELEVENLABS_REQUEST_TIMEOUT_MS` 读取，request timeout 默认 `180000ms`；Key 不进入浏览器、请求体、日志、fixture、共享文档或 Git。base URL 或 Key 缺失时失败关闭，不回退 Vidu。 +- 画布 Agent `generate-sound-effect` 不是独立契约：其确认后 payload 同样固定 canonical Prompt、`eleven_text_to_sound_v2`、`duration = null | 0.5-30` 和 `loop`，缺省仍为手动 `5s` / Loop false。Agent 的显式 `duration:null` 表示自动时长，不能被通用 null-default 兼容层改写；最终继续进入相同 `editor_sound_effect_generation` 队列与 Worker。 ## BGM Prompt 优化 V1.0 diff --git a/server-rs/crates/api-server/src/editor_agent/tool.rs b/server-rs/crates/api-server/src/editor_agent/tool.rs index 31fc50cd4..16ea0826b 100644 --- a/server-rs/crates/api-server/src/editor_agent/tool.rs +++ b/server-rs/crates/api-server/src/editor_agent/tool.rs @@ -214,6 +214,23 @@ fn parse_internal( }) } +/// SFX V2 的显式 `duration: null` 表示自动时长，不能经过通用的“顶层 null 当缺省” +/// 兼容层，否则会被错误恢复为手动 5 秒。 +fn parse_sound_effect_args( + label: &str, + value: &Value, + invalid_args: bool, +) -> Result { + serde_json::from_value(value.clone()).map_err(|error| { + let message = format!("failed to deserialize {label}: {error}"); + if invalid_args { + EditorAgentToolError::invalid_args(message) + } else { + EditorAgentToolError::internal(message) + } + }) +} + fn serialize_normalized_args( tool_name: &str, args: &T, @@ -982,8 +999,8 @@ impl EditorAgentTool for GenerateVideoTool { impl EditorAgentTool for GenerateSoundEffectTool { fn validate_args(&self, args: &Value) -> Result { - let args: GenerateSoundEffectToolArgs = parse_invalid_args(Self::NAME, args)?; - GenerateSoundEffectTool::validate_args(self, &args) + let args = parse_sound_effect_args(Self::NAME, args, true)?; + let args = GenerateSoundEffectTool::normalize_args(self, args) .map_err(|error| EditorAgentToolError::invalid_args(error.to_string()))?; serialize_normalized_args(Self::NAME, &args) } @@ -993,7 +1010,7 @@ impl EditorAgentTool for GenerateSoundEffectTool { pricing: &EditorGenerationPricingConfig, args: &Value, ) -> Result { - let args: GenerateSoundEffectToolArgs = parse_internal("generate sound effect args", args)?; + let args = parse_sound_effect_args("generate sound effect args", args, false)?; Ok(pricing.sound_effect_model_mud_points(Some(args.model.as_str()))) } @@ -1003,8 +1020,7 @@ impl EditorAgentTool for GenerateSoundEffectTool { pricing: &EditorGenerationPricingConfig, ) -> Result { let price_mud_points = self.pricing(pricing, args)?; - let args: GenerateSoundEffectToolArgs = - parse_internal("generate sound effect display args", args)?; + let args = parse_sound_effect_args("generate sound effect display args", args, false)?; let mut display_args = EditorAgentToolCallDisplayArgs::default(); push_string_display_arg(&mut display_args, "prompt", "音效描述", args.prompt); push_string_display_arg(&mut display_args, "model", "模型", args.model); @@ -1012,7 +1028,20 @@ impl EditorAgentTool for GenerateSoundEffectTool { &mut display_args, "duration", "时长（秒）", - args.duration.to_string(), + args.duration + .map(|duration| format!("{duration}秒")) + .unwrap_or_else(|| "自动".to_string()), + ); + push_string_display_arg( + &mut display_args, + "loop", + "Loop", + if args.loop_enabled { + "开启" + } else { + "关闭" + } + .to_string(), ); Ok(finish_display_args(display_args, price_mud_points)) } @@ -1023,13 +1052,13 @@ impl EditorAgentTool for GenerateSoundEffectTool { context: &EditorAgentPrepareJobContext<'_>, ) -> Result { let price_mud_points = self.pricing(context.pricing, args)?; - let args: GenerateSoundEffectToolArgs = parse_invalid_args(Self::NAME, args)?; + let args = parse_sound_effect_args(Self::NAME, args, true)?; let title = args.prompt.clone(); let payload = EditorSoundEffectGenerateRequest { prompt: args.prompt, model: Some(args.model), - duration: Some(f64::from(args.duration)), - loop_enabled: false, + duration: args.duration, + loop_enabled: args.loop_enabled, project_id: Some(context.conversation.project_id.clone()), canvas_completion: Some(build_editor_agent_canvas_completion( context.project, @@ -1053,7 +1082,7 @@ impl EditorAgentTool for GenerateSoundEffectTool { args: &Value, result: &Value, ) -> Result { - let args: GenerateSoundEffectToolArgs = parse_internal("generate sound effect args", args)?; + let args = parse_sound_effect_args("generate sound effect args", args, false)?; let result: EditorAudioGenerateResponse = parse_internal("generate sound effect result", result)?; Ok(GenerateSoundEffectTool::format_execute_message( @@ -1465,7 +1494,8 @@ mod tests { assert_eq!(display.extras.price_mud_points, 5); assert_eq!(display.string_args[0].label, "音效描述"); assert_eq!(display.string_args[0].value, "按钮点击声"); - assert_eq!(display.string_args[2].value, "5"); + assert_eq!(display.string_args[2].value, "5秒"); + assert_eq!(display.string_args[3].value, "关闭"); } #[test] @@ -1498,7 +1528,11 @@ mod tests { let tool = editor_agent_tool(GenerateSoundEffectTool::NAME, &EditorToolContext::default()) .expect("sound tool should resolve"); let args = tool - .validate_args(&json!({ "prompt": "按钮点击声" })) + .validate_args(&json!({ + "prompt": "\u{feff}按钮点击声\u{2003}", + "duration": null, + "loop": true + })) .expect("sound args should validate"); let result = json!({ "ok": true, @@ -1537,7 +1571,11 @@ mod tests { let tool = editor_agent_tool(GenerateSoundEffectTool::NAME, &EditorToolContext::default()) .expect("sound tool should resolve"); let args = tool - .validate_args(&json!({ "prompt": "按钮点击声" })) + .validate_args(&json!({ + "prompt": "\u{feff}按钮点击声\u{2003}", + "duration": null, + "loop": true + })) .expect("sound args should validate"); let job = tool .prepare_job( @@ -1553,11 +1591,13 @@ mod tests { assert_eq!(job.job_kind, EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND); assert_eq!(job.request_label, "画布 Agent 生成音效"); assert_eq!(job.price_mud_points, 5); + assert_eq!(job.payload["duration"], Value::Null); + assert_eq!(job.payload["prompt"], "按钮点击声"); assert_eq!( - job.payload["duration"], - json!(f64::from(GenerateSoundEffectTool::DEFAULT_DURATION)) + job.payload["model"], + shared_contracts::assets::EDITOR_SOUND_EFFECT_MODEL ); - assert_eq!(job.payload["loop"], json!(false)); + assert_eq!(job.payload["loop"], json!(true)); assert_eq!(job.payload["projectId"], "project-1"); assert_eq!(job.payload["generationInputs"]["toolCallMessageId"], 7); } diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation.rs index a86eca089..14ca2a31a 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation.rs @@ -5,6 +5,7 @@ mod persist; mod publish; mod settings; mod sound_effect_translation; +mod sound_effect_worker; mod types; pub(crate) use generation::{ diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/errors.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/errors.rs index 1d349c577..a1bd54c48 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation/errors.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/errors.rs @@ -98,6 +98,9 @@ pub(super) fn map_platform_audio_error(error: AudioError) -> AppError { } AudioError::MissingAudio { .. } => {} } + if let Some(reason) = error.reason_code() { + details["reason"] = json!(reason); + } AppError::from_status(status).with_details(details) } diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs index 5ca2e28e0..108f973d9 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs @@ -1,3 +1,5 @@ +use std::future::Future; + use axum::Extension; use axum::http::StatusCode; use axum::{ @@ -38,11 +40,16 @@ use super::{ publish::{build_audio_billing_asset_id, wait_for_generated_audio_asset}, settings::{require_elevenlabs_audio_settings, require_vector_engine_audio_settings}, sound_effect_translation::translate_sound_effect_prompt_for_worker, + sound_effect_worker::{ + PersistedSoundEffectAudio, SoundEffectWorkerBilling, SoundEffectWorkerInput, + SoundEffectWorkerStages, SoundEffectWritebackInput, SoundEffectWritebackOutput, + execute_sound_effect_worker_with_billing, with_sound_effect_failure_reason, + }, types::{AudioAssetBindingTarget, AudioAssetSlot}, }; -const EDITOR_AUDIO_WIDTH: u32 = 420; -const EDITOR_AUDIO_HEIGHT: u32 = 120; +pub(super) const EDITOR_AUDIO_WIDTH: u32 = 420; +pub(super) const EDITOR_AUDIO_HEIGHT: u32 = 120; #[derive(Clone, Debug, PartialEq)] pub(super) struct NormalizedEditorSoundEffectRequest { @@ -325,181 +332,36 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( ); let billing_asset_id = build_audio_billing_asset_id(task_id.as_str(), AudioAssetSlot::SoundEffect, &target); - let response = execute_billable_asset_operation_with_cost( - &state, - owner_user_id.as_str(), - target.asset_kind.as_str(), - billing_asset_id.as_str(), - u64::from(normalized.price_mud_points), - async { - let actual_prompt = translate_sound_effect_prompt_for_worker( - llm_client, - normalized.prompt.as_str(), - request_deadline, - ) - .await - .map_err(|error| { - AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ - "provider": "editor-sound-effect-translation", - "reason": error.reason_code(), - "message": error.to_string(), - })) - })?; - let generated = platform_audio::generate_elevenlabs_sound_effect( - &http_client, - &settings, - platform_audio::ElevenLabsSoundEffectRequest { - text: actual_prompt.clone(), - duration_seconds: normalized.duration, - loop_enabled: normalized.loop_enabled, - }, - ) - .await - .map_err(map_platform_audio_error)?; - let duration_mode = if normalized.duration.is_some() { - assets::EditorSoundEffectDurationMode::Manual - } else { - assets::EditorSoundEffectDurationMode::Auto - }; - let metadata = assets::EditorSoundEffectGenerationMetadataV2::try_new( - normalized.prompt.clone(), - actual_prompt.clone(), - duration_mode, - normalized.duration, - generated.duration_seconds, - normalized.loop_enabled, - ) - .map_err(|message| { - AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ - "provider": platform_audio::ELEVENLABS_PROVIDER, - "message": message, - })) - })?; - let generation_inputs = build_editor_sound_effect_generation_inputs( - payload.generation_inputs.as_ref(), - &metadata, - )?; - let persisted_audio = persist_generated_audio_asset( - &state, - state.editor_oss_http_client(), - owner_user_id.as_str(), - task_id.as_str(), - platform_audio::GeneratedAudioPersistSource { - provider: platform_audio::ELEVENLABS_PROVIDER.to_string(), - file_stem: "sound-effect".to_string(), - }, - target.clone(), - generated.audio, - ) - .await?; - let audio_src = persisted_audio.audio_src; - let object_key = Some(persisted_audio.object_key); - let asset_object_id = Some(persisted_audio.asset_object_id); - let project_id = payload.project_id.clone(); - let canvas_completion = payload.canvas_completion.clone(); - let (resource, asset) = persist_editor_generated_media_asset( - &state, - PersistEditorGeneratedAssetRequest { - project_id: project_id.clone(), - owner_user_id: owner_user_id.clone(), - folder_id: payload.asset_folder_id.clone(), - label: payload - .asset_label - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) - .unwrap_or_else(|| "生成音效".to_string()), - image_src: audio_src.clone(), - object_key: object_key.clone(), - asset_object_id: asset_object_id.clone(), - width: EDITOR_AUDIO_WIDTH, - height: EDITOR_AUDIO_HEIGHT, - prompt: normalized.prompt.clone(), - actual_prompt: Some(actual_prompt.clone()), - model: normalized.model.clone(), - provider: platform_audio::ELEVENLABS_PROVIDER.to_string(), - task_id: task_id.clone(), - source_resource_id: None, - asset_kind: Some("sound-effect".to_string()), - generation_inputs: Some(generation_inputs.clone()), - thumbnail_src: None, - generation_cost_mud_points: u64::from(normalized.price_mud_points), - image_sequence_frames: None, - image_sequence_duration_ms: None, - }, - ) - .await?; - let completed_project = if let Some(completion) = canvas_completion.as_ref() { - let layer_id = format!("layer-editor-sound-effect-{task_id}"); - let resource_id = resource - .as_ref() - .map(|resource| resource.resource_id.clone()) - .unwrap_or_else(|| format!("local-resource-editor-sound-effect-{task_id}")); - let item = build_editor_canvas_generated_layer_item( - completion, - EditorCanvasGeneratedLayerInput { - layer_id: layer_id.clone(), - resource_id, - title: completion.title.trim().to_string(), - src: audio_src.clone(), - thumbnail_src: None, - original_width: EDITOR_AUDIO_WIDTH, - original_height: EDITOR_AUDIO_HEIGHT, - display_width: completion.placeholder.width, - display_height: completion.placeholder.height, - source_type: "generated".to_string(), - prompt: Some(normalized.prompt.clone()), - actual_prompt: Some(actual_prompt.clone()), - model: Some(normalized.model.clone()), - provider: Some(platform_audio::ELEVENLABS_PROVIDER.to_string()), - task_id: Some(task_id.clone()), - object_key: object_key.clone(), - asset_object_id: asset_object_id.clone(), - source_resource_id: None, - asset_kind: Some("sound-effect".to_string()), - generation_inputs: Some(generation_inputs), - }, - ); - complete_editor_canvas_generation_with_items( - &state, - owner_user_id.as_str(), - project_id.as_deref(), - canvas_completion.as_ref(), - vec![item], - Some(layer_id), - ) - .await? - } else { - None - }; - let completed_project = - completed_project.and_then(|project| serde_json::to_value(project).ok()); - let resource = resource.and_then(|resource| serde_json::to_value(resource).ok()); - let asset = asset.and_then(|asset| serde_json::to_value(asset).ok()); - - Ok(assets::EditorAudioGenerateResponse { - ok: true, - audio_src, - object_key, - asset_object_id, - width: EDITOR_AUDIO_WIDTH, - height: EDITOR_AUDIO_HEIGHT, - source_type: "generated".to_string(), - prompt: normalized.prompt.clone(), - actual_prompt: Some(actual_prompt), - model: normalized.model.clone(), - provider: platform_audio::ELEVENLABS_PROVIDER.to_string(), - task_id: task_id.clone(), - price_mud_points: normalized.price_mud_points, - audio_kind: "sound-effect".to_string(), - duration_seconds: Some(generated.duration_seconds), - loop_enabled: Some(normalized.loop_enabled), - project: completed_project, - resource, - asset, - queue_state: None, - }) + let stages = ProductionSoundEffectWorkerStages { + state: &state, + llm_client, + http_client: &http_client, + settings: &settings, + request_deadline, + owner_user_id: owner_user_id.as_str(), + task_id: task_id.as_str(), + target: target.clone(), + payload: &payload, + normalized: &normalized, + }; + let billing = ProductionSoundEffectWorkerBilling { + state: &state, + owner_user_id: owner_user_id.as_str(), + asset_kind: target.asset_kind.as_str(), + billing_asset_id: billing_asset_id.as_str(), + points_cost: u64::from(normalized.price_mud_points), + }; + let response = execute_sound_effect_worker_with_billing( + &billing, + &stages, + SoundEffectWorkerInput { + user_prompt: normalized.prompt.clone(), + model: normalized.model.clone(), + duration_seconds: normalized.duration, + loop_enabled: normalized.loop_enabled, + task_id: task_id.clone(), + price_mud_points: normalized.price_mud_points, + original_generation_inputs: payload.generation_inputs.clone(), }, ) .await @@ -508,7 +370,205 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( Ok(json_success_body(Some(&request_context), response)) } -fn build_editor_sound_effect_generation_inputs( +struct ProductionSoundEffectWorkerBilling<'a> { + state: &'a AppState, + owner_user_id: &'a str, + asset_kind: &'a str, + billing_asset_id: &'a str, + points_cost: u64, +} + +impl SoundEffectWorkerBilling for ProductionSoundEffectWorkerBilling<'_> { + async fn execute(&self, operation: Fut) -> Result + where + Fut: Future>, + { + execute_billable_asset_operation_with_cost( + self.state, + self.owner_user_id, + self.asset_kind, + self.billing_asset_id, + self.points_cost, + operation, + ) + .await + } +} + +struct ProductionSoundEffectWorkerStages<'a> { + state: &'a AppState, + llm_client: &'a platform_llm::LlmClient, + http_client: &'a reqwest::Client, + settings: &'a platform_audio::ElevenLabsAudioSettings, + request_deadline: Option, + owner_user_id: &'a str, + task_id: &'a str, + target: AudioAssetBindingTarget, + payload: &'a assets::EditorSoundEffectGenerateRequest, + normalized: &'a NormalizedEditorSoundEffectRequest, +} + +impl SoundEffectWorkerStages for ProductionSoundEffectWorkerStages<'_> { + async fn translate(&self, user_prompt: &str) -> Result { + translate_sound_effect_prompt_for_worker( + self.llm_client, + user_prompt, + self.request_deadline, + ) + .await + .map_err(|error| { + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": "editor-sound-effect-translation", + "reason": error.reason_code(), + "message": error.to_string(), + })) + }) + } + + async fn generate( + &self, + request: platform_audio::ElevenLabsSoundEffectRequest, + ) -> Result { + platform_audio::generate_elevenlabs_sound_effect(self.http_client, self.settings, request) + .await + .map_err(map_platform_audio_error) + .map_err(|error| with_sound_effect_failure_reason(error, "elevenlabs_http_failed")) + } + + async fn persist_audio( + &self, + audio: platform_audio::DownloadedAudio, + ) -> Result { + persist_generated_audio_asset( + self.state, + self.state.editor_oss_http_client(), + self.owner_user_id, + self.task_id, + platform_audio::GeneratedAudioPersistSource { + provider: platform_audio::ELEVENLABS_PROVIDER.to_string(), + file_stem: "sound-effect".to_string(), + }, + self.target.clone(), + audio, + ) + .await + .map(|persisted| PersistedSoundEffectAudio { + asset_object_id: persisted.asset_object_id, + object_key: persisted.object_key, + audio_src: persisted.audio_src, + }) + .map_err(|error| { + let reason = if error + .details() + .and_then(|details| details.get("provider")) + .and_then(Value::as_str) + == Some("aliyun-oss") + { + "oss_failed" + } else { + "writeback_failed" + }; + error.with_detail_field("reason", json!(reason)) + }) + } + + async fn writeback( + &self, + input: SoundEffectWritebackInput, + ) -> Result { + let audio_src = input.persisted_audio.audio_src; + let object_key = Some(input.persisted_audio.object_key); + let asset_object_id = Some(input.persisted_audio.asset_object_id); + let project_id = self.payload.project_id.clone(); + let canvas_completion = self.payload.canvas_completion.clone(); + let (resource, asset) = persist_editor_generated_media_asset( + self.state, + PersistEditorGeneratedAssetRequest { + project_id: project_id.clone(), + owner_user_id: self.owner_user_id.to_string(), + folder_id: self.payload.asset_folder_id.clone(), + label: self + .payload + .asset_label + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| "生成音效".to_string()), + image_src: audio_src.clone(), + object_key: object_key.clone(), + asset_object_id: asset_object_id.clone(), + width: EDITOR_AUDIO_WIDTH, + height: EDITOR_AUDIO_HEIGHT, + prompt: self.normalized.prompt.clone(), + actual_prompt: Some(input.actual_prompt.clone()), + model: self.normalized.model.clone(), + provider: platform_audio::ELEVENLABS_PROVIDER.to_string(), + task_id: self.task_id.to_string(), + source_resource_id: None, + asset_kind: Some("sound-effect".to_string()), + generation_inputs: Some(input.generation_inputs.clone()), + thumbnail_src: None, + generation_cost_mud_points: u64::from(self.normalized.price_mud_points), + image_sequence_frames: None, + image_sequence_duration_ms: None, + }, + ) + .await?; + let completed_project = if let Some(completion) = canvas_completion.as_ref() { + let layer_id = format!("layer-editor-sound-effect-{}", self.task_id); + let resource_id = resource + .as_ref() + .map(|resource| resource.resource_id.clone()) + .unwrap_or_else(|| format!("local-resource-editor-sound-effect-{}", self.task_id)); + let item = build_editor_canvas_generated_layer_item( + completion, + EditorCanvasGeneratedLayerInput { + layer_id: layer_id.clone(), + resource_id, + title: completion.title.trim().to_string(), + src: audio_src, + thumbnail_src: None, + original_width: EDITOR_AUDIO_WIDTH, + original_height: EDITOR_AUDIO_HEIGHT, + display_width: completion.placeholder.width, + display_height: completion.placeholder.height, + source_type: "generated".to_string(), + prompt: Some(self.normalized.prompt.clone()), + actual_prompt: Some(input.actual_prompt), + model: Some(self.normalized.model.clone()), + provider: Some(platform_audio::ELEVENLABS_PROVIDER.to_string()), + task_id: Some(self.task_id.to_string()), + object_key, + asset_object_id, + source_resource_id: None, + asset_kind: Some("sound-effect".to_string()), + generation_inputs: Some(input.generation_inputs), + }, + ); + complete_editor_canvas_generation_with_items( + self.state, + self.owner_user_id, + project_id.as_deref(), + canvas_completion.as_ref(), + vec![item], + Some(layer_id), + ) + .await? + } else { + None + }; + + Ok(SoundEffectWritebackOutput { + completed_project: completed_project + .and_then(|project| serde_json::to_value(project).ok()), + resource: resource.and_then(|resource| serde_json::to_value(resource).ok()), + asset: asset.and_then(|asset| serde_json::to_value(asset).ok()), + }) + } +} + +pub(super) fn build_editor_sound_effect_generation_inputs( original: Option<&Value>, metadata: &assets::EditorSoundEffectGenerationMetadataV2, ) -> Result { diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/sound_effect_worker.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/sound_effect_worker.rs new file mode 100644 index 000000000..dd2154b17 --- /dev/null +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/sound_effect_worker.rs @@ -0,0 +1,619 @@ +use std::future::Future; + +use axum::http::StatusCode; +use platform_audio::{ + DownloadedAudio, ElevenLabsGeneratedSoundEffect, ElevenLabsSoundEffectRequest, +}; +use serde_json::{Value, json}; +use shared_contracts::assets; + +use crate::http_error::AppError; + +use super::generation::{ + EDITOR_AUDIO_HEIGHT, EDITOR_AUDIO_WIDTH, build_editor_sound_effect_generation_inputs, +}; + +#[derive(Clone, Debug)] +pub(super) struct SoundEffectWorkerInput { + pub(super) user_prompt: String, + pub(super) model: String, + pub(super) duration_seconds: Option, + pub(super) loop_enabled: bool, + pub(super) task_id: String, + pub(super) price_mud_points: u32, + pub(super) original_generation_inputs: Option, +} + +#[derive(Clone, Debug)] +pub(super) struct PersistedSoundEffectAudio { + pub(super) asset_object_id: String, + pub(super) object_key: String, + pub(super) audio_src: String, +} + +#[derive(Clone, Debug)] +pub(super) struct SoundEffectWritebackInput { + pub(super) actual_prompt: String, + pub(super) generation_inputs: Value, + pub(super) persisted_audio: PersistedSoundEffectAudio, +} + +#[derive(Clone, Debug, Default)] +pub(super) struct SoundEffectWritebackOutput { + pub(super) completed_project: Option, + pub(super) resource: Option, + pub(super) asset: Option, +} + +/// 正式 Worker 的四段外部边界。生产实现仍调用真实 LLM、ElevenLabs、OSS 和编辑器写回； +/// 测试实现可以逐段失败，从而验证相同编排函数的顺序、退款和零副作用约束。 +pub(super) trait SoundEffectWorkerStages { + async fn translate(&self, user_prompt: &str) -> Result; + + async fn generate( + &self, + request: ElevenLabsSoundEffectRequest, + ) -> Result; + + async fn persist_audio( + &self, + audio: DownloadedAudio, + ) -> Result; + + async fn writeback( + &self, + input: SoundEffectWritebackInput, + ) -> Result; +} + +/// 计费边界同样可替换，但生产实现只代理现有统一资产扣退费入口。 +pub(super) trait SoundEffectWorkerBilling { + async fn execute(&self, operation: Fut) -> Result + where + Fut: Future>; +} + +pub(super) async fn execute_sound_effect_worker_with_billing( + billing: &B, + stages: &S, + input: SoundEffectWorkerInput, +) -> Result +where + B: SoundEffectWorkerBilling, + S: SoundEffectWorkerStages, +{ + billing + .execute(run_sound_effect_worker(stages, input)) + .await +} + +async fn run_sound_effect_worker( + stages: &S, + input: SoundEffectWorkerInput, +) -> Result +where + S: SoundEffectWorkerStages, +{ + let actual_prompt = stages + .translate(input.user_prompt.as_str()) + .await + .map_err(|error| with_sound_effect_failure_reason(error, "translation_upstream_failed"))?; + let generated = stages + .generate(ElevenLabsSoundEffectRequest { + text: actual_prompt.clone(), + duration_seconds: input.duration_seconds, + loop_enabled: input.loop_enabled, + }) + .await + .map_err(|error| with_sound_effect_failure_reason(error, "elevenlabs_http_failed"))?; + let duration_mode = if input.duration_seconds.is_some() { + assets::EditorSoundEffectDurationMode::Manual + } else { + assets::EditorSoundEffectDurationMode::Auto + }; + let metadata = assets::EditorSoundEffectGenerationMetadataV2::try_new( + input.user_prompt.clone(), + actual_prompt.clone(), + duration_mode, + input.duration_seconds, + generated.duration_seconds, + input.loop_enabled, + ) + .map_err(|message| { + sound_effect_stage_error( + "duration_probe_failed", + platform_audio::ELEVENLABS_PROVIDER, + message, + ) + })?; + let generation_inputs = build_editor_sound_effect_generation_inputs( + input.original_generation_inputs.as_ref(), + &metadata, + ) + .map_err(|error| error.with_detail_field("reason", json!("writeback_failed")))?; + let actual_duration_seconds = generated.duration_seconds; + let persisted_audio = stages + .persist_audio(generated.audio) + .await + .map_err(|error| with_sound_effect_failure_reason(error, "oss_failed"))?; + let writeback = stages + .writeback(SoundEffectWritebackInput { + actual_prompt: actual_prompt.clone(), + generation_inputs, + persisted_audio: persisted_audio.clone(), + }) + .await + .map_err(|error| error.with_detail_field("reason", json!("writeback_failed")))?; + + Ok(assets::EditorAudioGenerateResponse { + ok: true, + audio_src: persisted_audio.audio_src, + object_key: Some(persisted_audio.object_key), + asset_object_id: Some(persisted_audio.asset_object_id), + width: EDITOR_AUDIO_WIDTH, + height: EDITOR_AUDIO_HEIGHT, + source_type: "generated".to_string(), + prompt: input.user_prompt, + actual_prompt: Some(actual_prompt), + model: input.model, + provider: platform_audio::ELEVENLABS_PROVIDER.to_string(), + task_id: input.task_id, + price_mud_points: input.price_mud_points, + audio_kind: "sound-effect".to_string(), + duration_seconds: Some(actual_duration_seconds), + loop_enabled: Some(input.loop_enabled), + project: writeback.completed_project, + resource: writeback.resource, + asset: writeback.asset, + queue_state: None, + }) +} + +pub(super) fn with_sound_effect_failure_reason( + error: AppError, + default_reason: &'static str, +) -> AppError { + if error + .details() + .and_then(|details| details.get("reason")) + .and_then(Value::as_str) + .is_some_and(|reason| !reason.trim().is_empty()) + { + error + } else { + error.with_detail_field("reason", json!(default_reason)) + } +} + +fn sound_effect_stage_error( + reason: &'static str, + provider: &'static str, + message: impl Into, +) -> AppError { + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": provider, + "reason": reason, + "message": message.into(), + })) +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::*; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum FailurePoint { + TranslationInvalid, + TranslationUpstream, + ElevenLabsHttp, + InvalidAudio, + DurationProbe, + OssPut, + OssHead, + AssetConfirm, + AssetBind, + ProjectResource, + AccountAsset, + CanvasWriteback, + } + + impl FailurePoint { + fn reason(self) -> &'static str { + match self { + Self::TranslationInvalid => "translation_invalid", + Self::TranslationUpstream => "translation_upstream_failed", + Self::ElevenLabsHttp => "elevenlabs_http_failed", + Self::InvalidAudio => "invalid_audio", + Self::DurationProbe => "duration_probe_failed", + Self::OssPut | Self::OssHead => "oss_failed", + Self::AssetConfirm + | Self::AssetBind + | Self::ProjectResource + | Self::AccountAsset + | Self::CanvasWriteback => "writeback_failed", + } + } + } + + #[derive(Clone, Debug, Default)] + struct StageCounters { + llm: usize, + provider_post: usize, + oss_put: usize, + oss_head: usize, + asset_confirm: usize, + asset_bind: usize, + project_resource: usize, + account_asset: usize, + canvas_writeback: usize, + provider_request: Option, + writeback_input: Option, + } + + struct MockStages { + failure: Option, + counters: Mutex, + } + + impl MockStages { + fn new(failure: Option) -> Self { + Self { + failure, + counters: Mutex::new(StageCounters::default()), + } + } + + fn snapshot(&self) -> StageCounters { + self.counters.lock().expect("stage lock").clone() + } + + fn injected_error(&self, point: FailurePoint) -> Option { + (self.failure == Some(point)).then(|| { + sound_effect_stage_error(point.reason(), "mock-sfx-stage", "injected failure") + }) + } + } + + impl SoundEffectWorkerStages for MockStages { + async fn translate(&self, _user_prompt: &str) -> Result { + self.counters.lock().expect("stage lock").llm += 1; + for point in [ + FailurePoint::TranslationInvalid, + FailurePoint::TranslationUpstream, + ] { + if let Some(error) = self.injected_error(point) { + return Err(error); + } + } + Ok("Bright metallic coin pickup.".to_string()) + } + + async fn generate( + &self, + request: ElevenLabsSoundEffectRequest, + ) -> Result { + { + let mut counters = self.counters.lock().expect("stage lock"); + counters.provider_post += 1; + counters.provider_request = Some(request); + } + for point in [ + FailurePoint::ElevenLabsHttp, + FailurePoint::InvalidAudio, + FailurePoint::DurationProbe, + ] { + if let Some(error) = self.injected_error(point) { + return Err(error); + } + } + Ok(ElevenLabsGeneratedSoundEffect { + audio: DownloadedAudio { + bytes: b"mock-mp3-after-adapter-validation".to_vec(), + mime_type: "audio/mpeg".to_string(), + extension: "mp3".to_string(), + }, + duration_seconds: 7.42, + }) + } + + async fn persist_audio( + &self, + _audio: DownloadedAudio, + ) -> Result { + self.counters.lock().expect("stage lock").oss_put += 1; + if let Some(error) = self.injected_error(FailurePoint::OssPut) { + return Err(error); + } + self.counters.lock().expect("stage lock").oss_head += 1; + if let Some(error) = self.injected_error(FailurePoint::OssHead) { + return Err(error); + } + self.counters.lock().expect("stage lock").asset_confirm += 1; + if let Some(error) = self.injected_error(FailurePoint::AssetConfirm) { + return Err(error); + } + self.counters.lock().expect("stage lock").asset_bind += 1; + if let Some(error) = self.injected_error(FailurePoint::AssetBind) { + return Err(error); + } + Ok(PersistedSoundEffectAudio { + asset_object_id: "asset-object-sfx-t6".to_string(), + object_key: "generated/editor/sfx-t6.mp3".to_string(), + audio_src: "/generated-editor-audios/sfx-t6.mp3".to_string(), + }) + } + + async fn writeback( + &self, + input: SoundEffectWritebackInput, + ) -> Result { + { + let mut counters = self.counters.lock().expect("stage lock"); + counters.project_resource += 1; + counters.writeback_input = Some(input.clone()); + } + if let Some(error) = self.injected_error(FailurePoint::ProjectResource) { + return Err(error); + } + self.counters.lock().expect("stage lock").account_asset += 1; + if let Some(error) = self.injected_error(FailurePoint::AccountAsset) { + return Err(error); + } + self.counters.lock().expect("stage lock").canvas_writeback += 1; + if let Some(error) = self.injected_error(FailurePoint::CanvasWriteback) { + return Err(error); + } + + let authoritative = json!({ + "prompt": "金币拾取", + "actualPrompt": input.actual_prompt, + "model": assets::EDITOR_SOUND_EFFECT_MODEL, + "provider": platform_audio::ELEVENLABS_PROVIDER, + "taskId": "job-sfx-t6", + "durationSeconds": input.generation_inputs["soundEffect"]["actualDurationSeconds"], + "loop": input.generation_inputs["soundEffect"]["loop"], + "generationInputs": input.generation_inputs, + }); + Ok(SoundEffectWritebackOutput { + completed_project: Some(json!({ "layer": authoritative.clone() })), + resource: Some(authoritative.clone()), + asset: Some(authoritative), + }) + } + } + + #[derive(Clone, Debug, Default)] + struct BillingCounters { + consume_attempts: usize, + charges: usize, + refunds: usize, + } + + struct MockBilling { + sufficient_balance: bool, + counters: Mutex, + } + + impl MockBilling { + fn new(sufficient_balance: bool) -> Self { + Self { + sufficient_balance, + counters: Mutex::new(BillingCounters::default()), + } + } + + fn snapshot(&self) -> BillingCounters { + self.counters.lock().expect("billing lock").clone() + } + } + + impl SoundEffectWorkerBilling for MockBilling { + async fn execute(&self, operation: Fut) -> Result + where + Fut: Future>, + { + self.counters.lock().expect("billing lock").consume_attempts += 1; + if !self.sufficient_balance { + return Err( + AppError::from_status(StatusCode::CONFLICT).with_details(json!({ + "provider": "wallet", + "reason": "insufficient_balance", + "message": "余额不足", + })), + ); + } + self.counters.lock().expect("billing lock").charges += 1; + let result = operation.await; + if result.is_err() { + self.counters.lock().expect("billing lock").refunds += 1; + } + result + } + } + + fn worker_input(duration_seconds: Option, loop_enabled: bool) -> SoundEffectWorkerInput { + SoundEffectWorkerInput { + user_prompt: "金币拾取".to_string(), + model: assets::EDITOR_SOUND_EFFECT_MODEL.to_string(), + duration_seconds, + loop_enabled, + task_id: "job-sfx-t6".to_string(), + price_mud_points: 5, + original_generation_inputs: Some(json!({ + "source": "editor-agent", + "conversationId": "conversation-t6", + "fields": [{ "title": "伪造", "value": "forged" }], + "soundEffect": { "actualPrompt": "forged", "loop": !loop_enabled }, + })), + } + } + + fn error_reason(error: &AppError) -> Option<&str> { + error + .details() + .and_then(|details| details.get("reason")) + .and_then(Value::as_str) + } + + #[tokio::test] + async fn success_matrix_keeps_one_provider_post_and_authoritative_values_equal() { + for (duration_seconds, loop_enabled) in [ + (None, false), + (None, true), + (Some(5.25), false), + (Some(5.25), true), + ] { + let stages = MockStages::new(None); + let billing = MockBilling::new(true); + let response = execute_sound_effect_worker_with_billing( + &billing, + &stages, + worker_input(duration_seconds, loop_enabled), + ) + .await + .expect("mock SFX worker should complete"); + let stage = stages.snapshot(); + let billed = billing.snapshot(); + + assert_eq!(billed.consume_attempts, 1); + assert_eq!(billed.charges, 1); + assert_eq!(billed.refunds, 0); + assert_eq!(stage.llm, 1); + assert_eq!(stage.provider_post, 1); + assert_eq!(stage.oss_put, 1); + assert_eq!(stage.oss_head, 1); + assert_eq!(stage.asset_confirm, 1); + assert_eq!(stage.asset_bind, 1); + assert_eq!(stage.project_resource, 1); + assert_eq!(stage.account_asset, 1); + assert_eq!(stage.canvas_writeback, 1); + + let provider_request = stage + .provider_request + .expect("provider request should be captured"); + assert_eq!(provider_request.text, "Bright metallic coin pickup."); + assert_eq!(provider_request.duration_seconds, duration_seconds); + assert_eq!(provider_request.loop_enabled, loop_enabled); + assert_eq!(response.prompt, "金币拾取"); + assert_eq!( + response.actual_prompt.as_deref(), + Some("Bright metallic coin pickup.") + ); + assert_eq!(response.model, assets::EDITOR_SOUND_EFFECT_MODEL); + assert_eq!(response.provider, platform_audio::ELEVENLABS_PROVIDER); + assert_eq!(response.task_id, "job-sfx-t6"); + assert_eq!(response.duration_seconds, Some(7.42)); + assert_eq!(response.loop_enabled, Some(loop_enabled)); + + let writeback = stage.writeback_input.expect("writeback should be captured"); + let metadata = &writeback.generation_inputs["soundEffect"]; + assert_eq!(metadata["schemaVersion"], 2); + assert_eq!(metadata["userPrompt"], response.prompt); + assert_eq!(metadata["actualPrompt"], response.actual_prompt.unwrap()); + assert_eq!(metadata["model"], response.model); + assert_eq!( + metadata["requestedDurationSeconds"], + json!(duration_seconds) + ); + assert_eq!(metadata["actualDurationSeconds"], 7.42); + assert_eq!(metadata["loop"], loop_enabled); + assert_eq!( + writeback.generation_inputs["fields"][0]["value"], + "金币拾取" + ); + assert_eq!( + response.resource.as_ref().expect("resource")["taskId"], + response.task_id + ); + assert_eq!( + response.asset.as_ref().expect("asset")["actualPrompt"], + "Bright metallic coin pickup." + ); + assert_eq!( + response.project.as_ref().expect("project")["layer"]["loop"], + loop_enabled + ); + } + } + + #[tokio::test] + async fn insufficient_balance_stops_before_llm_provider_oss_and_writeback() { + let stages = MockStages::new(None); + let billing = MockBilling::new(false); + let error = execute_sound_effect_worker_with_billing( + &billing, + &stages, + worker_input(Some(5.0), false), + ) + .await + .expect_err("insufficient balance should fail before the worker future is polled"); + let stage = stages.snapshot(); + let billed = billing.snapshot(); + + assert_eq!(error_reason(&error), Some("insufficient_balance")); + assert_eq!(billed.consume_attempts, 1); + assert_eq!(billed.charges, 0); + assert_eq!(billed.refunds, 0); + assert_eq!(stage.llm, 0); + assert_eq!(stage.provider_post, 0); + assert_eq!(stage.oss_put, 0); + assert_eq!(stage.project_resource, 0); + } + + #[tokio::test] + async fn every_failure_stage_refunds_once_and_never_retries_elevenlabs() { + let cases = [ + FailurePoint::TranslationInvalid, + FailurePoint::TranslationUpstream, + FailurePoint::ElevenLabsHttp, + FailurePoint::InvalidAudio, + FailurePoint::DurationProbe, + FailurePoint::OssPut, + FailurePoint::OssHead, + FailurePoint::AssetConfirm, + FailurePoint::AssetBind, + FailurePoint::ProjectResource, + FailurePoint::AccountAsset, + FailurePoint::CanvasWriteback, + ]; + + for failure in cases { + let stages = MockStages::new(Some(failure)); + let billing = MockBilling::new(true); + let error = execute_sound_effect_worker_with_billing( + &billing, + &stages, + worker_input(None, true), + ) + .await + .expect_err("injected stage failure should fail the worker"); + let stage = stages.snapshot(); + let billed = billing.snapshot(); + + assert_eq!(error_reason(&error), Some(failure.reason()), "{failure:?}"); + assert_eq!(billed.consume_attempts, 1, "{failure:?}"); + assert_eq!(billed.charges, 1, "{failure:?}"); + assert_eq!(billed.refunds, 1, "{failure:?}"); + assert!(stage.provider_post <= 1, "{failure:?}"); + if matches!( + failure, + FailurePoint::TranslationInvalid | FailurePoint::TranslationUpstream + ) { + assert_eq!(stage.provider_post, 0, "{failure:?}"); + } + if matches!( + failure, + FailurePoint::ElevenLabsHttp + | FailurePoint::InvalidAudio + | FailurePoint::DurationProbe + ) { + assert_eq!(stage.oss_put, 0, "{failure:?}"); + } + if matches!(failure, FailurePoint::OssPut | FailurePoint::OssHead) { + assert_eq!(stage.project_resource, 0, "{failure:?}"); + } + } + } +} diff --git a/server-rs/crates/platform-audio/src/elevenlabs.rs b/server-rs/crates/platform-audio/src/elevenlabs.rs index e3fc66785..6af03ebb1 100644 --- a/server-rs/crates/platform-audio/src/elevenlabs.rs +++ b/server-rs/crates/platform-audio/src/elevenlabs.rs @@ -228,8 +228,8 @@ async fn read_elevenlabs_sound_effect_response( } let bytes = bytes.freeze(); - let duration_seconds = - crate::mp3::probe_mp3_duration_seconds(bytes.clone()).map_err(invalid_elevenlabs_audio)?; + let duration_seconds = crate::mp3::probe_mp3_duration_seconds(bytes.clone()) + .map_err(elevenlabs_duration_probe_failed)?; Ok(ElevenLabsGeneratedSoundEffect { audio: DownloadedAudio { @@ -251,7 +251,11 @@ fn normalize_content_type(content_type: &str) -> String { } fn invalid_elevenlabs_audio(message: impl Into) -> AudioError { - AudioError::missing_audio_for(ELEVENLABS_PROVIDER, message) + AudioError::missing_audio_with_reason_for(ELEVENLABS_PROVIDER, "invalid_audio", message) +} + +fn elevenlabs_duration_probe_failed(message: impl Into) -> AudioError { + AudioError::missing_audio_with_reason_for(ELEVENLABS_PROVIDER, "duration_probe_failed", message) } fn map_elevenlabs_reqwest_error(endpoint: &str, body: bool, error: reqwest::Error) -> AudioError { diff --git a/server-rs/crates/platform-audio/src/error.rs b/server-rs/crates/platform-audio/src/error.rs index 428fe0d3f..eff75b005 100644 --- a/server-rs/crates/platform-audio/src/error.rs +++ b/server-rs/crates/platform-audio/src/error.rs @@ -45,6 +45,7 @@ pub enum AudioError { MissingAudio { provider: &'static str, message: String, + reason_code: Option<&'static str>, }, } @@ -71,6 +72,14 @@ impl AudioError { } } + /// 可安全写入任务记录和指标的稳定失败分类；不会携带 provider 正文或凭据。 + pub fn reason_code(&self) -> Option<&'static str> { + match self { + Self::MissingAudio { reason_code, .. } => *reason_code, + _ => None, + } + } + pub fn status_hint(&self) -> AudioStatusHint { match self { Self::InvalidConfig { .. } => AudioStatusHint::ServiceUnavailable, @@ -213,6 +222,19 @@ impl AudioError { Self::MissingAudio { provider, message: message.into(), + reason_code: None, + } + } + + pub fn missing_audio_with_reason_for( + provider: &'static str, + reason_code: &'static str, + message: impl Into, + ) -> Self { + Self::MissingAudio { + provider, + message: message.into(), + reason_code: Some(reason_code), } } } diff --git a/server-rs/crates/platform-audio/tests/elevenlabs.rs b/server-rs/crates/platform-audio/tests/elevenlabs.rs index c9df08581..30d49ca17 100644 --- a/server-rs/crates/platform-audio/tests/elevenlabs.rs +++ b/server-rs/crates/platform-audio/tests/elevenlabs.rs @@ -251,12 +251,21 @@ fn octet_stream_and_missing_content_type_require_and_accept_real_mp3() { #[test] fn explicit_wrong_mime_empty_and_corrupt_bodies_are_rejected() { let cases = [ - success_response(Some("text/html"), TEST_MP3), - success_response(Some("application/json"), br#"{"error":"no"}"#), - success_response(Some("audio/mpeg"), b""), - success_response(Some("audio/mpeg"), b"ID3-corrupt"), + ( + success_response(Some("text/html"), TEST_MP3), + "invalid_audio", + ), + ( + success_response(Some("application/json"), br#"{"error":"no"}"#), + "invalid_audio", + ), + (success_response(Some("audio/mpeg"), b""), "invalid_audio"), + ( + success_response(Some("audio/mpeg"), b"ID3-corrupt"), + "duration_probe_failed", + ), ]; - for response in cases { + for (response, expected_reason) in cases { let (base_url, _captured_request, server) = spawn_single_response_server(response); let settings = settings(base_url, 2_000); let client = build_elevenlabs_audio_http_client(&settings) @@ -270,6 +279,7 @@ fn explicit_wrong_mime_empty_and_corrupt_bodies_are_rejected() { )) .expect_err("invalid audio response should fail"); assert_eq!(error.provider(), ELEVENLABS_PROVIDER); + assert_eq!(error.reason_code(), Some(expected_reason)); server.join().expect("mock server should finish"); } } @@ -404,6 +414,9 @@ fn content_length_precheck_accepts_the_limit_and_rejects_limit_plus_one() { error.message().contains("超过 40 MiB"), should_be_size_error ); + if should_be_size_error { + assert_eq!(error.reason_code(), Some("invalid_audio")); + } assert_eq!(request_count.load(Ordering::SeqCst), 1); } } @@ -447,4 +460,5 @@ fn chunked_limit_plus_one_is_rejected_without_unbounded_allocation() { server.join().expect("mock server should finish"); assert!(error.message().contains("超过 40 MiB")); + assert_eq!(error.reason_code(), Some("invalid_audio")); } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs index 79daa4032..7e5e582fc 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs @@ -12,8 +12,8 @@ pub struct GenerateSoundEffectTool; #[derive(Debug, Clone)] pub enum GenerateSoundEffectError { InvalidModel(String), - InvalidDuration(u8), - PromptNotProvided, + InvalidDuration(f64), + InvalidPrompt(String), } impl Display for GenerateSoundEffectError { @@ -25,14 +25,9 @@ impl Display for GenerateSoundEffectError { ), Self::InvalidDuration(duration) => write!( f, - "{duration} is not a valid sound effect duration; supported durations: {}", - GenerateSoundEffectTool::SUPPORTED_DURATIONS - .iter() - .map(u8::to_string) - .collect::>() - .join(", ") + "{duration} is not a valid sound effect duration; use null for automatic duration or a number from 0.5 through 30" ), - Self::PromptNotProvided => write!(f, "sound effect prompt not provided"), + Self::InvalidPrompt(message) => write!(f, "invalid sound effect prompt: {message}"), } } } @@ -45,15 +40,17 @@ pub struct GenerateSoundEffectToolArgs { #[serde(default = "default_sound_effect_model")] pub model: String, #[serde(default = "default_sound_effect_duration")] - pub duration: u8, + pub duration: Option, + #[serde(rename = "loop", default)] + pub loop_enabled: bool, } fn default_sound_effect_model() -> String { ELEVENLABS_SOUND_EFFECT_MODEL.to_string() } -fn default_sound_effect_duration() -> u8 { - GenerateSoundEffectTool::DEFAULT_DURATION +fn default_sound_effect_duration() -> Option { + Some(GenerateSoundEffectTool::DEFAULT_DURATION) } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -77,7 +74,15 @@ impl Tool for GenerateSoundEffectTool { "properties": { "prompt": { "type": "string", "description": "音效内容、材质、节奏和情绪描述。" }, "model": { "type": "string", "enum": [ELEVENLABS_SOUND_EFFECT_MODEL], "default": ELEVENLABS_SOUND_EFFECT_MODEL, "description": "音效固定使用 ElevenLabs。" }, - "duration": { "type": "integer", "enum": GenerateSoundEffectTool::SUPPORTED_DURATIONS, "default": GenerateSoundEffectTool::DEFAULT_DURATION, "description": "音效时长（秒）。" }, + "duration": { + "anyOf": [ + { "type": "number", "minimum": 0.5, "maximum": 30 }, + { "type": "null" } + ], + "default": GenerateSoundEffectTool::DEFAULT_DURATION, + "description": "音效时长（秒）；null 表示由 ElevenLabs 自动决定。" + }, + "loop": { "type": "boolean", "default": false, "description": "是否生成可循环衔接的音效。" }, }, "required": ["prompt"], "additionalProperties": false @@ -89,7 +94,7 @@ impl Tool for GenerateSoundEffectTool { args: Self::Args, ) -> impl Future> + Send { async move { - self.validate_args(&args)?; + self.normalize_args(args)?; Ok(GenerateSoundEffectToolOutput { message: PENDING_USER_CONFIRMATION_MESSAGE.to_string(), }) @@ -107,23 +112,30 @@ impl Tool for GenerateSoundEffectTool { impl GenerateSoundEffectTool { pub const DEFAULT_MODEL: &'static str = ELEVENLABS_SOUND_EFFECT_MODEL; - pub const DEFAULT_DURATION: u8 = 5; - pub const SUPPORTED_DURATIONS: &'static [u8] = &[2, 3, 4, 5, 6, 7, 8, 9, 10]; + pub const DEFAULT_DURATION: f64 = 5.0; + + pub fn normalize_args( + &self, + mut args: GenerateSoundEffectToolArgs, + ) -> Result { + if args.model != ELEVENLABS_SOUND_EFFECT_MODEL { + return Err(GenerateSoundEffectError::InvalidModel(args.model)); + } + let prompt = platform_audio::validate_sound_effect_prompt(&args.prompt) + .map_err(|error| GenerateSoundEffectError::InvalidPrompt(error.to_string()))?; + args.prompt = prompt.prompt.to_string(); + args.duration = platform_audio::validate_sound_effect_duration_seconds(args.duration) + .map_err(|_| { + GenerateSoundEffectError::InvalidDuration(args.duration.unwrap_or(f64::NAN)) + })?; + Ok(args) + } pub fn validate_args( &self, args: &GenerateSoundEffectToolArgs, ) -> Result<(), GenerateSoundEffectError> { - if args.prompt.trim().is_empty() { - return Err(GenerateSoundEffectError::PromptNotProvided); - } - if args.model != ELEVENLABS_SOUND_EFFECT_MODEL { - return Err(GenerateSoundEffectError::InvalidModel(args.model.clone())); - } - if !Self::SUPPORTED_DURATIONS.contains(&args.duration) { - return Err(GenerateSoundEffectError::InvalidDuration(args.duration)); - } - Ok(()) + self.normalize_args(args.clone()).map(|_| ()) } pub fn format_execute_message( @@ -144,21 +156,22 @@ impl GenerateSoundEffectTool { mod tests { use super::*; - fn args(duration: u8) -> GenerateSoundEffectToolArgs { + fn args(duration: Option) -> GenerateSoundEffectToolArgs { GenerateSoundEffectToolArgs { prompt: "按钮点击声".to_string(), model: GenerateSoundEffectTool::DEFAULT_MODEL.to_string(), duration, + loop_enabled: false, } } #[test] - fn validates_schema_durations_and_default() { - assert_eq!(GenerateSoundEffectTool::DEFAULT_DURATION, 5); - for duration in GenerateSoundEffectTool::SUPPORTED_DURATIONS { + fn validates_auto_decimal_boundaries_and_default() { + assert_eq!(GenerateSoundEffectTool::DEFAULT_DURATION, 5.0); + for duration in [None, Some(0.5), Some(1.234_567_89), Some(30.0)] { assert!( GenerateSoundEffectTool - .validate_args(&args(*duration)) + .validate_args(&args(duration)) .is_ok() ); } @@ -166,11 +179,27 @@ mod tests { #[tokio::test] async fn planning_rejects_durations_outside_schema() { - for duration in [1, 11, u8::MAX] { + for duration in [0.49, 30.01, f64::INFINITY] { assert!(matches!( - GenerateSoundEffectTool.call(args(duration)).await, + GenerateSoundEffectTool.call(args(Some(duration))).await, Err(GenerateSoundEffectError::InvalidDuration(value)) if value == duration )); } } + + #[test] + fn normalization_uses_the_shared_ecmascript_prompt_contract() { + let normalized = GenerateSoundEffectTool + .normalize_args(GenerateSoundEffectToolArgs { + prompt: "\u{feff}金币拾取\u{2003}".to_string(), + model: GenerateSoundEffectTool::DEFAULT_MODEL.to_string(), + duration: None, + loop_enabled: true, + }) + .expect("canonical prompt should pass"); + + assert_eq!(normalized.prompt, "金币拾取"); + assert_eq!(normalized.duration, None); + assert!(normalized.loop_enabled); + } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs index a9205f524..69daaf15b 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs @@ -85,7 +85,8 @@ mod tests { assert_eq!(video.resolution, "720p"); assert_eq!(video.sound, "on"); assert_eq!(sound.model, ELEVENLABS_SOUND_EFFECT_MODEL); - assert_eq!(sound.duration, 5); + assert_eq!(sound.duration, Some(5.0)); + assert!(!sound.loop_enabled); assert_eq!(music.model, SUNO_DEFAULT_MODEL); } @@ -109,12 +110,6 @@ mod tests { "sound": null }))) .expect("null video defaults should deserialize as omitted fields"); - let sound: GenerateSoundEffectToolArgs = - serde_json::from_value(null_tool_args_as_missing(json!({ - "prompt": "按钮点击声", - "duration": null - }))) - .expect("null sound defaults should deserialize as omitted fields"); assert_eq!(image.aspect_ratio, "1:1"); assert_eq!(image.image_size, "1K"); @@ -122,7 +117,6 @@ mod tests { assert_eq!(video.duration_seconds, 4); assert_eq!(video.resolution, "720p"); assert_eq!(video.sound, "on"); - assert_eq!(sound.duration, 5); } #[test] @@ -187,7 +181,7 @@ mod tests { } #[test] - fn confirmation_schemas_expose_all_finite_choices_as_enums() { + fn confirmation_schemas_expose_frozen_model_and_parameter_boundaries() { let context = EditorToolContext::default(); let image = GenerateImageTool { context: context.clone(), @@ -224,11 +218,11 @@ mod tests { video["properties"]["resolution"]["enum"], json!(["480p", "720p", "1080p"]) ); - assert_eq!( - sound["properties"]["duration"]["enum"], - json!([2, 3, 4, 5, 6, 7, 8, 9, 10]) - ); - assert_eq!(sound["properties"]["duration"]["default"], json!(5)); + assert_eq!(sound["properties"]["duration"]["anyOf"][0]["minimum"], 0.5); + assert_eq!(sound["properties"]["duration"]["anyOf"][0]["maximum"], 30); + assert_eq!(sound["properties"]["duration"]["anyOf"][1]["type"], "null"); + assert_eq!(sound["properties"]["duration"]["default"], json!(5.0)); + assert_eq!(sound["properties"]["loop"]["default"], json!(false)); assert_eq!(video["properties"]["sound"]["default"], json!("on")); assert!(music["properties"].get("make_instrumental").is_none());