From c3b3b47c9ac39c0097fa94c8917fcf1ce09c780c Mon Sep 17 00:00:00 2001 From: Linghong Date: Fri, 7 Aug 2026 02:50:59 +0000 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=20SFX=20V2=20T3=20ElevenLabs?= =?UTF-8?q?=20=E9=9F=B3=E9=A2=91=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 ElevenLabs 直接二进制适配、固定请求契约和单次 POST 边界。 实现 40 MiB 有界读取、MIME/MP3 校验和 600 秒纯 Rust 时长探测。 接入服务端环境配置、失败关闭 settings guard 与持久化来源解耦。 补齐 mock 回归测试、环境模板和 T3 tracked 文档记录。 --- .env.example | 5 + deploy/env/api-server.env.example | 3 + ...计划】SFX生成优化V2.0任务拆解-2026-08-06.md | 4 +- ...】server-rs与SpacetimeDB数据契约-2026-05-15.md | 2 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 3 + ...编辑器】画板音乐生成入口设计-2026-06-18.md | 4 +- server-rs/Cargo.lock | 61 +++ server-rs/Cargo.toml | 1 + server-rs/crates/api-server/src/config.rs | 68 +++- .../vector_engine_audio_generation/persist.rs | 7 +- .../settings.rs | 74 +++- server-rs/crates/platform-audio/Cargo.toml | 5 + .../crates/platform-audio/src/elevenlabs.rs | 297 ++++++++++++++ server-rs/crates/platform-audio/src/error.rs | 71 +++- server-rs/crates/platform-audio/src/lib.rs | 13 +- server-rs/crates/platform-audio/src/mp3.rs | 137 +++++++ .../crates/platform-audio/src/persist.rs | 82 +++- .../crates/platform-audio/tests/elevenlabs.rs | 374 ++++++++++++++++++ .../tests/fixtures/vbr-id3.mp3.base64 | 1 + 19 files changed, 1190 insertions(+), 22 deletions(-) create mode 100644 server-rs/crates/platform-audio/src/elevenlabs.rs create mode 100644 server-rs/crates/platform-audio/src/mp3.rs create mode 100644 server-rs/crates/platform-audio/tests/elevenlabs.rs create mode 100644 server-rs/crates/platform-audio/tests/fixtures/vbr-id3.mp3.base64 diff --git a/.env.example b/.env.example index 58b202e3d..85a1c3c81 100644 --- a/.env.example +++ b/.env.example @@ -130,6 +130,11 @@ VECTOR_ENGINE_BASE_URL="https://api.vectorengine.cn" VECTOR_ENGINE_API_KEY="" VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS="1000000" +# ElevenLabs editor sound-effect generation is server-side only. +ELEVENLABS_BASE_URL="https://api.elevenlabs.io" +ELEVENLABS_API_KEY="" +ELEVENLABS_REQUEST_TIMEOUT_MS="180000" + # 阿里云 OSS 配置。 # Rust `server-rs` 的 `api-server` 会优先从 `.env` / `.env.local` 读取这些变量， # 用于签发浏览器 PostObject 直传票据，并保持 `/generated-*` 旧路径习惯。 diff --git a/deploy/env/api-server.env.example b/deploy/env/api-server.env.example index 0bcee9522..88f48ed31 100644 --- a/deploy/env/api-server.env.example +++ b/deploy/env/api-server.env.example @@ -86,6 +86,9 @@ VECTOR_ENGINE_BASE_URL=https://api.vectorengine.cn VECTOR_ENGINE_API_KEY= VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS=1000000 VECTOR_ENGINE_AUDIO_REQUEST_TIMEOUT_MS=180000 +ELEVENLABS_BASE_URL=https://api.elevenlabs.io +ELEVENLABS_API_KEY= +ELEVENLABS_REQUEST_TIMEOUT_MS=180000 HYPER3D_BASE_URL=https://api.hyper3d.com/api/v2 HYPER3D_API_KEY= diff --git a/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md b/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md index 10073d113..00099725d 100644 --- a/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md +++ b/docs/project-memory/plans/【实施计划】SFX生成优化V2.0任务拆解-2026-08-06.md @@ -2,7 +2,7 @@ 日期：`2026-08-06` -状态：`T1、T2 已完成，可以继续进入 T3、T4；T2 翻译 service 待 T5 正式接线，当前仍不是可发布切点` +状态：`T1、T2、T3 已完成，可以继续进入 T4；T2 翻译 service 与 T3 ElevenLabs adapter 均待 T5 正式接线，当前仍不是可发布切点` 开发分支：`feat/sound_opt` @@ -154,6 +154,8 @@ External v1 `model` 先删除首尾 Unicode `White_Space`，再按大小写敏 - 配置增加 `ELEVENLABS_BASE_URL / ELEVENLABS_API_KEY / ELEVENLABS_REQUEST_TIMEOUT_MS`，Key 只在服务端。 - 测试断言 429 / 5xx / timeout / 读取失败都只有一次 provider POST，不执行真实付费请求。 +实施记录（`2026-08-07`）：T3 已完成。`platform-audio` 已增加独立 ElevenLabs 直接二进制 adapter，固定 endpoint、header、query、model、influence、nullable 小数时长和 Loop；专用 HTTP client 禁止重定向且没有 retry。成功响应先做 `40 MiB` Content-Length 预检，再以 `limit + 1` 有界读取，严格执行 MIME / 真实 MP3 门禁，并以纯 Rust MP3 probe 取得有限正实际时长和独立 `600s` 上限；请求格式 `mp3_44100_128` 不扩展为返回码率硬校验。配置、环境模板和 fail-closed settings guard 已落地，持久化准备已把 provider / file stem 从轮询任务枚举中最小解耦；正式 handler、Worker、计费、OSS 写回、External v1 和 OpenAPI 均未接线，继续归属 T5。 + ### T4：SFX 前端 controller、预设与参数 UI - 新增 SFX Prompt 纯模型、预设纯模型和 dialog-scoped controller；抽取音频预设跑马灯内核，BGM / SFX 保留各自 wrapper。 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index 7917a1a3a..393400598 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -308,7 +308,7 @@ Responses 的终态载荷既是工具调用的恢复源，也是正文的恢复 - Match3D 1:1 容器 UI：VectorEngine `/v1/images/edits` multipart 参考图。该容器参考图是后端生图协议输入，必须通过 `include_bytes!` 随 `api-server` 编译进二进制，避免 API 单独发布或运行目录缺少 `public/` 时生成失败。 - 敲木鱼敲击物和背景环境图：VectorEngine `/v1/images/edits`，模型固定 `gpt-image-2`。敲击物支持 multipart 多参考图，第一张固定为后端内嵌默认木鱼图，用户上传图只作为新主题参考；prompt 必须要求 `1:1` 单一纯绿色 `#00FF00 / RGB(0,255,0)` 绿幕背景主体图，并禁止黑底、白底、棋盘格和任何实底背景。当前敲击物和返回按钮上传 OSS 前只做服务端绿幕去背后处理，避免泛抠图误伤玉米等主体像素。背景环境图只使用第一步抠图完成后的透明敲击物图作为参考，prompt 必须要求中央主体预留区保持干净，中央 40% 区域禁止出现主题主体、主体局部特写、轮廓影子或重复元素，主题元素只能作为外围氛围，且必须显式声明不继承任何绿色底色、绿幕底色或纯绿色画布。 - Hyper3D / Rodin：只保留后端安全代理和旧数据兼容；Rodin 提交、状态、下载和响应解析归属 `platform-hyper3d`，`api-server/src/hyper3d_generation.rs` 只做路由、配置和错误 envelope 映射；新 Match3D 草稿和批量新增不再生成 GLB。 -- 音频：视觉小说专用音频路由保留；VectorEngine Suno/Vidu provider 协议、任务提交/查询、音频 URL 提取、下载、MIME/extension 归一和 OSS put 请求准备归属 `platform-audio`。`api-server/src/vector_engine_audio_generation.rs` 只做路由、配置、计费、asset object confirm、entity binding 和错误 envelope 映射；拼图、抓大鹅和敲木鱼提示词生成音效入口暂时关闭，通用 `/api/creation/audio/*` 对这些目标返回 `410 Gone`。敲木鱼创作只接收上传 / 录音音频资产；前端选择或录音阶段只在浏览器本地处理待提交音频，统一限制裁切后最长 1 秒、裁掉前后声音过小片段，并用浏览器端近似响度算法平衡到 `-15 LKFS` 后做峰值保护。点击生成时才直传 OSS 并确认 `asset_object`，创作 JSON 只提交轻量 `WoodenFishAudioAsset`，不得继续上传 Data URL 音频；未提供时由 `api-server` 写回内置默认木鱼音 `/wooden-fish/default-hit-sound.mp3`。 +- 音频：视觉小说专用音频路由保留；VectorEngine Suno/Vidu provider 协议、任务提交/查询、音频 URL 提取、下载、MIME/extension 归一，以及 ElevenLabs SFX 单次同步二进制请求、`40 MiB` 有界读取、MP3 验证和 `600s` 技术异常上限内的实际时长探测均归属 `platform-audio`。ElevenLabs 直接 adapter 不进入 Suno/Vidu 的 submit + poll 枚举，OSS put 请求准备以显式 provider / file stem 描述来源。`api-server/src/vector_engine_audio_generation.rs` 只做路由、配置、计费、asset object confirm、entity binding 和错误 envelope 映射；拼图、抓大鹅和敲木鱼提示词生成音效入口暂时关闭，通用 `/api/creation/audio/*` 对这些目标返回 `410 Gone`。敲木鱼创作只接收上传 / 录音音频资产；前端选择或录音阶段只在浏览器本地处理待提交音频，统一限制裁切后最长 1 秒、裁掉前后声音过小片段，并用浏览器端近似响度算法平衡到 `-15 LKFS` 后做峰值保护。点击生成时才直传 OSS 并确认 `asset_object`，创作 JSON 只提交轻量 `WoodenFishAudioAsset`，不得继续上传 Data URL 音频；未提供时由 `api-server` 写回内置默认木鱼音 `/wooden-fish/default-hit-sound.mp3`。 - OSS：私有 generated path 进入浏览器前必须通过 `/api/assets/read-url` 换签；不要裸请求 `/generated-*`。请求参数的安全语义不能混用：`legacyPublicPath` 是历史公开作品兼容口，只允许 `platform_oss::LEGACY_PUBLIC_PREFIXES` 中的 curated 前缀匿名换签；`objectKey` 是正式对象引用，绝不能复用该前缀旁路，必须查询 `asset_object` 并校验配置 bucket、精确 key、`PublicRead` 或当前 owner。External OpenAPI 的 `/api/external/v1/assets/read-url` 还必须有 `editor:asset` scope，并始终以 API Key 绑定的 `owner_user_id` 执行同一 owner 校验；后台跨账号预览只能走管理员鉴权后的 `/admin/api/assets/read-url`。`/api/assets/read-bytes` 与主站 read-url 共用完全相同的授权，默认仍应由浏览器使用 signed URL 直读，bytes 只作跨域字节读取 fallback。前端如果收到同一 OSS bucket 的完整 `https://*.oss-*.aliyuncs.com/generated-*` 地址，也必须先归一为 legacy path 后走同一换签链路，避免裸连私有 bucket 403 或绕过签名缓存。OSS 签名、读签名、HEAD 和 PUT 的结构化日志由 `platform-oss` 输出，排查资产写入 / 确认失败时优先按 `operation`、`object_key` / `key_prefix`、`status_class`、`error_kind` 和 `elapsed_ms` 下钻。新上传 generated 私有对象默认写入 `Cache-Control: public, max-age=31536000, immutable`；旧对象若缺该头，只能依赖 `ETag` / `Last-Modified` 协商缓存，应通过 OSS 元数据刷新或 CDN 配置补齐，不要恢复 api-server 静态代理。`editor-agent/` 前缀只用于服务端内部读写画布 Agent 会话消息文档，不属于浏览器直传 legacy public prefix；`/api/assets/direct-upload-tickets` 必须拒绝 `legacyPrefix=editor-agent`，内部读取只允许 `editor-agent/{conversationId}.json` 形态。 - 外部 API 失败审计：外部供应商调用未成功时，`api-server` 必须发送 OTLP 失败事件并写入 `tracking_event`。VectorEngine 图片 provider 在 `platform-image` 内输出结构化日志和 `PlatformImageFailureAudit`，覆盖 `request_send`、`response_body`、`upstream_status`、`response_parse`、`missing_image` 和 `image_download` 阶段；编辑器 `screenColor=auto` 的 gpt-5-mini 背景色决策同样必须审计每次已发出的 LLM 调用失败，包括传输 / 超时、上游拒绝、响应体解析、空响应和返回候选外颜色；即使随后降级默认背景色并继续主流程也不得只记 warning。`api-server` 将这些失败映射成 `external_api_call_failure`，`scope_kind = module`、`scope_id = provider`、`module_key = external-api`。metadata 固定包含 provider、endpoint、operation、failureStage、statusCode、statusClass、timeout、retryable、errorMessage、latencyMs、promptChars、referenceImageCount、imageModel、rawExcerpt，以及在调用方可获得上下文时补充的 `userId`（触发者）和 `profileId`（草稿 / 作品 / 场景作用域）。图片生成入口应优先把 owner user id 和 profile id 透传到失败审计，不要只保留 provider 级聚合，否则很难按“谁触发、哪个作品触发”定位问题。普通调用入库优先复用 tracking outbox，outbox 不可写或保护阈值拒绝时回退同步写 SpacetimeDB；不得新增前端兜底或在 SpacetimeDB reducer 内做外部 I/O。`bgfilter-worker` 是受限资源例外：它使用共享 tracking outbox 基础目录下独立的 `bgfilter-worker/` 子目录，provider 失败审计在 spawn 前受进程级 `1024` 硬上限保护并由 shutdown tracker 跟踪；满载、outbox 缺失、保护阈值拒绝或写盘失败时直接丢弃并观测，不回退同步直写 SpacetimeDB。优雅退出先排空已获准任务的 enqueue，再封存并尽力 flush；进程被强杀时只有已 enqueue 记录可在下次启动重放。 - 外部生成运行记录：所有外部生成编排的完成态统一写入 `tracking_event`，`event_key = external_generation_run`，`scope_kind = module`，`scope_id = provider`，`module_key = external-generation`。metadata 固定包含 `runId`、`provider`、`operation`、`requestLabel`、`requestPayload`、`status`、`success`、`failureReason`、`providerRequestId`、`resultPayload`、`startedAtMicros`、`completedAtMicros` 和 `durationMs`。这类记录只用于运行审计和排障，不再走 `ai_task` 旧表。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 8e1c58a19..bdb5710af 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -183,6 +183,8 @@ spacetime sql "SELECT * FROM runtime_setting LIMIT 1" --server http:/ 本地 `.env`、`.env.local` 或 `.env.secrets.local` 修改后必须重启 `api-server` 才会生效；若已经通过 `npm run dev` 启动完整联调，可在该终端输入 `rs api-server`。排查图片编辑器 VectorEngine 生成链路时，确认 `VECTOR_ENGINE_BASE_URL`、`VECTOR_ENGINE_API_KEY` 和 `VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS` 只在本地或服务器密钥文件中配置，不能写入 Git。`VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS` 是单次 attempt 的配置上限，默认 `1000000`；配置加载层允许显式值低于该默认值，不再在读取环境变量时强制抬高。业务模型和 VectorEngine provider 首选请求都使用 `gpt-image-2`，符合条件时才回退到兜底模型 `gpt-image-2-c`；图片协议、URL / base64 响应解析、远端图片下载和 provider 侧结构化日志在 `server-rs/crates/platform-image`，`api-server` 只做编辑器请求编排、OSS / asset 持久化、计费和失败审计落库。`platform-image` 会在 JSON 生成和 multipart 编辑请求发送前按同一 GPT-image-2 family 规则归一显式像素尺寸；若请求发送失败，先按同一 `request_id` 查看 provider 日志与 `external_api_call_failure.metadata_json.errorSource`，当前 multipart `/v1/images/edits` 单独强制 HTTP/1.1。 +编辑器 ElevenLabs 音效生成只从服务端读取 `ELEVENLABS_BASE_URL`、`ELEVENLABS_API_KEY` 和 `ELEVENLABS_REQUEST_TIMEOUT_MS`，timeout 默认 `180000ms`；base URL 或 Key 缺失时失败关闭，不回退 Vidu。生产 API 与 external-generation worker 通过共享 API env 取得同一配置，模板见 `deploy/env/api-server.env.example`；Key 不得进入 Web/Vite 环境、命令参数、日志、fixture 或仓库。普通测试只使用 loopback mock，禁止把真实付费请求作为 T3 自动验收。 + VectorEngine 图片生成 / 编辑在 `request_send` 阶段出现 `timeout`、`connect`、libcurl 35 SSL connect reset、libcurl 56 receive error / `unexpected eof while reading`、recv failure 等临时传输错误，或在 `upstream_status` 阶段收到 408 / 429 / 5xx（例如 Nginx HTML `502 Bad Gateway`）时，`platform-image` 会在一次业务请求总上限 5 次内处理；multipart 图片编辑每次重试都会重新构造 form，避免复用已消费的 body。首个 provider attempt 使用 `gpt-image-2`；明确模型不可用、408 / 非拒绝类 429 / 5xx、响应解析失败或非拒绝类缺图时，下一 attempt 直接切兜底模型 `gpt-image-2-c`，之后只在剩余次数内重试兜底模型。发送 / 连接错误无法确认上游是否已受理，只重试同一首选模型，不切模型；认证、普通参数、安全拒绝、图片下载和 budget 错误同样不切。worker 从 job 开始的同一时钟起点计算绝对 deadline，常规保留最后 `60` 秒给审计、OSS 和终态写回；job 预算小于 `120` 秒时保留一半。VectorEngine 单次 attempt timeout 取配置值和剩余 provider 预算的较小值；退避或模型切换后已没有下一次 attempt 的预算时立即停止。该 deadline 覆盖参考图、provider 请求 / 响应和响应图片下载的整次 provider future，但只在 worker 进程内通过 `RequestContext` 传递；普通 HTTP / `inline` 没有该 deadline，继续保持原有 timeout 和重试行为。日志中 `VectorEngine 首选图片模型失败，切换兼容模型` 会携带 `fallback_from_model` / `fallback_to_model`；即使回退成功，首选模型错误仍写入 `external_api_call_failure`，成功运行摘要的 `recoveredFailureCount` 同时递增。排查生产失败时应同时统计 fallback / retry 日志和最终 audit，避免把一次用户请求内的多次发送误判成多个用户请求。这项收口不修改 lease 续租 / fencing、迟到写回仲裁、attempt 耗尽与原子退款语义。 图片编辑器生成属于持久队列长任务：提交接口返回 job 后，前端通过 `/api/runtime/external-generation/jobs/{jobId}` 与编辑器项目资源状态收敛。生产排查小程序或 WebView `Failed to fetch` 时，若 Nginx access log 为 `499`、`upstream_status=-`，先按提交请求的 `request_id`、job id、worker 日志和 `external_api_call_failure` 对齐真实任务，不把客户端断开直接判定为 provider 失败。 @@ -699,6 +701,7 @@ OpenTelemetry 现阶段默认开启 OTLP traces / metrics / logs，但本地日 - `GENARRATIVE_DATABASE_BACKUP_*` - `GENARRATIVE_LLM_*` - `VECTOR_ENGINE_*` +- `ELEVENLABS_*` - ~~`APIMART_*`~~（已弃用，LLM 文本调用统一迁移到 VectorEngine） - `APIMART_*`（历史残留，创意 Agent LLM 已迁移到 VectorEngine） - `HYPER3D_*` diff --git a/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md b/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md index 169cbd3c0..fa28aba84 100644 --- a/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md +++ b/docs/【编辑器】画板音乐生成入口设计-2026-06-18.md @@ -140,7 +140,7 @@ SFX V2 已完成产品与技术口径冻结，T0 已通过，T1、T2 已完成 - `isEnglish = true` 是 LLM 语义判断，程序侧还必须执行 Unicode Script 门禁：候选至少包含一个 Script=Latin 的 alphabetic code point，且所有 alphabetic code point 的 Script 都是 Latin；Common / Inherited 的数字、标点、空白和声音设计符号允许保留。Han、Hiragana、Katakana、Hangul、Cyrillic、Greek、Arabic 等非 Latin 字母都失败。 - 第一次成功响应的候选为空、`isEnglish != true`、非英文 Script 门禁失败、结构非法、格式 / 保真判断失败、超过 2048 Unicode code points 或 `finish_reason = length` 时，使用同一份原始 `userPrompt` 进行唯一一次业务重试；不得把第一次候选当作新事实源。第二次仍使用 `max_output_tokens = 8192`，第二次 `finish_reason = length` 按最终翻译失败处理。首轮 `content_filter` 直接失败；transport、timeout 或上游最终失败只使用该轮 `LlmClient` 内部 transport retry，不额外开启第二业务语义轮。 - 第二次任何失败都是最终翻译失败；不返回候选，不调用 ElevenLabs，job 进入失败 / 退款链路。只有验收通过后才允许构造 `actualPrompt` 并调用 provider。 -- ElevenLabs endpoint 为 `POST /v1/sound-generation`，鉴权 `xi-api-key` 只在服务端 header 注入。body 固定包含 `text = actualPrompt`、`model_id = eleven_text_to_sound_v2`、`duration_seconds = null | frozen manual value`、`loop = frozen boolean`、`prompt_influence = 0.3`；query 固定 `output_format=mp3_44100_128`。调用方不能覆盖 model、influence 或 output format。 +- ElevenLabs endpoint 为 `POST /v1/sound-generation`，鉴权 `xi-api-key` 只在服务端 header 注入。body 固定包含 `text = actualPrompt`、`model_id = eleven_text_to_sound_v2`、`duration_seconds = null | frozen manual value`、`loop = frozen boolean`、`prompt_influence = 0.3`；query 固定 `output_format=mp3_44100_128`。调用方不能覆盖 model、influence 或 output format；`mp3_44100_128` 是请求格式，不把响应必须精确为 `128 kbps` 扩展成硬校验，合法结果不得只因探测码率不同被拒绝。 - ElevenLabs 没有本链路可用的幂等键，所以 provider POST 不做自动 retry；浏览器正式生成 POST 也保持 0 次 unsafe retry，队列 `max_attempts = 1`。一个平台 job 最多调用一次 ElevenLabs。 - 成功响应是 MP3 二进制。复用现有 `MAX_GENERATED_AUDIO_BYTES = 40 * 1024 * 1024`，即 `40 MiB`：`Content-Length` 存在且大于该值时在读取前拒绝；长度头缺失或未超限时仍有界流式读取到 `MAX_GENERATED_AUDIO_BYTES + 1`，实际累计达到 `40 MiB + 1 byte` 时失败，不得先无限读入内存。空 body、超限、HTML / JSON 错误页、损坏 MP3 和无法探测正时长的响应均失败。明确接受 `audio/mpeg` / `audio/mp3`；`application/octet-stream` 或缺失 Content-Type 只有在真实 MP3 探测成功时才可接受，显式非音频类型不能仅靠扩展名回退通过。 - 使用纯 Rust MP3 探测获得实际时长，持久化和响应的 `durationSeconds` 必须来自 MP3 而不是请求时长。实际时长必须是有限正数且不大于独立技术异常上限 `600s`；`600s` 允许，任何大于 `600s` 的结果拒绝。该上限不由请求最大 `30s` 推导，也不要求实际时长接近请求值；通过 MP3、MIME、字节和时长门禁的 `30.5s-600s` 结果均可接受。 @@ -165,7 +165,7 @@ SFX V2 已完成产品与技术口径冻结，T0 已通过，T1、T2 已完成 | 其它未知非空值 | `400 BAD_REQUEST` | 不入队 | - 所有接受形态在定价、预扣和 enqueue 前收敛为同一 canonical model，因而 omitted / null / 空白 / 显式新模型不得产生不同幂等 payload。`audio1.0` 与未知模型必须在入队前失败，并由测试证明零入队、零预扣、零 LLM 和零 provider。成功响应增加可选 `durationSeconds` 和 `loop`；Rust DTO、`docs/openapi/genarrative-external-v1.openapi.json`、`202 / poll / final response`、Idempotency-Key 重放测试和 compact result 必须同批保持一致。 -- ElevenLabs 配置只允许从服务端 `ELEVENLABS_BASE_URL`、`ELEVENLABS_API_KEY`、`ELEVENLABS_REQUEST_TIMEOUT_MS` 读取，Key 不进入浏览器、请求体、日志、fixture、共享文档或 Git。运行配置缺失时失败关闭，不回退 Vidu。 +- ElevenLabs 配置只允许从服务端 `ELEVENLABS_BASE_URL`、`ELEVENLABS_API_KEY`、`ELEVENLABS_REQUEST_TIMEOUT_MS` 读取，request timeout 默认 `180000ms`；Key 不进入浏览器、请求体、日志、fixture、共享文档或 Git。base URL 或 Key 缺失时失败关闭，不回退 Vidu。 ## BGM Prompt 优化 V1.0 diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index b8bf25a53..75331a1fb 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -1588,6 +1588,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "enum-as-inner" version = "0.6.1" @@ -4065,10 +4074,13 @@ dependencies = [ name = "platform-audio" version = "0.1.0" dependencies = [ + "base64 0.22.1", + "bytes", "platform-oss", "regex", "reqwest", "serde_json", + "symphonia", "tokio", "tracing", "urlencoding", @@ -5851,6 +5863,55 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symphonia" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" +dependencies = [ + "lazy_static", + "symphonia-bundle-mp3", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/server-rs/Cargo.toml b/server-rs/Cargo.toml index ee89fbca2..1274350bf 100644 --- a/server-rs/Cargo.toml +++ b/server-rs/Cargo.toml @@ -121,6 +121,7 @@ serde_urlencoded = "0.7" sha1 = "0.10" sha2 = "0.10" socket2 = "0.6" +symphonia = { version = "0.5", default-features = false, features = ["mp3"] } spacetimedb = "=2.7.0" spacetimedb-sdk = "=2.7.0" spacetimedb-lib = { version = "=2.7.0", default-features = false } diff --git a/server-rs/crates/api-server/src/config.rs b/server-rs/crates/api-server/src/config.rs index 77eb6a15c..25c6f41a5 100644 --- a/server-rs/crates/api-server/src/config.rs +++ b/server-rs/crates/api-server/src/config.rs @@ -17,6 +17,7 @@ const DEFAULT_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS: u64 = 600; const DEFAULT_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS: u64 = 900; const DEFAULT_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS: u64 = 1_800; pub(crate) const DEFAULT_VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS: u64 = 1_000_000; +pub(crate) const DEFAULT_ELEVENLABS_REQUEST_TIMEOUT_MS: u64 = 180_000; const DEFAULT_EDITOR_BGFILTER_BASE_URL: &str = "http://58.87.105.82/bgfilter"; const DEFAULT_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS: u64 = 5_000; const BGFILTER_ATTEMPT_SAFETY_FACTOR: u64 = 2; @@ -189,6 +190,9 @@ pub struct AppConfig { pub vector_engine_api_key: Option, pub vector_engine_image_request_timeout_ms: u64, pub vector_engine_audio_request_timeout_ms: u64, + pub elevenlabs_base_url: String, + pub elevenlabs_api_key: Option, + pub elevenlabs_request_timeout_ms: u64, pub hyper3d_base_url: String, pub hyper3d_api_key: Option, pub hyper3d_model_request_timeout_ms: u64, @@ -451,6 +455,9 @@ impl Default for AppConfig { vector_engine_api_key: None, vector_engine_image_request_timeout_ms: DEFAULT_VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS, vector_engine_audio_request_timeout_ms: 180_000, + elevenlabs_base_url: String::new(), + elevenlabs_api_key: None, + elevenlabs_request_timeout_ms: DEFAULT_ELEVENLABS_REQUEST_TIMEOUT_MS, hyper3d_base_url: "https://api.hyper3d.com/api/v2".to_string(), hyper3d_api_key: None, hyper3d_model_request_timeout_ms: 180_000, @@ -1169,6 +1176,16 @@ impl AppConfig { config.vector_engine_audio_request_timeout_ms = vector_engine_audio_request_timeout_ms; } + if let Some(elevenlabs_base_url) = read_first_non_empty_env(&["ELEVENLABS_BASE_URL"]) { + config.elevenlabs_base_url = elevenlabs_base_url; + } + config.elevenlabs_api_key = read_first_non_empty_env(&["ELEVENLABS_API_KEY"]); + if let Some(elevenlabs_request_timeout_ms) = + read_first_positive_u64_env(&["ELEVENLABS_REQUEST_TIMEOUT_MS"]) + { + config.elevenlabs_request_timeout_ms = elevenlabs_request_timeout_ms; + } + if let Some(hyper3d_base_url) = read_first_non_empty_env(&["HYPER3D_BASE_URL", "RODIN_BASE_URL"]) { @@ -1596,7 +1613,7 @@ mod tests { AppConfig, DEFAULT_EDITOR_BGFILTER_BASE_URL, DEFAULT_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS, DEFAULT_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD, - DEFAULT_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS, + DEFAULT_EDITOR_BGFILTER_SINGLE_IMAGE_ESTIMATE_MS, DEFAULT_ELEVENLABS_REQUEST_TIMEOUT_MS, DEFAULT_EXTERNAL_GENERATION_WORKER_JOB_TIMEOUT_SECONDS, DEFAULT_EXTERNAL_GENERATION_WORKER_LEASE_SECONDS, DEFAULT_EXTERNAL_GENERATION_WORKER_LONG_JOB_TIMEOUT_SECONDS, ExternalGenerationMode, @@ -1806,6 +1823,55 @@ mod tests { } } + #[test] + fn default_elevenlabs_settings_fail_closed_with_a_bounded_timeout() { + let config = AppConfig::default(); + + assert!(config.elevenlabs_base_url.is_empty()); + assert!(config.elevenlabs_api_key.is_none()); + assert_eq!( + config.elevenlabs_request_timeout_ms, + DEFAULT_ELEVENLABS_REQUEST_TIMEOUT_MS + ); + } + + #[test] + fn from_env_reads_elevenlabs_settings() { + let _guard = ENV_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("env lock should not poison"); + + unsafe { + std::env::remove_var("ELEVENLABS_BASE_URL"); + std::env::remove_var("ELEVENLABS_API_KEY"); + std::env::remove_var("ELEVENLABS_REQUEST_TIMEOUT_MS"); + std::env::set_var( + "ELEVENLABS_BASE_URL", + "https://elevenlabs.internal.example/v1", + ); + std::env::set_var("ELEVENLABS_API_KEY", "elevenlabs-test-key"); + std::env::set_var("ELEVENLABS_REQUEST_TIMEOUT_MS", "190000"); + } + + let config = AppConfig::from_env(); + assert_eq!( + config.elevenlabs_base_url, + "https://elevenlabs.internal.example/v1" + ); + assert_eq!( + config.elevenlabs_api_key.as_deref(), + Some("elevenlabs-test-key") + ); + assert_eq!(config.elevenlabs_request_timeout_ms, 190_000); + + unsafe { + std::env::remove_var("ELEVENLABS_BASE_URL"); + std::env::remove_var("ELEVENLABS_API_KEY"); + std::env::remove_var("ELEVENLABS_REQUEST_TIMEOUT_MS"); + } + } + #[test] fn from_env_reads_non_public_models_and_urls() { let _guard = ENV_LOCK diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/persist.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/persist.rs index 5900466fb..8de160dd8 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation/persist.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/persist.rs @@ -3,7 +3,10 @@ use module_assets::{ AssetObjectAccessPolicy, build_asset_entity_binding_input, build_asset_object_upsert_input, generate_asset_binding_id, generate_asset_object_id, }; -use platform_audio::{DownloadedAudio, GeneratedAudioPersistInput, GeneratedAudioPersistTarget}; +use platform_audio::{ + DownloadedAudio, GeneratedAudioPersistInput, GeneratedAudioPersistSource, + GeneratedAudioPersistTarget, +}; use serde_json::json; use crate::{http_error::AppError, platform_errors::map_oss_error, state::AppState}; @@ -43,7 +46,7 @@ pub(super) async fn persist_generated_audio_asset( platform_audio::prepare_generated_audio_put_request(GeneratedAudioPersistInput { owner_user_id: owner_user_id.to_string(), task_id: task_id.to_string(), - task_kind, + source: GeneratedAudioPersistSource::from_task_kind(task_kind), target: GeneratedAudioPersistTarget { entity_kind: target.entity_kind.clone(), entity_id: target.entity_id.clone(), diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/settings.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/settings.rs index 3b5c698f5..4bf401048 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation/settings.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/settings.rs @@ -1,5 +1,5 @@ use axum::http::StatusCode; -use platform_audio::VectorEngineAudioSettings; +use platform_audio::{ElevenLabsAudioSettings, VectorEngineAudioSettings}; use serde_json::json; use crate::{http_error::AppError, state::AppState}; @@ -42,3 +42,75 @@ pub(super) fn require_vector_engine_audio_settings( request_timeout_ms: state.config.vector_engine_audio_request_timeout_ms.max(1), }) } + +#[allow(dead_code)] +pub(super) fn require_elevenlabs_audio_settings( + state: &AppState, +) -> Result { + let base_url = state + .config + .elevenlabs_base_url + .trim() + .trim_end_matches('/'); + if base_url.is_empty() { + return Err( + AppError::from_status(StatusCode::SERVICE_UNAVAILABLE).with_details(json!({ + "provider": platform_audio::ELEVENLABS_PROVIDER, + "reason": "ELEVENLABS_BASE_URL 未配置", + })), + ); + } + + let api_key = state + .config + .elevenlabs_api_key + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + AppError::from_status(StatusCode::SERVICE_UNAVAILABLE).with_details(json!({ + "provider": platform_audio::ELEVENLABS_PROVIDER, + "reason": "ELEVENLABS_API_KEY 未配置", + })) + })?; + + Ok(ElevenLabsAudioSettings { + base_url: base_url.to_string(), + api_key: api_key.to_string(), + request_timeout_ms: state.config.elevenlabs_request_timeout_ms.max(1), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::AppConfig; + + #[test] + fn elevenlabs_settings_require_base_url_and_key_without_exposing_the_key() { + let state = AppState::new(AppConfig::default()).expect("state should build"); + let missing_base = require_elevenlabs_audio_settings(&state) + .expect_err("missing ElevenLabs base URL should fail closed"); + assert_eq!(missing_base.status_code(), StatusCode::SERVICE_UNAVAILABLE); + + let mut config = AppConfig::default(); + config.elevenlabs_base_url = " https://api.elevenlabs.test/v1/ ".to_string(); + let state = AppState::new(config).expect("state should build"); + let missing_key = require_elevenlabs_audio_settings(&state) + .expect_err("missing ElevenLabs key should fail closed"); + assert_eq!(missing_key.status_code(), StatusCode::SERVICE_UNAVAILABLE); + + let mut config = AppConfig::default(); + config.elevenlabs_base_url = " https://api.elevenlabs.test/v1/ ".to_string(); + config.elevenlabs_api_key = Some(" elevenlabs-test-key ".to_string()); + config.elevenlabs_request_timeout_ms = 12_345; + let state = AppState::new(config).expect("state should build"); + let settings = require_elevenlabs_audio_settings(&state) + .expect("complete ElevenLabs settings should pass"); + + assert_eq!(settings.base_url, "https://api.elevenlabs.test/v1"); + assert_eq!(settings.api_key, "elevenlabs-test-key"); + assert_eq!(settings.request_timeout_ms, 12_345); + assert!(!format!("{settings:?}").contains(settings.api_key.as_str())); + } +} diff --git a/server-rs/crates/platform-audio/Cargo.toml b/server-rs/crates/platform-audio/Cargo.toml index a640f0f30..dde5f97eb 100644 --- a/server-rs/crates/platform-audio/Cargo.toml +++ b/server-rs/crates/platform-audio/Cargo.toml @@ -5,10 +5,15 @@ version.workspace = true license.workspace = true [dependencies] +bytes = { workspace = true } platform-oss = { workspace = true } reqwest = { workspace = true, features = ["json", "rustls-tls"] } regex = { workspace = true } serde_json = { workspace = true } +symphonia = { workspace = true } tokio = { workspace = true, features = ["time"] } tracing = { workspace = true } urlencoding = { workspace = true } + +[dev-dependencies] +base64 = { workspace = true } diff --git a/server-rs/crates/platform-audio/src/elevenlabs.rs b/server-rs/crates/platform-audio/src/elevenlabs.rs new file mode 100644 index 000000000..4ef1ea57f --- /dev/null +++ b/server-rs/crates/platform-audio/src/elevenlabs.rs @@ -0,0 +1,297 @@ +use std::error::Error; + +use bytes::BytesMut; +use reqwest::header; +use serde_json::{Value, json}; + +use crate::{ + AudioError, DownloadedAudio, MAX_GENERATED_AUDIO_BYTES, validate_sound_effect_duration_seconds, + validate_sound_effect_prompt, +}; + +pub const ELEVENLABS_PROVIDER: &str = "elevenlabs"; +pub const ELEVENLABS_SOUND_EFFECT_MODEL: &str = "eleven_text_to_sound_v2"; +pub const ELEVENLABS_SOUND_EFFECT_OUTPUT_FORMAT: &str = "mp3_44100_128"; +pub const ELEVENLABS_SOUND_EFFECT_PROMPT_INFLUENCE: f64 = 0.3; +const ELEVENLABS_SOUND_GENERATION_PATH: &str = "/v1/sound-generation"; + +#[derive(Clone)] +pub struct ElevenLabsAudioSettings { + pub base_url: String, + pub api_key: String, + pub request_timeout_ms: u64, +} + +impl std::fmt::Debug for ElevenLabsAudioSettings { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ElevenLabsAudioSettings") + .field("base_url", &self.base_url) + .field("api_key", &"[redacted]") + .field("request_timeout_ms", &self.request_timeout_ms) + .finish() + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ElevenLabsSoundEffectRequest { + pub text: String, + pub duration_seconds: Option, + pub loop_enabled: bool, +} + +#[derive(Clone, Debug)] +pub struct ElevenLabsGeneratedSoundEffect { + pub audio: DownloadedAudio, + pub duration_seconds: f64, +} + +pub fn build_elevenlabs_audio_http_client( + settings: &ElevenLabsAudioSettings, +) -> Result { + reqwest::Client::builder() + .timeout(std::time::Duration::from_millis( + settings.request_timeout_ms.max(1), + )) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| { + AudioError::invalid_config_for( + ELEVENLABS_PROVIDER, + format!("构造 ElevenLabs 音效生成 HTTP 客户端失败：{error}"), + ) + }) +} + +pub fn elevenlabs_sound_generation_endpoint(base_url: &str) -> String { + let normalized = base_url.trim().trim_end_matches('/'); + let root = normalized + .strip_suffix("/v1") + .unwrap_or(normalized) + .trim_end_matches('/'); + format!("{root}{ELEVENLABS_SOUND_GENERATION_PATH}") +} + +pub fn build_elevenlabs_sound_effect_body( + request: &ElevenLabsSoundEffectRequest, +) -> Result { + let prompt = validate_sound_effect_prompt(&request.text).map_err(|error| { + AudioError::invalid_request_for(ELEVENLABS_PROVIDER, error.message().to_string()) + })?; + let duration_seconds = validate_sound_effect_duration_seconds(request.duration_seconds) + .map_err(|error| { + AudioError::invalid_request_for(ELEVENLABS_PROVIDER, error.message().to_string()) + })?; + Ok(json!({ + "text": prompt.prompt, + "model_id": ELEVENLABS_SOUND_EFFECT_MODEL, + "duration_seconds": duration_seconds, + "loop": request.loop_enabled, + "prompt_influence": ELEVENLABS_SOUND_EFFECT_PROMPT_INFLUENCE, + })) +} + +pub async fn generate_elevenlabs_sound_effect( + http_client: &reqwest::Client, + settings: &ElevenLabsAudioSettings, + request: ElevenLabsSoundEffectRequest, +) -> Result { + let endpoint = elevenlabs_sound_generation_endpoint(&settings.base_url); + let body = build_elevenlabs_sound_effect_body(&request)?; + let response = http_client + .post(endpoint.as_str()) + .query(&[("output_format", ELEVENLABS_SOUND_EFFECT_OUTPUT_FORMAT)]) + .header("xi-api-key", settings.api_key.as_str()) + .header( + header::ACCEPT, + "audio/mpeg,audio/mp3,application/octet-stream", + ) + .header(header::ACCEPT_ENCODING, "identity") + .header(header::CONTENT_TYPE, "application/json") + .json(&body) + .send() + .await + .map_err(|error| map_elevenlabs_reqwest_error(endpoint.as_str(), false, error))?; + + read_elevenlabs_sound_effect_response(response, endpoint.as_str()).await +} + +async fn read_elevenlabs_sound_effect_response( + mut response: reqwest::Response, + endpoint: &str, +) -> Result { + let status = response.status(); + if !status.is_success() { + return Err(AudioError::upstream_for( + ELEVENLABS_PROVIDER, + format!("ElevenLabs 音效生成失败：HTTP {}", status.as_u16()), + status.as_u16(), + "", + )); + } + if response + .content_length() + .is_some_and(|length| length > MAX_GENERATED_AUDIO_BYTES as u64) + { + return Err(invalid_elevenlabs_audio("生成音频超过 40 MiB 大小上限")); + } + + let content_type = response + .headers() + .get(header::CONTENT_TYPE) + .map(|value| { + value + .to_str() + .map(normalize_content_type) + .map_err(|_| invalid_elevenlabs_audio("生成音频 Content-Type 非法")) + }) + .transpose()?; + match content_type.as_deref() { + Some("audio/mpeg" | "audio/mp3" | "application/octet-stream") | None => {} + Some(_) => return Err(invalid_elevenlabs_audio("生成结果不是允许的 MP3 MIME 类型")), + } + + let mut bytes = BytesMut::with_capacity( + response + .content_length() + .and_then(|length| usize::try_from(length).ok()) + .unwrap_or(0) + .min(MAX_GENERATED_AUDIO_BYTES), + ); + let read_limit = MAX_GENERATED_AUDIO_BYTES + 1; + while let Some(chunk) = response + .chunk() + .await + .map_err(|error| map_elevenlabs_reqwest_error(endpoint, true, error))? + { + let remaining = read_limit.saturating_sub(bytes.len()); + bytes.extend_from_slice(&chunk[..chunk.len().min(remaining)]); + if bytes.len() > MAX_GENERATED_AUDIO_BYTES { + return Err(invalid_elevenlabs_audio("生成音频超过 40 MiB 大小上限")); + } + } + if bytes.is_empty() { + return Err(invalid_elevenlabs_audio("生成音频内容为空")); + } + + let bytes = bytes.freeze(); + let duration_seconds = + crate::mp3::probe_mp3_duration_seconds(bytes.clone()).map_err(invalid_elevenlabs_audio)?; + + Ok(ElevenLabsGeneratedSoundEffect { + audio: DownloadedAudio { + bytes: bytes.to_vec(), + mime_type: "audio/mpeg".to_string(), + extension: "mp3".to_string(), + }, + duration_seconds, + }) +} + +fn normalize_content_type(content_type: &str) -> String { + content_type + .split(';') + .next() + .map(str::trim) + .unwrap_or_default() + .to_ascii_lowercase() +} + +fn invalid_elevenlabs_audio(message: impl Into) -> AudioError { + AudioError::missing_audio_for(ELEVENLABS_PROVIDER, message) +} + +fn map_elevenlabs_reqwest_error(endpoint: &str, body: bool, error: reqwest::Error) -> AudioError { + AudioError::request_for( + ELEVENLABS_PROVIDER, + if body { + format!("读取 ElevenLabs 音效响应失败：{error}") + } else { + format!("请求 ElevenLabs 音效生成失败：{error}") + }, + Some(endpoint.to_string()), + error.is_timeout(), + error.is_connect(), + error.is_request(), + body || error.is_body(), + error.status().map(|status| status.as_u16()), + Error::source(&error).map(ToString::to_string), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn endpoint_normalization_appends_the_sound_generation_path_once() { + for base_url in [ + "https://api.elevenlabs.io", + "https://api.elevenlabs.io/", + "https://api.elevenlabs.io/v1", + " https://api.elevenlabs.io/v1/ ", + ] { + assert_eq!( + elevenlabs_sound_generation_endpoint(base_url), + "https://api.elevenlabs.io/v1/sound-generation" + ); + } + } + + #[test] + fn request_body_fixes_provider_owned_fields_and_preserves_duration_precision() { + for (duration_seconds, loop_enabled) in [ + (None, false), + (None, true), + (Some(1.234_567_89), false), + (Some(30.0), true), + ] { + let body = build_elevenlabs_sound_effect_body(&ElevenLabsSoundEffectRequest { + text: " crisp metallic coin pickup ".to_string(), + duration_seconds, + loop_enabled, + }) + .expect("valid ElevenLabs request body should build"); + + assert_eq!(body["text"], "crisp metallic coin pickup"); + assert_eq!(body["model_id"], ELEVENLABS_SOUND_EFFECT_MODEL); + assert_eq!(body["duration_seconds"], json!(duration_seconds)); + assert_eq!(body["loop"], loop_enabled); + assert_eq!(body["prompt_influence"], 0.3); + assert_eq!(body.as_object().expect("body should be an object").len(), 5); + } + } + + #[test] + fn settings_debug_redacts_the_api_key() { + let settings = ElevenLabsAudioSettings { + base_url: "https://api.elevenlabs.test".to_string(), + api_key: "elevenlabs-secret-test-key".to_string(), + request_timeout_ms: 1_000, + }; + let debug = format!("{settings:?}"); + + assert!(debug.contains("[redacted]")); + assert!(!debug.contains(settings.api_key.as_str())); + } + + #[test] + fn request_validation_errors_are_attributed_to_elevenlabs() { + let error = build_elevenlabs_sound_effect_body(&ElevenLabsSoundEffectRequest { + text: " ".to_string(), + duration_seconds: None, + loop_enabled: false, + }) + .expect_err("empty text should fail"); + + assert_eq!(error.provider(), ELEVENLABS_PROVIDER); + } + + #[test] + fn canonical_prompt_helper_still_preserves_non_whitespace_invisibles() { + assert_eq!( + crate::canonicalize_sound_effect_prompt("\u{0085}\u{feff}sound\u{200b}\u{0085}"), + "\u{feff}sound\u{200b}" + ); + } +} diff --git a/server-rs/crates/platform-audio/src/error.rs b/server-rs/crates/platform-audio/src/error.rs index ad4955dc6..428fe0d3f 100644 --- a/server-rs/crates/platform-audio/src/error.rs +++ b/server-rs/crates/platform-audio/src/error.rs @@ -93,15 +93,23 @@ impl AudioError { } pub fn invalid_config(message: impl Into) -> Self { + Self::invalid_config_for(VECTOR_ENGINE_PROVIDER, message) + } + + pub fn invalid_config_for(provider: &'static str, message: impl Into) -> Self { Self::InvalidConfig { - provider: VECTOR_ENGINE_PROVIDER, + provider, message: message.into(), } } pub fn invalid_request(message: impl Into) -> Self { + Self::invalid_request_for(VECTOR_ENGINE_PROVIDER, message) + } + + pub fn invalid_request_for(provider: &'static str, message: impl Into) -> Self { Self::InvalidRequest { - provider: VECTOR_ENGINE_PROVIDER, + provider, message: message.into(), } } @@ -115,9 +123,34 @@ impl AudioError { body: bool, status_code: Option, source: Option, + ) -> Self { + Self::request_for( + VECTOR_ENGINE_PROVIDER, + message, + endpoint, + timeout, + connect, + request, + body, + status_code, + source, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn request_for( + provider: &'static str, + message: impl Into, + endpoint: Option, + timeout: bool, + connect: bool, + request: bool, + body: bool, + status_code: Option, + source: Option, ) -> Self { Self::Request { - provider: VECTOR_ENGINE_PROVIDER, + provider, message: message.into(), endpoint, timeout, @@ -133,9 +166,23 @@ impl AudioError { message: impl Into, upstream_status: u16, raw_excerpt: impl Into, + ) -> Self { + Self::upstream_for( + VECTOR_ENGINE_PROVIDER, + message, + upstream_status, + raw_excerpt, + ) + } + + pub fn upstream_for( + provider: &'static str, + message: impl Into, + upstream_status: u16, + raw_excerpt: impl Into, ) -> Self { Self::Upstream { - provider: VECTOR_ENGINE_PROVIDER, + provider, message: message.into(), upstream_status, raw_excerpt: raw_excerpt.into(), @@ -143,16 +190,28 @@ impl AudioError { } pub fn response_parse(message: impl Into, raw_excerpt: impl Into) -> Self { + Self::response_parse_for(VECTOR_ENGINE_PROVIDER, message, raw_excerpt) + } + + pub fn response_parse_for( + provider: &'static str, + message: impl Into, + raw_excerpt: impl Into, + ) -> Self { Self::ResponseParse { - provider: VECTOR_ENGINE_PROVIDER, + provider, message: message.into(), raw_excerpt: raw_excerpt.into(), } } pub fn missing_audio(message: impl Into) -> Self { + Self::missing_audio_for(VECTOR_ENGINE_PROVIDER, message) + } + + pub fn missing_audio_for(provider: &'static str, message: impl Into) -> Self { Self::MissingAudio { - provider: VECTOR_ENGINE_PROVIDER, + provider, message: message.into(), } } diff --git a/server-rs/crates/platform-audio/src/lib.rs b/server-rs/crates/platform-audio/src/lib.rs index 0d78cea7a..08a1beac5 100644 --- a/server-rs/crates/platform-audio/src/lib.rs +++ b/server-rs/crates/platform-audio/src/lib.rs @@ -1,7 +1,9 @@ mod background_music_prompt; mod client; mod download; +mod elevenlabs; mod error; +mod mp3; mod persist; mod request; mod response; @@ -19,9 +21,18 @@ pub use client::{ submit_editor_sound_effect_task, submit_sound_effect_task, }; pub use download::{audio_mime_to_extension, download_generated_audio, normalize_audio_mime_type}; +pub use elevenlabs::{ + ELEVENLABS_PROVIDER, ELEVENLABS_SOUND_EFFECT_MODEL, ELEVENLABS_SOUND_EFFECT_OUTPUT_FORMAT, + ELEVENLABS_SOUND_EFFECT_PROMPT_INFLUENCE, ElevenLabsAudioSettings, + ElevenLabsGeneratedSoundEffect, ElevenLabsSoundEffectRequest, + build_elevenlabs_audio_http_client, build_elevenlabs_sound_effect_body, + elevenlabs_sound_generation_endpoint, generate_elevenlabs_sound_effect, +}; pub use error::{AudioError, AudioStatusHint}; +pub use mp3::MAX_GENERATED_AUDIO_DURATION_SECONDS; pub use persist::{ - GeneratedAudioPersistInput, GeneratedAudioPersistTarget, prepare_generated_audio_put_request, + GeneratedAudioPersistInput, GeneratedAudioPersistSource, GeneratedAudioPersistTarget, + prepare_generated_audio_put_request, }; pub use request::{ build_background_music_task_body, build_editor_background_music_task_body, diff --git a/server-rs/crates/platform-audio/src/mp3.rs b/server-rs/crates/platform-audio/src/mp3.rs new file mode 100644 index 000000000..7e0fc736f --- /dev/null +++ b/server-rs/crates/platform-audio/src/mp3.rs @@ -0,0 +1,137 @@ +use std::io::Cursor; + +use bytes::Bytes; +use symphonia::{ + core::{ + codecs::CODEC_TYPE_MP3, + errors::Error as SymphoniaError, + formats::FormatOptions, + io::{MediaSourceStream, MediaSourceStreamOptions}, + meta::MetadataOptions, + probe::Hint, + }, + default::get_probe, +}; + +pub const MAX_GENERATED_AUDIO_DURATION_SECONDS: f64 = 600.0; + +pub(crate) fn probe_mp3_duration_seconds(audio_bytes: Bytes) -> Result { + if audio_bytes.is_empty() { + return Err("MP3 内容为空".to_string()); + } + + let source = Box::new(Cursor::new(audio_bytes)); + let media_source_stream = MediaSourceStream::new( + source, + MediaSourceStreamOptions { + buffer_len: 64 * 1024, + }, + ); + let mut hint = Hint::new(); + hint.with_extension("mp3"); + let probed = get_probe() + .format( + &hint, + media_source_stream, + &FormatOptions::default(), + &MetadataOptions::default(), + ) + .map_err(|error| format!("无法识别 MP3：{error}"))?; + let mut format = probed.format; + let track = format + .default_track() + .filter(|track| track.codec_params.codec == CODEC_TYPE_MP3) + .ok_or_else(|| "响应不是 MP3 音频".to_string())?; + let track_id = track.id; + let time_base = track + .codec_params + .time_base + .ok_or_else(|| "MP3 缺少时间基准".to_string())?; + let mut end_timestamp = 0_u64; + + loop { + match format.next_packet() { + Ok(packet) => { + if packet.track_id() == track_id { + end_timestamp = end_timestamp.max(packet.ts().saturating_add(packet.dur())); + } + } + Err(SymphoniaError::IoError(error)) + if error.kind() == std::io::ErrorKind::UnexpectedEof => + { + break; + } + Err(error) => return Err(format!("读取 MP3 帧失败：{error}")), + } + } + + if end_timestamp == 0 { + return Err("MP3 不包含可用音频帧".to_string()); + } + let duration = time_base.calc_time(end_timestamp); + validate_generated_audio_duration_seconds(duration.seconds as f64 + duration.frac) +} + +pub(crate) fn validate_generated_audio_duration_seconds( + duration_seconds: f64, +) -> Result { + if duration_seconds.is_finite() + && duration_seconds > 0.0 + && duration_seconds <= MAX_GENERATED_AUDIO_DURATION_SECONDS + { + return Ok(duration_seconds); + } + Err(format!( + "MP3 实际时长必须是有限正数且不超过 {MAX_GENERATED_AUDIO_DURATION_SECONDS} 秒" + )) +} + +#[cfg(test)] +mod tests { + use base64::Engine; + + use super::*; + + #[test] + fn duration_guard_uses_only_the_independent_six_hundred_second_limit() { + for duration in [0.001, 30.5, 60.0, 600.0] { + assert_eq!( + validate_generated_audio_duration_seconds(duration) + .expect("finite positive durations through 600 seconds should pass"), + duration + ); + } + for duration in [0.0, -1.0, 600.000_001, f64::NAN, f64::INFINITY] { + assert!(validate_generated_audio_duration_seconds(duration).is_err()); + } + } + + #[test] + fn probe_reads_the_tracked_id3_mp3_fixture() { + let fixture = Bytes::from_static(include_bytes!( + "../../../../public/wooden-fish/default-hit-sound.mp3" + )); + let duration = probe_mp3_duration_seconds(fixture) + .expect("tracked ID3 MP3 fixture should have a duration"); + + assert!((0.65..=0.75).contains(&duration), "duration={duration}"); + } + + #[test] + fn probe_reads_vbr_mp3_with_id3_and_encoder_padding() { + let fixture = base64::engine::general_purpose::STANDARD + .decode(include_str!("../tests/fixtures/vbr-id3.mp3.base64").trim()) + .expect("VBR MP3 fixture should decode"); + let duration = probe_mp3_duration_seconds(Bytes::from(fixture)) + .expect("VBR MP3 fixture should have a duration"); + + assert!((0.39..=0.46).contains(&duration), "duration={duration}"); + } + + #[test] + fn probe_rejects_non_mp3_and_empty_payloads() { + for payload in [Bytes::new(), Bytes::from_static(b"error")] { + assert!(probe_mp3_duration_seconds(payload).is_err()); + } + } +} diff --git a/server-rs/crates/platform-audio/src/persist.rs b/server-rs/crates/platform-audio/src/persist.rs index b1e4b910c..79e3a3474 100644 --- a/server-rs/crates/platform-audio/src/persist.rs +++ b/server-rs/crates/platform-audio/src/persist.rs @@ -4,6 +4,21 @@ use platform_oss::{LegacyAssetPrefix, OssObjectAccess, OssPutObjectRequest}; use crate::{AudioTaskKind, DownloadedAudio}; +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GeneratedAudioPersistSource { + pub provider: String, + pub file_stem: String, +} + +impl GeneratedAudioPersistSource { + pub fn from_task_kind(task_kind: AudioTaskKind) -> Self { + Self { + provider: task_kind.provider().to_string(), + file_stem: task_kind.file_stem().to_string(), + } + } +} + #[derive(Clone, Debug)] pub struct GeneratedAudioPersistTarget { pub entity_kind: String, @@ -19,7 +34,7 @@ pub struct GeneratedAudioPersistTarget { pub struct GeneratedAudioPersistInput { pub owner_user_id: String, pub task_id: String, - pub task_kind: AudioTaskKind, + pub source: GeneratedAudioPersistSource, pub target: GeneratedAudioPersistTarget, pub audio: DownloadedAudio, } @@ -29,9 +44,7 @@ pub fn prepare_generated_audio_put_request( ) -> OssPutObjectRequest { let file_name = format!( "{}-{}.{}", - input.task_kind.file_stem(), - input.task_id, - input.audio.extension + input.source.file_stem, input.task_id, input.audio.extension ); OssPutObjectRequest { prefix: input.target.storage_prefix, @@ -55,7 +68,7 @@ pub fn prepare_generated_audio_put_request( input.owner_user_id.as_str(), input.target.profile_id.as_deref(), &input.target, - input.task_kind, + input.source.provider.as_str(), ), body: input.audio.bytes, } @@ -65,7 +78,7 @@ fn build_audio_asset_metadata( owner_user_id: &str, profile_id: Option<&str>, target: &GeneratedAudioPersistTarget, - task_kind: AudioTaskKind, + provider: &str, ) -> BTreeMap { let mut metadata = BTreeMap::from([ ("asset-kind".to_string(), target.asset_kind.clone()), @@ -73,7 +86,7 @@ fn build_audio_asset_metadata( ("entity-kind".to_string(), target.entity_kind.clone()), ("entity-id".to_string(), target.entity_id.clone()), ("slot".to_string(), target.slot.clone()), - ("provider".to_string(), task_kind.provider().to_string()), + ("provider".to_string(), provider.to_string()), ]); if let Some(profile_id) = profile_id { metadata.insert("profile-id".to_string(), profile_id.to_string()); @@ -104,3 +117,58 @@ fn sanitize_audio_path_segment(raw: &str, fallback: &str) -> String { collapsed.chars().take(80).collect() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn direct_provider_source_controls_file_stem_and_metadata_without_polling_kind() { + let request = prepare_generated_audio_put_request(GeneratedAudioPersistInput { + owner_user_id: "owner-1".to_string(), + task_id: "operation-1".to_string(), + source: GeneratedAudioPersistSource { + provider: crate::ELEVENLABS_PROVIDER.to_string(), + file_stem: "sound-effect".to_string(), + }, + target: GeneratedAudioPersistTarget { + entity_kind: "editor-project".to_string(), + entity_id: "project-1".to_string(), + slot: "sound-effect".to_string(), + asset_kind: "sound-effect".to_string(), + profile_id: None, + storage_prefix: LegacyAssetPrefix::EditorAgent, + storage_scope: "editor".to_string(), + }, + audio: DownloadedAudio { + bytes: vec![1, 2, 3], + mime_type: "audio/mpeg".to_string(), + extension: "mp3".to_string(), + }, + }); + + assert_eq!(request.file_name, "sound-effect-operation-1.mp3"); + assert_eq!( + request.metadata.get("provider").map(String::as_str), + Some(crate::ELEVENLABS_PROVIDER) + ); + } + + #[test] + fn polling_task_source_keeps_existing_provider_and_file_stem() { + assert_eq!( + GeneratedAudioPersistSource::from_task_kind(AudioTaskKind::BackgroundMusic), + GeneratedAudioPersistSource { + provider: crate::VECTOR_ENGINE_SUNO_PROVIDER.to_string(), + file_stem: "background-music".to_string(), + } + ); + assert_eq!( + GeneratedAudioPersistSource::from_task_kind(AudioTaskKind::SoundEffect), + GeneratedAudioPersistSource { + provider: crate::VECTOR_ENGINE_VIDU_PROVIDER.to_string(), + file_stem: "sound-effect".to_string(), + } + ); + } +} diff --git a/server-rs/crates/platform-audio/tests/elevenlabs.rs b/server-rs/crates/platform-audio/tests/elevenlabs.rs new file mode 100644 index 000000000..eaea9a8c2 --- /dev/null +++ b/server-rs/crates/platform-audio/tests/elevenlabs.rs @@ -0,0 +1,374 @@ +use std::{ + io::{Read, Write}, + net::{TcpListener, TcpStream}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + mpsc, + }, + thread, + time::{Duration, Instant}, +}; + +use platform_audio::{ + AudioError, ELEVENLABS_PROVIDER, ELEVENLABS_SOUND_EFFECT_MODEL, ElevenLabsAudioSettings, + ElevenLabsSoundEffectRequest, MAX_GENERATED_AUDIO_BYTES, build_elevenlabs_audio_http_client, + generate_elevenlabs_sound_effect, +}; + +const TEST_API_KEY: &str = "elevenlabs-test-key"; +const TEST_MP3: &[u8] = include_bytes!("../../../../public/wooden-fish/default-hit-sound.mp3"); + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime should build") +} + +fn settings(base_url: String, request_timeout_ms: u64) -> ElevenLabsAudioSettings { + ElevenLabsAudioSettings { + base_url, + api_key: TEST_API_KEY.to_string(), + request_timeout_ms, + } +} + +fn request() -> ElevenLabsSoundEffectRequest { + ElevenLabsSoundEffectRequest { + text: "crisp metallic coin pickup".to_string(), + duration_seconds: Some(1.234_567_89), + loop_enabled: true, + } +} + +fn read_http_request(stream: &mut TcpStream) -> Vec { + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .expect("mock request read timeout should configure"); + let mut request_bytes = Vec::new(); + let mut buffer = [0_u8; 4096]; + let mut expected_len = None; + loop { + let read = stream.read(&mut buffer).expect("mock request should read"); + if read == 0 { + break; + } + request_bytes.extend_from_slice(&buffer[..read]); + if expected_len.is_none() + && let Some(header_end) = find_header_end(&request_bytes) + { + let headers = String::from_utf8_lossy(&request_bytes[..header_end]); + let content_length = headers + .lines() + .find_map(|line| { + line.split_once(':').and_then(|(name, value)| { + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + }) + .unwrap_or(0); + expected_len = Some(header_end + 4 + content_length); + } + if expected_len.is_some_and(|length| request_bytes.len() >= length) { + break; + } + } + request_bytes +} + +fn find_header_end(bytes: &[u8]) -> Option { + bytes.windows(4).position(|window| window == b"\r\n\r\n") +} + +fn spawn_single_response_server( + response: Vec, +) -> (String, mpsc::Receiver>, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("mock server should bind"); + let address = listener + .local_addr() + .expect("mock address should be readable"); + let (request_sender, request_receiver) = mpsc::channel(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("mock request should connect"); + let request_bytes = read_http_request(&mut stream); + request_sender + .send(request_bytes) + .expect("captured request should send"); + stream + .write_all(&response) + .expect("mock response should write"); + }); + (format!("http://{address}"), request_receiver, server) +} + +fn spawn_counting_response_server( + response: Vec, + response_delay: Duration, +) -> (String, Arc, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("mock server should bind"); + let address = listener + .local_addr() + .expect("mock address should be readable"); + let request_count = Arc::new(AtomicUsize::new(0)); + let request_count_for_server = Arc::clone(&request_count); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("mock request should connect"); + request_count_for_server.fetch_add(1, Ordering::SeqCst); + let _ = read_http_request(&mut stream); + thread::sleep(response_delay); + let _ = stream.write_all(&response); + drop(stream); + + listener + .set_nonblocking(true) + .expect("mock listener should become nonblocking"); + let deadline = Instant::now() + Duration::from_millis(150); + while Instant::now() < deadline { + match listener.accept() { + Ok((mut retry_stream, _)) => { + request_count_for_server.fetch_add(1, Ordering::SeqCst); + let _ = read_http_request(&mut retry_stream); + let _ = retry_stream.write_all(&response); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("mock listener failed: {error}"), + } + } + }); + (format!("http://{address}"), request_count, server) +} + +fn success_response(content_type: Option<&str>, body: &[u8]) -> Vec { + let content_type_header = content_type + .map(|value| format!("Content-Type: {value}\r\n")) + .unwrap_or_default(); + let mut response = format!( + "HTTP/1.1 200 OK\r\n{content_type_header}Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(body); + response +} + +#[test] +fn direct_client_sends_the_fixed_contract_and_returns_probed_mp3() { + let (base_url, captured_request, server) = + spawn_single_response_server(success_response(Some("audio/mpeg"), TEST_MP3)); + let settings = settings(format!("{base_url}/v1/"), 2_000); + let client = + build_elevenlabs_audio_http_client(&settings).expect("ElevenLabs HTTP client should build"); + + let generated = runtime() + .block_on(generate_elevenlabs_sound_effect( + &client, + &settings, + request(), + )) + .expect("valid MP3 response should pass"); + let captured_request = captured_request + .recv_timeout(Duration::from_secs(2)) + .expect("mock request should be captured"); + server.join().expect("mock server should finish"); + + assert_eq!(generated.audio.bytes, TEST_MP3); + assert_eq!(generated.audio.mime_type, "audio/mpeg"); + assert_eq!(generated.audio.extension, "mp3"); + assert!((0.65..=0.75).contains(&generated.duration_seconds)); + + let header_end = find_header_end(&captured_request).expect("request headers should terminate"); + let headers = String::from_utf8_lossy(&captured_request[..header_end]).to_ascii_lowercase(); + let body: serde_json::Value = serde_json::from_slice(&captured_request[header_end + 4..]) + .expect("request body should be JSON"); + assert!( + headers.starts_with("post /v1/sound-generation?output_format=mp3_44100_128 http/1.1\r\n") + ); + assert!(headers.contains("xi-api-key: elevenlabs-test-key\r\n")); + assert!(headers.contains("accept-encoding: identity\r\n")); + assert_eq!(body["text"], "crisp metallic coin pickup"); + assert_eq!(body["model_id"], ELEVENLABS_SOUND_EFFECT_MODEL); + assert_eq!(body["duration_seconds"], 1.234_567_89); + assert_eq!(body["loop"], true); + assert_eq!(body["prompt_influence"], 0.3); + assert!(!String::from_utf8_lossy(&captured_request[header_end + 4..]).contains(TEST_API_KEY)); +} + +#[test] +fn octet_stream_and_missing_content_type_require_and_accept_real_mp3() { + for content_type in [Some("audio/mp3"), Some("application/octet-stream"), None] { + let (base_url, _captured_request, server) = + spawn_single_response_server(success_response(content_type, TEST_MP3)); + let settings = settings(base_url, 2_000); + let client = build_elevenlabs_audio_http_client(&settings) + .expect("ElevenLabs HTTP client should build"); + + let generated = runtime() + .block_on(generate_elevenlabs_sound_effect( + &client, + &settings, + request(), + )) + .expect("real MP3 should pass MIME fallback"); + assert_eq!(generated.audio.mime_type, "audio/mpeg"); + server.join().expect("mock server should finish"); + } +} + +#[test] +fn explicit_wrong_mime_empty_and_corrupt_bodies_are_rejected() { + let cases = [ + success_response(Some("text/html"), TEST_MP3), + success_response(Some("application/json"), br#"{"error":"no"}"#), + success_response(Some("audio/mpeg"), b""), + success_response(Some("audio/mpeg"), b"ID3-corrupt"), + ]; + for response in cases { + let (base_url, _captured_request, server) = spawn_single_response_server(response); + let settings = settings(base_url, 2_000); + let client = build_elevenlabs_audio_http_client(&settings) + .expect("ElevenLabs HTTP client should build"); + + let error = runtime() + .block_on(generate_elevenlabs_sound_effect( + &client, + &settings, + request(), + )) + .expect_err("invalid audio response should fail"); + assert_eq!(error.provider(), ELEVENLABS_PROVIDER); + server.join().expect("mock server should finish"); + } +} + +#[test] +fn redirects_http_failures_and_body_failures_do_not_retry_the_provider_post() { + let responses = [ + b"HTTP/1.1 307 Temporary Redirect\r\nLocation: /v1/sound-generation\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec(), + b"HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec(), + b"HTTP/1.1 422 Unprocessable Entity\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec(), + b"HTTP/1.1 429 Too Many Requests\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec(), + b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 29\r\nConnection: close\r\n\r\nprovider-secret-response-body".to_vec(), + b"HTTP/1.1 200 OK\r\nContent-Type: audio/mpeg\r\nContent-Length: 20\r\nConnection: close\r\n\r\nID3".to_vec(), + ]; + for response in responses { + let (base_url, request_count, server) = + spawn_counting_response_server(response, Duration::ZERO); + let settings = settings(base_url, 500); + let client = build_elevenlabs_audio_http_client(&settings) + .expect("ElevenLabs HTTP client should build"); + + let error = runtime() + .block_on(generate_elevenlabs_sound_effect( + &client, + &settings, + request(), + )) + .expect_err("provider failure should fail once"); + server.join().expect("mock server should finish"); + + assert_eq!(error.provider(), ELEVENLABS_PROVIDER); + assert_eq!(request_count.load(Ordering::SeqCst), 1); + assert!(!format!("{error:?}").contains(TEST_API_KEY)); + assert!(!format!("{error:?}").contains("provider-secret-response-body")); + } +} + +#[test] +fn timeout_does_not_retry_the_provider_post() { + let response = success_response(Some("audio/mpeg"), TEST_MP3); + let (base_url, request_count, server) = + spawn_counting_response_server(response, Duration::from_millis(100)); + let settings = settings(base_url, 20); + let client = + build_elevenlabs_audio_http_client(&settings).expect("ElevenLabs HTTP client should build"); + + let error = runtime() + .block_on(generate_elevenlabs_sound_effect( + &client, + &settings, + request(), + )) + .expect_err("provider timeout should fail once"); + server.join().expect("mock server should finish"); + + assert!(matches!(error, AudioError::Request { timeout: true, .. })); + assert_eq!(request_count.load(Ordering::SeqCst), 1); +} + +#[test] +fn content_length_precheck_accepts_the_limit_and_rejects_limit_plus_one() { + for (content_length, should_be_size_error) in [ + (MAX_GENERATED_AUDIO_BYTES, false), + (MAX_GENERATED_AUDIO_BYTES + 1, true), + ] { + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: audio/mpeg\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\nID3" + ) + .into_bytes(); + let (base_url, request_count, server) = + spawn_counting_response_server(response, Duration::ZERO); + let settings = settings(base_url, 500); + let client = build_elevenlabs_audio_http_client(&settings) + .expect("ElevenLabs HTTP client should build"); + + let error = runtime() + .block_on(generate_elevenlabs_sound_effect( + &client, + &settings, + request(), + )) + .expect_err("incomplete or oversized response should fail"); + server.join().expect("mock server should finish"); + + assert_eq!( + error.message().contains("超过 40 MiB"), + should_be_size_error + ); + assert_eq!(request_count.load(Ordering::SeqCst), 1); + } +} + +#[test] +fn chunked_limit_plus_one_is_rejected_without_unbounded_allocation() { + let listener = TcpListener::bind("127.0.0.1:0").expect("mock server should bind"); + let address = listener + .local_addr() + .expect("mock address should be readable"); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("mock request should connect"); + let _ = read_http_request(&mut stream); + stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: audio/mpeg\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n", + ) + .expect("chunked response headers should write"); + let chunk = vec![0_u8; 64 * 1024]; + for _ in 0..(MAX_GENERATED_AUDIO_BYTES / chunk.len()) { + if stream.write_all(b"10000\r\n").is_err() + || stream.write_all(&chunk).is_err() + || stream.write_all(b"\r\n").is_err() + { + return; + } + } + let _ = stream.write_all(b"1\r\nX\r\n0\r\n\r\n"); + }); + let settings = settings(format!("http://{address}"), 5_000); + let client = + build_elevenlabs_audio_http_client(&settings).expect("ElevenLabs HTTP client should build"); + + let error = runtime() + .block_on(generate_elevenlabs_sound_effect( + &client, + &settings, + request(), + )) + .expect_err("chunked limit plus one should fail"); + server.join().expect("mock server should finish"); + + assert!(error.message().contains("超过 40 MiB")); +} diff --git a/server-rs/crates/platform-audio/tests/fixtures/vbr-id3.mp3.base64 b/server-rs/crates/platform-audio/tests/fixtures/vbr-id3.mp3.base64 new file mode 100644 index 000000000..1ae2043e3 --- /dev/null +++ b/server-rs/crates/platform-audio/tests/fixtures/vbr-id3.mp3.base64 @@ -0,0 +1 @@ +SUQzAwAAAAAAI1RTU0UAAAAPAAAATGF2ZjYyLjEyLjEwMgAAAAAAAAAAAAAA//tAwAAAAAAAAAAAAAAAAAAAAAAAWGluZwAAAA8AAAARAAAOcAA/Pz8/P01NTU1NTVxcXFxcXGZmZmZmZnNzc3Nzc35+fn5+fomJiYmJiZWVlZWVlaCgoKCgra2tra2tt7e3t7e3xMTExMTEz8/Pz8/P2tra2tra5ubm5ubm+Pj4+Pj4//////8AAAAATGF2YzYyLjI4AAAAAAAAAAAAAAAAJAVYAAAAAAAADnCpcTduAAAAAAD/+8DEAAAImAFp9BAAAwKyLj8e8AFHVDRUNI8JJcQPjgQBAEw+PB8HwQBAMQcBA5ny4P8Hwff5cH34Ph/KHAQBD4E8MRO/EAIfwfD5QHwfPwICAIO4IHBICDoiIZlpnRmhlZpK222miiQTuLUSQXMySxthNxDy5EJWHMyATpcCkNBKFHz3H0sLBrt8MT49zFOpRuSA2vLS4LqXIlT+U1Yriqlf0eeTKqlzOfyJSTXEUi6YsNEXcKExJRUOb5qZWNjUbM9mezszUknzJM+jO38WRTR2ij+sTcLN2lkXVc7b3J+3U1M7tRt1u9Jrwpr/3ltBZ5r3izPYes7evPmr60au4T3f1M5eBSv977pmNeTFcfX+84y9hQ6ae5QKmLmXelQo2kiSAfH41AQBMeh6Vk4qKhIdkeiYJQNigfBwaQKOkP8jWVnqlcetPN8Fy6IYLCIyBTTSAPHi5dzIUpEbQ0NSo+dSNSYx56XbKFUDTRhZaKVBsQjUjUULP7ol+8Wl6iJZnU420SQAoDrAwFwsRpMEwAgBEvVC2TXoFUqObnjE1i51JNowm9gnGCMpcciIo1ybEu1LTUytCdsccakIN1/0/19y9C9yKqmomadWT/mCCAEAOAgKk4UTBdRclKBSLjj8oIGOLbhuJQ9pYs20+E/mqKJEQLiePFRyRVBdlq3JcmooJR+k2193VX6/08brnApnigskMAyTqayZeHU40kUQCwIxUHkQimRS0nBExbiiEYUJHECK6o0vBVITUHC0oy0y0gaILIRFFFJTZWyy+tk7YL0zv31//6fZ/oqmiqiIVF+qhA8UkHR4dDgJGRCHuMhNBM0iUWzZZSUpkCQQEpgqGhOKGxQXePwMQU1clHGQkaAK33rXATRZwlGChX+vfTboMFfIerf++8q4l3ZVcbSQBCAHAYBkRcRhAqGD0h1S0BAeUXUXtnRQVG29XteNRKzIQqSMHABx+9kbLv/7UMTgAAxIbXf8xAABJAgvfJSMMIYkkKXkr6V1vtiv//R9Xp9i+irKuJqHZFlcTQCGQW8NiA+Jj5suYE5VdMgPEqmtrBUOEhM4GRxxSgfAgQE7wMkGVBUeQD7IouegnnYlGKsz1P3/6P0/+xnh0v3VUxEK6kbiTJAA8lcflwVk0luBQwiiF3XUFrYJLyshdiEeUXRb/vlSspQpMi1gMja/qaWCSAB8jeqm6lDF3V/Vu0jlf/1p6buqmZh1WRxtIg0BslBMTsg6JSMRkzAZHiD/+1DE54AKAHF7xIhuAQ4GL7zEmUAXYIugs9spAThkIBEaFw0PCgZFFtYMAICFzY5kUKoGiRVy3tR1qfYfVtAKttfbdRW4l9//6Kmqe4ZjRxsogAfphJhoJQsBxUSgGWQl1yZXUu3raEdNmM4Sme1RlRMLXg2JoBItEiH1m4RY8XXF+S7iyqxb3/T/Tv/T3rqJd5lnVE6WAB5Ny+eDBaOhcWkF86OTFCEhFoaxhCCNtRtbLa0s2lSgivIjPv5iWL0Q1rrbzEyOLENxU93IsXo0//swxPsACZBDe8YkaMEXCy+8tIw497f9P6/6PTMRUSzKhxMJEEAEGIj4bBcLEhYRIxAkOUQcu5MtEA+LHRMJklAMHDXFA/9w1pI5eGXn+kc0N6NMalu2zut1O8ybtFWfzVWqmZmFVGsjSYBFpLqPIQXZZBciGjigoCacZs9ptXKq7IguFgUATgVEJ9oOsByUFm4NERZyVqDcx7XD//tAxPUACQwrf+elIAEalG98wwzYs2toUWkf/dv4v3/7FottmZeniGQ4200QQSDQ6KyckIjwJpoSdpAFETRZI8gIjFIHsDljlhpA8NoIkng6HCz5MPC6YdEaR5pYiHukdakwtexnLRjBn/p/vVUZH0vVqYl5iHZL5RQBQLVAEqEyMZaPMiia4IAhOZMhMNxXZ+kV/Y0vmtOHVHMnCFwCqS8Yi4M6Z6xmnrspIIdN9/JL972Ipb3/+zDE/gAJwDV/5KTEwRYMb3z0jDjfVcSzs6KhV1IJJ27CKaEUH1xUgo8VREZIHElgACDY0iA2GHtDaBg4+TOSL2kyUbJSTHCZnKOitlyciir/bZ+v8m8JIVQ0iqmqeZd1VxuNoF8P9YJAGG2h2ohlS5i8ox1tQNOBJiAMFSSgmAREVWljSYeFR4sIjkmDIoIhG4zNrtQOaVQaZLr/+zDE94AI7LF5xhhNARMGb3yTDOBgJT+FLKLmf/37v32VTxUwyqjiTKIJdIEApTDY22GwSIT6FYLGJEZIgi2bDWhYjgiEBwCDCoTi7z72MNCs0DxzeQ13t4Hs2LZSmh4sj2f2/6fbpbWrlruJdEbiaQJDgyBkDZOVMsDw0JYmDjhoQP5EjWiXQdrBwAUNJBQuQAZkItjmJLTaFlH/+0DE9IAJID9956RhwTUFL7yUpACuJrNpei1yms2AVFn+zj5ty0qnd9tN1NHTM1ETLKZuJJEkAobDJwKE4iAEPrgIhHgtIVrSiDgksDYdLhQVYSDjwiUGNNhMKjVk6ZIa9r1n0HwGTQ5yazyFpWfN/PjX9MoAq10oPo3We/jliYh3ZmNPqUUgPCoMEGqhGZgWRlhSwqj0TzGiHXoShDwlSLAwxw60TKJQCqiLWhFq+i/r5s6n6v/7MMT5gAi0a3vEmGxBFAXvePSImKv3Xyuv3bDL++dy7ayZh4dVKtJIkgoFMd67MM0WQDj6sB1DEeRHZvYZw8Q0X8UHjqHF2FgQLNHigeAwukKDgbXz6zBbmpVjhA5KOLdz0Rdh6N691N2EVGf3KquIaIiFSSRIpAIj8zAklLdiiLsMKTNHykSRMBxOTLUmFwSFxwZDI0aMAwWAJP/7QMT3gAmMN3/nsGGBHAavvMSYYAFRgFPtT028ym+3+ztVpbTpQXv016V1f/rmqe3hlNHGkkQRyXFUdRelQFhKVL26MFLz2+FYZVKsqPcqEMCIkIgsoRMSfdUk+JVHXGrn0zfeXX5azS5WXo2fb93VffjKmpqIqEU421EQR/Bm1gVeGxkjJjYIFkA0jibWMg+dPQ7n6WJfTylikedljEl4f3+dlK6hQ7Igs2xA56ABFC8k3cb0//swxP4ACXw9feSYaoFChS98lJgo7Gfuq132Kaz3VUy8vDIcaKSBQlDorFIwDIWswTmDLgquDk6lZ56QP1XdWgpQ5iFHhO6jjsWelDI9y0bUJxr8wP2+3R87eS/pfP1NK7iaS7Tdi5mGmEIo0USCQyoJUgdIAPIAWQQNeKypEJD1ppPaJCqRCGEBUKiQVSweaPvqi0UD41ZJEa66//tAxPMACGA/e8SYaoE0iW989Iyo2MxtiOG/0Ku9kWQ/ta86je918rbOMepp4mGY23CSCA9IgXIg0GBKLwEKEyWEvYDKBE02cIGBc8hwKEpA2LpLjyiWFwmXd3r3XVvzkjC9X/sr6/sW9P/Um5Z3d2VI3GiyBQCJCDoqMn1VyE8XH0MGlyU8crMZ4xKKt21R1imDWHRUFCxYBNBAXUgIlbiIazBZJlO1at6UIoJrBQ/EMpFmM/f/+zDE+wAIyDl/5jDAgRUJL7zBDcB/6MuriIdndkVJQrZIEyWCgAGsocXdIVYOUlAzROJY0ddlqDSXcalHp4DYegxBhpKHcPI7lwTSHNjZMAMEwsCRQeMDBraSIMdBlB848mjpM3W5JUpbWT3SQJwuKSS328lRxQ8dizlUmobfuvck9lTT3VRom64aqdLYdM7p3JWceg7TafdR5Sb/+zDE+IAJZKt95IxPgR4Ir7yTDYha8kybXz8t9vxG+e9abd+/3Ry6Lds+Y2//+lVMQU1FMy4xMDBVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVMQU1FMy4xMDBVVVVVVVVVVVVVVVVVVVVVVVX/+0DE8oAJeCt75JkoAQSFr7xkpABVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVf/7YMT8gAmAR330kwACfbKuvzCwCFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVf/7EMTWA8AAAf4cAAAgAAA0gAAABFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV