From d58e5cbebd6192fac239e5b1f8fa71cedec28ffd Mon Sep 17 00:00:00 2001 From: Linghong Date: Sat, 8 Aug 2026 09:52:18 +0000 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20inline=20=E6=B3=84?= =?UTF-8?q?=E9=9C=B2=20provider=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...架构】图片画布编辑器MVP接入方案-2026-06-11.md | 2 +- .../api-server/src/external_generation.rs | 31 ++- .../generation.rs | 190 +++++++++++++++--- 3 files changed, 197 insertions(+), 26 deletions(-) diff --git a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md index 74eb79be7..b06bc3a5a 100644 --- a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md +++ b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md @@ -25,7 +25,7 @@ - 编辑器自己生成的标准纯色背景抠图资产在保存源图后统一以 `background_mode=flat` 调用内部 `bgfilter-worker`。角色形象生成、图标 spritesheet 生成、UI 设计图素材提取和角色动作的用户路径都固定使用 `screenColor=auto`，但用户可见 `generationInputs.fields` 不记录 `抠图背景色` 或 `抠图模型`；api-server 在组装 prompt 前调用背景决策模块，从 12 个候选色中选择具体 hex，最多重试 3 次，失败后兜底 `#CFEFFF`。后端仍保留手动 hex 解析能力供内部兼容。最终生图 prompt、动作视频实色背景和子 worker 发往 provider 的 `screen_color` multipart 字段只接收解析后的具体 hex，不透传 `auto`。角色、图标 spritesheet 和 UI 设计图素材提取的同源画布请求由前端自动提交默认 `segModel=birefnet`，api-server 负责 allowlist 校验并在缺失时回落默认值；角色动作逐帧去背的 `seg_model` 由后端固定。四条 flat 路径再由 api-server 向 worker 显式传递 `background_mode=flat` 与 `cross_check`：角色形象生成、图标 spritesheet 和角色动作逐帧去背传 `on`，UI 设计图素材提取传 `off`，不依赖 BgFilter 服务端默认值。前端不展示抠图模型选择，`segModel` 不进入 `generationInputs`、响应、搜索、详情或导出；`background_mode`、`cross_check` 只存在于 api-server 到 worker 的内部 RPC。子 worker 为 flat / complex 分别维护独立进程级熔断，并对一次逻辑调用严格最多执行两次顺序 provider attempt；两种模式共享 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD=3` 和 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS=120` 默认值，但失败和成功只更新当前模式；父侧至多让 worker 接收一次内部 RPC（连接从未建立时按调度方案 §5.1 有界重连）。flat 两次失败、熔断、overload、内部 deadline 或断连后，只要父业务预算仍有效，父流程才按同一 object key 进入“阿里云通用抠图 → 本地 `editor_green_screen` 键色”降级；阿里云 fallback 不属于 `bgfilter-worker`。角色动作生成的序列帧背景色已与生图统一：后端把源角色图合成到视觉决策出的具体 hex 后再图生视频；抽帧后逐帧进入同一条 `内部 bgfilter-worker（background_mode=flat，cross_check=on）→ 父侧阿里云 → 父侧本地键色` 链路。 - 请求与可见性边界：同源画布 BFF 的角色、图标 spritesheet 和 UI 设计图素材提取 request DTO 保留 `segModel`，前端固定自动填充 `birefnet`，该字段不是用户选择或用户可见生成输入。External OpenAPI 是否接受 `segModel` 是独立契约，不能由站内 BFF 自动外推。`background_mode`、`cross_check` 以及角色动作的 `seg_model` 只由 api-server 决定并通过 loopback RPC 传给 worker。 - 普通用户生成结果不携带生成 provider：图片、图标图集、视频、音频和角色动画完成响应的公开类型均无 `provider`，前端也不得把 provider 写入新建结果图层；项目资源、素材和历史画布快照中的 provider 由后端 User/Public mapper 统一省略。真实 provider 仅留在服务端持久化、tracking / tracing 与后台原始审计。历史派生画布资源的内部处理模型被 User mapper 清空后，图层 hydration 必须继续沿 `sourceResourceId` 回溯第一个正常生成模型，供图片信息和画布 ZIP 展示；来源链缺失或没有正常模型时显示 `-`。 -- 手动去背景与角色动作透明化失败时，前端只消费后端按任务类型返回的稳定业务文案；不得从队列 `lastErrorMessage`、HTTP error details 或图层状态恢复和展示 BgFilter、分割模型、provider 等内部诊断。 +- 手动去背景、角色动作透明化、音效和背景音乐生成失败时，前端只消费后端按任务类型返回的稳定业务文案；不得从队列 `lastErrorMessage`、HTTP error details 或图层状态恢复和展示 BgFilter、分割模型、provider、请求端点、传输错误或上游状态等内部诊断。 - BgFilter 单次真实 provider attempt 不再使用独立固定 timeout，而由父子共同按 `attempt = N × est × 2` 运行时派生；一次逻辑调用的 `callBudgetMs = 2 × attempt + 1s`，从子 worker 取得 provider permit 后才开始计时。当前冻结 `N=16 / est=5000ms` 时为 `160s / 321s`。父侧继续管理父 job / request 总预算，为每个内部 RPC 单独派生 `maxQueueWaitMs`；排队只消耗该字段，不侵蚀 `callBudgetMs`，flat 还需预留阿里云和本地键色 fallback 时间。角色动作不再按本次实际帧数增加 attempt，`32 / 40 / 48` 帧使用同一公式。角色动画继续用 `buffer_unordered(frame_count.max(1))` 同时提交单帧逻辑调用，由唯一子 worker 保证健康进程内实际在飞的 provider 请求不超过 `N`、admission 不超过默认保险丝 `Q=2048`；父流程仍按“对应绿幕源图上传 OSS 并释放原帧字节 → 以 object key 调内部 worker / 按 object key 降级 → 父侧完成透明帧处理并落 OSS”连续组成无序在途流水线，允许响应乱序，并在收口时 collect / drain 全部已提交 frame future、按 `frameIndex` 恢复顺序。任一帧最终失败时仍先排空全部已启动请求，再使整个动作任务失败退款，不发布缺帧动画；最终图片处理、OSS、画布写回和计费始终属于父流程。 - 多产物生成以后端项目快照为唯一画布真相：同一任务实际产生的原始产物、抠图 / 透明化结果和拆分结果都要先登记为 `editor_project_resource`，再通过一次 `canvasCompletion` 原子写入画布。角色形象、图标 spritesheet 和 UI 素材提取的纯色背景原图不能只留在 OSS。透明后处理成功时，处理结果保持主图层和 `generatedLayerId` 锚点，三类任务同时把 provider 原图作为第二个图层放在透明主结果右侧，图标和 UI 的实际拆分素材从 provider 原图右侧开始放置。透明背景处理最终失败时，只把已保存的原图作为唯一主图完成占位，不放透明处理图，图标和 UI 不继续拆分。source-only fallback 的前端只消费后端返回的 `project` / `resource` 快照，不按缺失字段自行构造透明图、切片或图层；任务以 `completed + warning` 收口。该收口只捕获透明背景处理本身的最终失败；phase 上报、provider 原图持久化、透明处理图持久化和 `canvasCompletion` 写回错误仍正常传播，不能被原图降级吞掉。通用 `warning.reason` 是可直接展示的完整原因。它与 `sliceWarning` 只在 `postprocess-failed-source-preserved` 这一条上互斥（透明背景最终失败不会进入拆分）；风格归一化和像素规整产生的通用 `warning` 可与 `sliceWarning` 并存，此时 inline 与队列两条链路都必须按“通用在前、拆分在后”拼成同一条提示展示，不得只取其一。既有 `sliceWarning.reason` 只表示透明图成功后的自动拆分失败，保留后端原始诊断，inline 前端仅在展示时补充“图集已生成，但自动拆分未完成：”提示，queue worker 则把归一后的字符串交给 BFF `warning` 由前端直接展示。无项目上下文时不创建项目资源或画布图层。 - 图片快速编辑面板只保留一个提示词输入框和模型选择，不展示额外参考图或比例 / 尺寸控件；原图 / 原素材作为 `/api/editor/images/edits` 的 `sourceImageSrc` 直接提交，不作为 `referenceImageSrcs`。完整图标图集 `icon-spritesheet` 支持快速编辑，拆分后的单个 `icon` 不提供该入口，前后端必须使用同一素材类型规则。打开快速编辑时画布必须自动平移缩放，让原素材完整落在可视区上半部分，底部面板固定出现在素材下方且不遮挡内容，竖屏 UI 素材也必须完整展示。快速编辑右侧显示矩形、椭圆、画笔框选工具，但进入时不默认启用；点击工具后显示选中态，再点同一工具取消启用。完成框选后，画布红色细框显示连续序号，提示词可按这些编号填写每个区域怎么改。点击 `修改` 后仍停留在当前快速编辑面板显示修改中，不创建独立 `Quick Edit Generator` 画布占位；生成成功后直接用结果覆盖原图图层，失败时保留当前面板并在错误红框中显示具体错误文案。 diff --git a/server-rs/crates/api-server/src/external_generation.rs b/server-rs/crates/api-server/src/external_generation.rs index 679e06c5a..776201f13 100644 --- a/server-rs/crates/api-server/src/external_generation.rs +++ b/server-rs/crates/api-server/src/external_generation.rs @@ -20,8 +20,8 @@ use spacetime_client::{ }; use crate::editor_generation_queue::{ - EDITOR_BACKGROUND_REMOVAL_JOB_KIND, EDITOR_CHARACTER_ANIMATION_GENERATION_JOB_KIND, - EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND, + EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, EDITOR_BACKGROUND_REMOVAL_JOB_KIND, + EDITOR_CHARACTER_ANIMATION_GENERATION_JOB_KIND, EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND, }; use crate::{ api_response::json_success_body, auth::AuthenticatedAccessToken, http_error::AppError, @@ -218,6 +218,9 @@ fn user_visible_external_generation_error(job_kind: &str, error: Option) if job_kind == EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND && error.is_some() { return Some("音效生成失败，请稍后重试。".to_string()); } + if job_kind == EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND && error.is_some() { + return Some("背景音乐生成失败，请稍后重试。".to_string()); + } error } @@ -519,6 +522,30 @@ mod tests { } } + #[test] + fn background_music_failure_hides_provider_and_transport_details_from_owner() { + let message = user_visible_external_generation_error( + EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, + Some( + "提交 Vector Engine 背景音乐任务失败：error sending request for url \ + (https://vector-engine.internal/v1/audio/generations): connection error: \ + Connection refused (os error 111)" + .to_string(), + ), + ); + + assert_eq!(message.as_deref(), Some("背景音乐生成失败，请稍后重试。")); + let lower = message + .expect("失败任务应返回稳定文案") + .to_ascii_lowercase(); + for forbidden in ["vector engine", "vector-engine", "https://", "os error"] { + assert!( + !lower.contains(forbidden), + "普通用户文案不应包含 {forbidden}" + ); + } + } + #[test] fn unrelated_generation_failure_keeps_its_user_visible_message() { let message = user_visible_external_generation_error( diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs index 873c6de88..f394044ce 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs @@ -314,6 +314,14 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( ) -> Result, Response> { let Json(mut payload) = parse_json_payload(&request_context, payload)?; let owner_user_id = caller.owner_user_id.clone(); + let error_response = |error| { + editor_audio_generation_error_response( + &request_context, + &caller, + "音效生成失败，请稍后重试。", + error, + ) + }; let target = preflight_editor_billable_generation_target( &state, owner_user_id.as_str(), @@ -321,7 +329,7 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( payload.asset_folder_id.clone(), ) .await - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; payload.project_id = target.project_id; payload.asset_folder_id = target.asset_folder_id; let project_id = payload.project_id.clone(); @@ -337,16 +345,15 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( "message": error.to_string(), })) }) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; let normalized = normalize_editor_sound_effect_request_with_pricing(payload.clone(), &pricing) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; let request_deadline = request_context.external_call_deadline(); - let mut settings = require_elevenlabs_audio_settings(&state) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + let mut settings = require_elevenlabs_audio_settings(&state).map_err(&error_response)?; settings.request_deadline = request_deadline; let http_client = platform_audio::build_elevenlabs_audio_http_client(&settings) .map_err(map_platform_audio_error) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; let llm_client = state .editor_agent_llm_client() .ok_or_else(|| { @@ -356,18 +363,17 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( "message": "音效 Prompt 英文化服务未配置", })) }) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; let target = build_editor_audio_target( require_editor_audio_operation(&caller) - .map_err(|error| error.into_response_with_context(Some(&request_context)))? + .map_err(&error_response)? .operation_id .as_str(), "sound_effect", "editor_sound_effect", Some(u64::from(normalized.price_mud_points)), ); - let operation = require_editor_audio_operation(&caller) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + let operation = require_editor_audio_operation(&caller).map_err(&error_response)?; let stages = ProductionSoundEffectWorkerStages { state: &state, caller: &caller, @@ -406,7 +412,7 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( }, ) .await - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; Ok(json_success_body(Some(&request_context), response)) } @@ -981,6 +987,14 @@ pub(crate) async fn generate_editor_background_music_for_owner( ) -> Result, Response> { let Json(mut payload) = parse_json_payload(&request_context, payload)?; let owner_user_id = caller.owner_user_id.clone(); + let error_response = |error| { + editor_audio_generation_error_response( + &request_context, + &caller, + "背景音乐生成失败，请稍后重试。", + error, + ) + }; let target = preflight_editor_billable_generation_target( &state, owner_user_id.as_str(), @@ -988,7 +1002,7 @@ pub(crate) async fn generate_editor_background_music_for_owner( payload.asset_folder_id.clone(), ) .await - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; payload.project_id = target.project_id; payload.asset_folder_id = target.asset_folder_id; let project_id = payload.project_id.clone(); @@ -1005,14 +1019,13 @@ pub(crate) async fn generate_editor_background_music_for_owner( "message": error.to_string(), })) }) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; let normalized = normalize_editor_background_music_request_with_pricing(payload, &pricing) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; - let settings = require_vector_engine_audio_settings(&state) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; + let settings = require_vector_engine_audio_settings(&state).map_err(&error_response)?; let http_client = platform_audio::build_vector_engine_audio_http_client(&settings) .map_err(map_platform_audio_error) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; let task = platform_audio::submit_editor_background_music_task( &http_client, &settings, @@ -1024,10 +1037,10 @@ pub(crate) async fn generate_editor_background_music_for_owner( ) .await .map_err(map_platform_audio_error) - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; let target = build_editor_audio_target( require_editor_audio_operation(&caller) - .map_err(|error| error.into_response_with_context(Some(&request_context)))? + .map_err(&error_response)? .operation_id .as_str(), "background_music", @@ -1045,7 +1058,7 @@ pub(crate) async fn generate_editor_background_music_for_owner( target, ) .await - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; let audio_src = generated.audio_src.clone(); let persisted = persist_editor_audio_generation( &state, @@ -1066,7 +1079,7 @@ pub(crate) async fn generate_editor_background_music_for_owner( }, ) .await - .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + .map_err(&error_response)?; Ok(json_success_body( Some(&request_context), @@ -1158,6 +1171,29 @@ fn build_inline_editor_audio_caller( }) } +/// 编辑器音频的 inline 兼容模式会把生成结果直接回给普通用户；provider / 传输诊断只能 +/// 留在 worker 路径和服务端审计中，不能随这个同步 HTTP 响应泄露出去。 +/// +/// 队列 worker 带有 `worker_id`，仍保留原始 `AppError` 响应以便现有 worker 抽取失败文本、 +/// 写入任务记录并由任务读取边界统一脱敏。4xx 也保持原状，避免丢失用户可操作的校验提示。 +fn editor_audio_generation_error_response( + request_context: &RequestContext, + caller: &EditorGenerationCaller, + user_message: &'static str, + error: AppError, +) -> Response { + let is_inline_user_request = caller + .operation + .as_ref() + .is_some_and(|operation| operation.worker_id.is_none()); + if is_inline_user_request && error.status_code().is_server_error() { + return AppError::from_status(error.status_code()) + .with_message(user_message) + .into_response_with_context(Some(request_context)); + } + error.into_response_with_context(Some(request_context)) +} + fn require_editor_audio_operation( caller: &EditorGenerationCaller, ) -> Result<&EditorGenerationOperationContext, AppError> { @@ -1581,6 +1617,7 @@ fn editor_audio_bad_request(message: impl Into) -> AppError { #[cfg(test)] mod tests { + use axum::{body::to_bytes, http::StatusCode}; use serde::Deserialize; use serde_json::{Value, json}; use shared_contracts::assets; @@ -1590,8 +1627,8 @@ mod tests { CanonicalEditorBackgroundMusicSubmissionPayload, NormalizedEditorBackgroundMusicRequest, NormalizedEditorSoundEffectRequest, build_editor_background_music_generate_response, build_inline_editor_audio_caller, build_sound_effect_job_result_payload, - normalize_editor_background_music_request, normalize_editor_sound_effect_request, - prepare_editor_background_music_queue_job, + editor_audio_generation_error_response, normalize_editor_background_music_request, + normalize_editor_sound_effect_request, prepare_editor_background_music_queue_job, }; use crate::{ editor_generation_queue::{ @@ -1599,6 +1636,7 @@ mod tests { serialize_editor_generation_job_payload_for_test, }, editor_project::{EditorGenerationCaller, EditorGenerationOperationContext}, + http_error::AppError, request_context::RequestContext, }; @@ -1690,6 +1728,112 @@ mod tests { assert!(operation.lease_token.is_none()); } + #[tokio::test] + async fn inline_audio_server_error_hides_provider_diagnostics_in_both_response_shapes() { + for wants_envelope in [false, true] { + let request_context = RequestContext::new( + format!("request-audio-error-{wants_envelope}"), + "POST /api/editor/audios/sound-effects/generations".to_string(), + std::time::Duration::ZERO, + wants_envelope, + ); + let payload = sound_effect_payload("机械齿轮转动", None, None, false); + let caller = build_inline_editor_audio_caller( + "user-1".to_string(), + &request_context, + EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND, + &payload, + ) + .expect("inline caller should be prepared"); + let response = editor_audio_generation_error_response( + &request_context, + &caller, + "音效生成失败，请稍后重试。", + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": "elevenlabs", + "message": "请求 ElevenLabs 音效生成失败：connection refused", + "endpoint": "https://api.elevenlabs.test/v1/sound-generation", + "source": "tcp connect error", + "status": 502, + })), + ); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + let bytes = to_bytes(response.into_body(), 64 * 1024) + .await + .expect("error response body should be readable"); + let response_payload: Value = + serde_json::from_slice(&bytes).expect("error response should be JSON"); + + assert_eq!( + response_payload["error"]["message"], + json!("音效生成失败，请稍后重试。") + ); + assert!(response_payload["error"].get("details").is_none()); + let serialized = response_payload.to_string().to_ascii_lowercase(); + for forbidden in [ + "elevenlabs", + "api.elevenlabs.test", + "sound-generation", + "tcp connect error", + "\"endpoint\"", + "\"source\"", + ] { + assert!( + !serialized.contains(forbidden), + "inline 用户响应不应包含 {forbidden}" + ); + } + } + } + + #[tokio::test] + async fn queued_audio_server_error_keeps_diagnostics_for_worker_processing() { + let request_context = RequestContext::new( + "request-audio-worker-error".to_string(), + "external-generation-worker".to_string(), + std::time::Duration::ZERO, + true, + ); + let caller = EditorGenerationCaller { + owner_user_id: "user-1".to_string(), + audit_subject_user_id: Some("user-1".to_string()), + audit_project_id: None, + phase_reporter: None, + operation: Some(EditorGenerationOperationContext { + operation_kind: EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND.to_string(), + operation_id: "task-audio-worker-error".to_string(), + operation_fingerprint: "fingerprint-audio-worker-error".to_string(), + worker_id: Some("worker-audio-1".to_string()), + lease_token: Some("lease-audio-1".to_string()), + queue_result_context: None, + }), + }; + let response = editor_audio_generation_error_response( + &request_context, + &caller, + "音效生成失败，请稍后重试。", + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": "elevenlabs", + "endpoint": "https://api.elevenlabs.test/v1/sound-generation", + "source": "tcp connect error", + })), + ); + let bytes = to_bytes(response.into_body(), 64 * 1024) + .await + .expect("worker error response body should be readable"); + let response_payload: Value = + serde_json::from_slice(&bytes).expect("worker error response should be JSON"); + + assert_eq!( + response_payload["error"]["details"]["provider"], + json!("elevenlabs") + ); + assert_eq!( + response_payload["error"]["details"]["endpoint"], + json!("https://api.elevenlabs.test/v1/sound-generation") + ); + } + #[test] fn audio_queue_result_keeps_required_editor_agent_response_fields() { let source = include_str!("generation.rs");