From d0412705c45af081c57fa9f78fe937361f562960 Mon Sep 17 00:00:00 2001 From: Linghong Date: Wed, 8 Jul 2026 03:50:32 +0000 Subject: [PATCH 01/41] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=98=BF=E9=87=8C?= =?UTF-8?q?=E4=BA=91=E9=80=9A=E7=94=A8=E6=8A=A0=E5=9B=BE=E9=93=BE=E8=B7=AF?= =?UTF-8?q?=EF=BC=8C=E5=9B=BE=E7=89=87=E6=8A=A0=E5=9B=BE=E5=8D=87=E7=BA=A7?= =?UTF-8?q?=E4=B8=89=E6=A1=A3=E9=99=8D=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 platform-matting crate：手搓 SegmentCommonImage ACS3 签名调用、 VIAPI 临时桶上传、分辨率守卫（缩图送抠 + alpha 回贴原图） - api-server 接入 GENARRATIVE_ALIYUN_MATTING_* 配置与 MattingClient - 画布生成图片抠图链改为 BgFilter → 阿里云 → 本地键色三档降级 Co-Authored-By: Claude Fable 5 --- server-rs/Cargo.lock | 24 + server-rs/Cargo.toml | 2 + server-rs/crates/api-server/Cargo.toml | 1 + .../crates/api-server/src/aliyun_matting.rs | 124 +++ server-rs/crates/api-server/src/config.rs | 88 +++ .../crates/api-server/src/editor_project.rs | 22 +- server-rs/crates/api-server/src/main.rs | 1 + server-rs/crates/api-server/src/state.rs | 40 + server-rs/crates/platform-matting/Cargo.toml | 26 + .../examples/crop_size_probe.rs | 104 +++ .../examples/segment_smoke.rs | 158 ++++ server-rs/crates/platform-matting/src/lib.rs | 724 ++++++++++++++++++ 12 files changed, 1312 insertions(+), 2 deletions(-) create mode 100644 server-rs/crates/api-server/src/aliyun_matting.rs create mode 100644 server-rs/crates/platform-matting/Cargo.toml create mode 100644 server-rs/crates/platform-matting/examples/crop_size_probe.rs create mode 100644 server-rs/crates/platform-matting/examples/segment_smoke.rs create mode 100644 server-rs/crates/platform-matting/src/lib.rs diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index 181cd0ac8..ae19afaf0 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -241,6 +241,7 @@ dependencies = [ "platform-hyper3d", "platform-image", "platform-llm", + "platform-matting", "platform-oss", "platform-speech", "platform-wechat", @@ -4476,6 +4477,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "platform-matting" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "dotenvy", + "hex", + "hmac", + "httpdate", + "image", + "platform-oss", + "reqwest 0.12.28", + "serde", + "serde_json", + "serde_urlencoded", + "sha1", + "sha2", + "time", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "platform-oss" version = "0.1.0" diff --git a/server-rs/Cargo.toml b/server-rs/Cargo.toml index c7681f2fb..64ce7ba85 100644 --- a/server-rs/Cargo.toml +++ b/server-rs/Cargo.toml @@ -38,6 +38,7 @@ members = [ "crates/platform-hyper3d", "crates/platform-image", "crates/platform-llm", + "crates/platform-matting", "crates/platform-wechat", "crates/platform-speech", "crates/platform-agent", @@ -88,6 +89,7 @@ platform-audio = { path = "crates/platform-audio", default-features = false } platform-hyper3d = { path = "crates/platform-hyper3d", default-features = false } platform-image = { path = "crates/platform-image", default-features = false } platform-llm = { path = "crates/platform-llm", default-features = false } +platform-matting = { path = "crates/platform-matting", default-features = false } platform-oss = { path = "crates/platform-oss", default-features = false } platform-speech = { path = "crates/platform-speech", default-features = false } platform-wechat = { path = "crates/platform-wechat", default-features = false } diff --git a/server-rs/crates/api-server/Cargo.toml b/server-rs/crates/api-server/Cargo.toml index a38b50da4..8949fdca5 100644 --- a/server-rs/crates/api-server/Cargo.toml +++ b/server-rs/crates/api-server/Cargo.toml @@ -43,6 +43,7 @@ platform-auth = { workspace = true } platform-hyper3d = { workspace = true } platform-image = { workspace = true } platform-llm = { workspace = true } +platform-matting = { workspace = true } platform-oss = { workspace = true } platform-speech = { workspace = true } platform-wechat = { workspace = true } diff --git a/server-rs/crates/api-server/src/aliyun_matting.rs b/server-rs/crates/api-server/src/aliyun_matting.rs new file mode 100644 index 000000000..79fc2e986 --- /dev/null +++ b/server-rs/crates/api-server/src/aliyun_matting.rs @@ -0,0 +1,124 @@ +//! 阿里云通用抠图在 api-server 侧的适配层。 +//! +//! 输入 URL 策略：自有 OSS 在上海地域时直接上传草稿区并签名读 URL（抠图服务只认 +//! 上海地域 OSS）；其他地域交给 platform-matting 走 VIAPI 官方临时桶。 + +use std::collections::BTreeMap; + +use axum::http::StatusCode; +use platform_oss::{ + LegacyAssetPrefix, OssObjectAccess, OssPutObjectRequest, OssSignedGetObjectUrlRequest, +}; +use serde_json::json; + +use crate::{ + http_error::AppError, openai_image_generation::DownloadedOpenAiImage, state::AppState, +}; + +const MATTING_INPUT_PATH_SEGMENT: &str = "matting-input"; +const MATTING_INPUT_READ_EXPIRE_SECONDS: u64 = 10 * 60; +const ALIYUN_MATTING_OSS_REGION_MARKER: &str = "oss-cn-shanghai"; + +/// 图片字节 → 阿里云通用抠图 → 原尺寸透明 PNG。 +/// 未配置抠图客户端时返回错误，由调用方决定是否降级本地算法。 +pub(crate) async fn segment_image_with_aliyun_matting( + state: &AppState, + image: &DownloadedOpenAiImage, + log_label: &str, +) -> Result { + let matting_client = state.matting_client().ok_or_else(|| { + AppError::from_status(StatusCode::SERVICE_UNAVAILABLE).with_details(json!({ + "provider": "aliyun-matting", + "message": "阿里云抠图客户端未配置或未启用。", + })) + })?; + + let accessible_url = prepare_shanghai_oss_input_url(state, image, log_label).await; + let file_name = format!("{log_label}.png"); + let started_at = std::time::Instant::now(); + let output_bytes = matting_client + .segment_image_to_transparent_png(image.bytes.as_slice(), &file_name, accessible_url) + .await + .map_err(|error| { + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": "aliyun-matting", + "message": error.message(), + })) + })?; + tracing::info!( + provider = "aliyun-matting", + log_label, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "阿里云通用抠图完成" + ); + + Ok(DownloadedOpenAiImage { + bytes: output_bytes, + mime_type: "image/png".to_string(), + extension: "png".to_string(), + }) +} + +/// 自有 OSS 在上海地域时，把待抠图图片传到草稿区并返回签名读 URL； +/// 其他情况返回 None（platform-matting 会转走官方临时桶）。 +/// 该步骤失败不阻断抠图，只是退化为临时桶通道。 +async fn prepare_shanghai_oss_input_url( + state: &AppState, + image: &DownloadedOpenAiImage, + log_label: &str, +) -> Option { + let oss_client = state.oss_client()?; + let endpoint = state.config.oss_endpoint.as_deref().unwrap_or(""); + if !endpoint.contains(ALIYUN_MATTING_OSS_REGION_MARKER) { + return None; + } + + let http_client = reqwest::Client::new(); + let file_name = format!( + "{log_label}-{}.{}", + shared_kernel::new_uuid_simple_string(), + image.extension + ); + let put_result = oss_client + .put_object( + &http_client, + OssPutObjectRequest { + prefix: LegacyAssetPrefix::CharacterDrafts, + path_segments: vec![MATTING_INPUT_PATH_SEGMENT.to_string()], + file_name, + content_type: Some(image.mime_type.clone()), + access: OssObjectAccess::Private, + metadata: BTreeMap::new(), + body: image.bytes.clone(), + }, + ) + .await; + let put_result = match put_result { + Ok(result) => result, + Err(error) => { + tracing::warn!( + provider = "aliyun-matting", + log_label, + error = %error, + "抠图输入图上传自有 OSS 失败，回退官方临时桶" + ); + return None; + } + }; + + match oss_client.sign_get_object_url(OssSignedGetObjectUrlRequest { + object_key: put_result.object_key, + expire_seconds: Some(MATTING_INPUT_READ_EXPIRE_SECONDS), + }) { + Ok(signed) => Some(signed.signed_url), + Err(error) => { + tracing::warn!( + provider = "aliyun-matting", + log_label, + error = %error, + "抠图输入图签名读 URL 失败，回退官方临时桶" + ); + None + } + } +} diff --git a/server-rs/crates/api-server/src/config.rs b/server-rs/crates/api-server/src/config.rs index 8a93c51d7..e3881e7a7 100644 --- a/server-rs/crates/api-server/src/config.rs +++ b/server-rs/crates/api-server/src/config.rs @@ -18,6 +18,8 @@ const DEFAULT_EDITOR_BACKGROUND_REMOVAL_BASE_URL: &str = "http://58.87.105.82"; const DEFAULT_EDITOR_BACKGROUND_REMOVAL_REQUEST_TIMEOUT_MS: u64 = 120_000; const DEFAULT_EDITOR_BGFILTER_BASE_URL: &str = "http://58.87.105.82/bgfilter"; const DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS: u64 = 120_000; +const DEFAULT_ALIYUN_MATTING_ENDPOINT: &str = "imageseg.cn-shanghai.aliyuncs.com"; +const DEFAULT_ALIYUN_MATTING_REQUEST_TIMEOUT_MS: u64 = 30_000; // 集中管理 api-server 的启动配置，避免入口层直接散落环境变量解析逻辑。 #[derive(Clone, Debug)] @@ -61,6 +63,11 @@ pub struct AppConfig { pub editor_bgfilter_base_url: String, pub editor_bgfilter_token: Option, pub editor_bgfilter_request_timeout_ms: u64, + pub aliyun_matting_enabled: bool, + pub aliyun_matting_endpoint: String, + pub aliyun_matting_access_key_id: Option, + pub aliyun_matting_access_key_secret: Option, + pub aliyun_matting_request_timeout_ms: u64, pub image_editor_agent_sidebar_enabled: bool, pub log_filter: String, pub otel_enabled: bool, @@ -292,6 +299,11 @@ impl Default for AppConfig { editor_bgfilter_base_url: DEFAULT_EDITOR_BGFILTER_BASE_URL.to_string(), editor_bgfilter_token: None, editor_bgfilter_request_timeout_ms: DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS, + aliyun_matting_enabled: true, + aliyun_matting_endpoint: DEFAULT_ALIYUN_MATTING_ENDPOINT.to_string(), + aliyun_matting_access_key_id: None, + aliyun_matting_access_key_secret: None, + aliyun_matting_request_timeout_ms: DEFAULT_ALIYUN_MATTING_REQUEST_TIMEOUT_MS, image_editor_agent_sidebar_enabled: false, log_filter: "info,tower_http=info".to_string(), otel_enabled: false, @@ -496,6 +508,25 @@ impl AppConfig { { config.editor_bgfilter_request_timeout_ms = timeout_ms; } + if let Some(enabled) = read_first_bool_env(&["GENARRATIVE_ALIYUN_MATTING_ENABLED"]) { + config.aliyun_matting_enabled = enabled; + } + if let Some(endpoint) = read_first_non_empty_env(&["GENARRATIVE_ALIYUN_MATTING_ENDPOINT"]) { + config.aliyun_matting_endpoint = endpoint; + } + config.aliyun_matting_access_key_id = read_first_non_empty_env(&[ + "GENARRATIVE_ALIYUN_MATTING_ACCESS_KEY_ID", + "ALIBABA_CLOUD_ACCESS_KEY_ID", + ]); + config.aliyun_matting_access_key_secret = read_first_non_empty_env(&[ + "GENARRATIVE_ALIYUN_MATTING_ACCESS_KEY_SECRET", + "ALIBABA_CLOUD_ACCESS_KEY_SECRET", + ]); + if let Some(timeout_ms) = + read_first_positive_u64_env(&["GENARRATIVE_ALIYUN_MATTING_REQUEST_TIMEOUT_MS"]) + { + config.aliyun_matting_request_timeout_ms = timeout_ms; + } if let Some(enabled) = read_first_bool_env(&["GENARRATIVE_ENABLE_IMAGE_EDITOR_AGENT_SIDEBAR"]) { @@ -2263,6 +2294,63 @@ mod tests { } } + #[test] + fn from_env_reads_aliyun_matting_settings_and_reuses_alibaba_cloud_keys() { + let _guard = ENV_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("env lock should not poison"); + + unsafe { + std::env::remove_var("GENARRATIVE_ALIYUN_MATTING_ENABLED"); + std::env::remove_var("GENARRATIVE_ALIYUN_MATTING_ENDPOINT"); + std::env::remove_var("GENARRATIVE_ALIYUN_MATTING_ACCESS_KEY_ID"); + std::env::remove_var("GENARRATIVE_ALIYUN_MATTING_ACCESS_KEY_SECRET"); + std::env::remove_var("GENARRATIVE_ALIYUN_MATTING_REQUEST_TIMEOUT_MS"); + std::env::remove_var("ALIBABA_CLOUD_ACCESS_KEY_ID"); + std::env::remove_var("ALIBABA_CLOUD_ACCESS_KEY_SECRET"); + std::env::set_var("ALIBABA_CLOUD_ACCESS_KEY_ID", "shared-ak-id"); + std::env::set_var("ALIBABA_CLOUD_ACCESS_KEY_SECRET", "shared-ak-secret"); + std::env::set_var("GENARRATIVE_ALIYUN_MATTING_REQUEST_TIMEOUT_MS", "45000"); + } + + let config = AppConfig::from_env(); + assert!(config.aliyun_matting_enabled); + assert_eq!( + config.aliyun_matting_endpoint, + "imageseg.cn-shanghai.aliyuncs.com" + ); + assert_eq!( + config.aliyun_matting_access_key_id.as_deref(), + Some("shared-ak-id") + ); + assert_eq!( + config.aliyun_matting_access_key_secret.as_deref(), + Some("shared-ak-secret") + ); + assert_eq!(config.aliyun_matting_request_timeout_ms, 45_000); + + unsafe { + std::env::set_var("GENARRATIVE_ALIYUN_MATTING_ACCESS_KEY_ID", "matting-ak-id"); + std::env::set_var("GENARRATIVE_ALIYUN_MATTING_ENABLED", "false"); + } + + let config = AppConfig::from_env(); + assert!(!config.aliyun_matting_enabled); + assert_eq!( + config.aliyun_matting_access_key_id.as_deref(), + Some("matting-ak-id") + ); + + unsafe { + std::env::remove_var("GENARRATIVE_ALIYUN_MATTING_ENABLED"); + std::env::remove_var("GENARRATIVE_ALIYUN_MATTING_ACCESS_KEY_ID"); + std::env::remove_var("GENARRATIVE_ALIYUN_MATTING_REQUEST_TIMEOUT_MS"); + std::env::remove_var("ALIBABA_CLOUD_ACCESS_KEY_ID"); + std::env::remove_var("ALIBABA_CLOUD_ACCESS_KEY_SECRET"); + } + } + #[test] fn from_env_reads_image_editor_agent_sidebar_runtime_flag() { let _guard = ENV_LOCK diff --git a/server-rs/crates/api-server/src/editor_project.rs b/server-rs/crates/api-server/src/editor_project.rs index 7f080430d..1e7d1dd66 100644 --- a/server-rs/crates/api-server/src/editor_project.rs +++ b/server-rs/crates/api-server/src/editor_project.rs @@ -2221,9 +2221,27 @@ async fn remove_editor_generated_screen_background_with_bgfilter( seg_model, error = %error, error_details = ?error.details(), - "editor_bgfilter_fallback_to_local_screen_background_removal" + "editor_bgfilter_fallback_to_aliyun_matting" ); - remove_editor_generated_green_screen_background(image, screen_color) + match crate::aliyun_matting::segment_image_with_aliyun_matting( + state, + image, + "editor-screen-background", + ) + .await + { + Ok(image) => Ok(image), + Err(error) => { + tracing::warn!( + provider = "aliyun-matting", + screen_color = screen_color.hex, + error = %error, + error_details = ?error.details(), + "editor_aliyun_matting_fallback_to_local_screen_background_removal" + ); + remove_editor_generated_green_screen_background(image, screen_color) + } + } } } } diff --git a/server-rs/crates/api-server/src/main.rs b/server-rs/crates/api-server/src/main.rs index b64b8bf79..06b145b50 100644 --- a/server-rs/crates/api-server/src/main.rs +++ b/server-rs/crates/api-server/src/main.rs @@ -2,6 +2,7 @@ mod admin; mod ai_generation_drafts; +mod aliyun_matting; mod ai_tasks; mod api_response; mod app; diff --git a/server-rs/crates/api-server/src/state.rs b/server-rs/crates/api-server/src/state.rs index 923e7906b..f62b7a705 100644 --- a/server-rs/crates/api-server/src/state.rs +++ b/server-rs/crates/api-server/src/state.rs @@ -24,6 +24,7 @@ use platform_auth::{ SmsAuthProviderKind, SmsProviderError, WechatProvider, sign_access_token, verify_access_token, }; use platform_llm::{LlmClient, LlmConfig, LlmError, LlmProvider}; +use platform_matting::{MattingClient, MattingConfig}; use platform_oss::{OssClient, OssConfig, OssError}; use platform_wechat::{WechatClient, WechatConfig, pay::WechatPayClient}; use serde_json::Value; @@ -269,6 +270,7 @@ pub struct AppStateInner { editor_generation_pricing_store: EditorGenerationPricingStore, llm_client: Option, creative_agent_gpt5_client: Option, + matting_client: Option, creative_agent_executor: Arc, // Phase 1 任务 E 的 creative session facade 暂存在 api-server。 // creative_agent_* 表由任务 D 收口后，这里只保留读写 facade。 @@ -418,6 +420,7 @@ impl AppState { .map_err(|error| AppStateInitError::DependencyUnavailable(error.to_string()))?; let llm_client = build_llm_client(&config)?; let creative_agent_gpt5_client = build_creative_agent_gpt5_client(&config)?; + let matting_client = build_matting_client(&config)?; let http_request_permit_pools = HttpRequestPermitPools::from_config(&config); let (profile_recharge_order_updates, _) = broadcast::channel(128); @@ -455,6 +458,7 @@ impl AppState { editor_generation_pricing_store, llm_client, creative_agent_gpt5_client, + matting_client, creative_agent_executor: Arc::new(MockLangChainRustAgentExecutor), creative_agent_sessions: Arc::new(Mutex::new(HashMap::new())), profile_recharge_order_updates, @@ -917,6 +921,10 @@ impl AppState { self.creative_agent_gpt5_client.as_ref() } + pub fn matting_client(&self) -> Option<&MattingClient> { + self.matting_client.as_ref() + } + pub fn creative_agent_executor(&self) -> Arc { self.creative_agent_executor.clone() } @@ -1446,6 +1454,38 @@ fn build_oss_client(config: &AppConfig) -> Result, AppStateIni Ok(Some(OssClient::new(oss_config))) } +fn build_matting_client(config: &AppConfig) -> Result, AppStateInitError> { + if !config.aliyun_matting_enabled { + return Ok(None); + } + let (Some(access_key_id), Some(access_key_secret)) = ( + config + .aliyun_matting_access_key_id + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()), + config + .aliyun_matting_access_key_secret + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()), + ) else { + warn!("阿里云抠图 AccessKey 未配置，跳过抠图客户端初始化"); + return Ok(None); + }; + + let matting_config = MattingConfig::with_timeout( + config.aliyun_matting_endpoint.clone(), + access_key_id.to_string(), + access_key_secret.to_string(), + config.aliyun_matting_request_timeout_ms, + ) + .map_err(|error| AppStateInitError::DependencyUnavailable(error.to_string()))?; + MattingClient::new(matting_config) + .map(Some) + .map_err(|error| AppStateInitError::DependencyUnavailable(error.to_string())) +} + fn build_wechat_client(config: &AppConfig) -> WechatClient { WechatClient::new(WechatConfig { app_id: config.wechat_mini_program_app_id.clone(), diff --git a/server-rs/crates/platform-matting/Cargo.toml b/server-rs/crates/platform-matting/Cargo.toml new file mode 100644 index 000000000..87a387af0 --- /dev/null +++ b/server-rs/crates/platform-matting/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "platform-matting" +edition.workspace = true +version.workspace = true +license.workspace = true + +[dependencies] +base64 = { workspace = true } +hmac = { workspace = true } +hex = { workspace = true } +httpdate = { workspace = true } +image = { workspace = true, features = ["png", "jpeg", "webp"] } +sha1 = { workspace = true } +reqwest = { workspace = true, features = ["json", "rustls-tls"] } +serde = { workspace = true } +serde_json = { workspace = true } +serde_urlencoded = { workspace = true } +sha2 = { workspace = true } +time = { workspace = true, features = ["std"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["v4"] } + +[dev-dependencies] +dotenvy = { workspace = true } +platform-oss = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/server-rs/crates/platform-matting/examples/crop_size_probe.rs b/server-rs/crates/platform-matting/examples/crop_size_probe.rs new file mode 100644 index 000000000..21f89a891 --- /dev/null +++ b/server-rs/crates/platform-matting/examples/crop_size_probe.rs @@ -0,0 +1,104 @@ +//! 探测 SegmentCommonImage 在小尺寸（480p 级）输入下 crop/mask 的输出尺寸行为。 +//! +//! cargo run -p platform-matting --example crop_size_probe + +use std::io::Cursor; +use std::path::Path; + +use image::{DynamicImage, GenericImage, ImageFormat, Rgba, RgbaImage}; +use platform_matting::{ + DEFAULT_IMAGESEG_ENDPOINT, MattingClient, MattingConfig, SegmentCommonImageRequest, + SegmentReturnForm, +}; + +fn load_env_files() { + for candidate in [ + ".env", + ".env.local", + ".env.secrets.local", + "../.env", + "../.env.local", + "../.env.secrets.local", + ] { + if Path::new(candidate).exists() { + let _ = dotenvy::from_path_override(candidate); + } + } +} + +fn encode_png(image: &DynamicImage) -> Vec { + let mut buffer = Cursor::new(Vec::new()); + image + .write_to(&mut buffer, ImageFormat::Png) + .expect("PNG 编码应成功"); + buffer.into_inner() +} + +#[tokio::main] +async fn main() { + load_env_files(); + + let input_path = std::env::args().nth(1).unwrap_or_else(|| { + r"C:\Users\lingh\Downloads\gpt-image-2-elf-archer-2048x2048.png".to_string() + }); + let source = image::open(&input_path).expect("测试图片应可解码"); + + // 变体1：等比缩到 480（模拟 1:1 480p 视频帧，主体居中） + let centered = source.resize(480, 480, image::imageops::FilterType::CatmullRom); + + // 变体2：把 360 尺寸的主体贴到 640x480 画布左上角（模拟横版帧、主体偏移） + let small = source.resize(360, 360, image::imageops::FilterType::CatmullRom); + let corner_pixel = centered.to_rgba8().get_pixel(0, 0).0; + let mut offset_canvas = RgbaImage::from_pixel(640, 480, Rgba(corner_pixel)); + offset_canvas + .copy_from(&small.to_rgba8(), 20, 40) + .expect("贴图应成功"); + let offset = DynamicImage::ImageRgba8(offset_canvas); + + let matting_config = MattingConfig::new( + std::env::var("ALIYUN_IMAGESEG_ENDPOINT") + .unwrap_or_else(|_| DEFAULT_IMAGESEG_ENDPOINT.to_string()), + std::env::var("ALIBABA_CLOUD_ACCESS_KEY_ID").expect("需要 ALIBABA_CLOUD_ACCESS_KEY_ID"), + std::env::var("ALIBABA_CLOUD_ACCESS_KEY_SECRET") + .expect("需要 ALIBABA_CLOUD_ACCESS_KEY_SECRET"), + ) + .expect("抠图配置应有效"); + let client = MattingClient::new(matting_config).expect("客户端应可构建"); + let http_client = reqwest::Client::new(); + + for (label, variant) in [("居中480x480", ¢ered), ("偏移640x480", &offset)] { + let bytes = encode_png(variant); + let temp_url = client + .upload_temp_image(bytes, &format!("probe-{label}.png"), "image/png") + .await + .expect("上传临时桶应成功"); + + for return_form in [None, Some(SegmentReturnForm::Crop), Some(SegmentReturnForm::Mask)] { + let result = client + .segment_common_image(SegmentCommonImageRequest { + image_url: temp_url.clone(), + return_form, + }) + .await + .expect("抠图调用应成功"); + let output_bytes = http_client + .get(&result.image_url) + .send() + .await + .expect("下载结果应成功") + .bytes() + .await + .expect("读取结果字节应成功"); + let output = image::load_from_memory(&output_bytes).expect("结果应可解码"); + println!( + "{label} 输入 {}x{} -> {} 输出 {}x{} (alpha通道: {})", + variant.width(), + variant.height(), + return_form.map_or("默认(不传)".to_string(), |f| format!("{f:?}")), + output.width(), + output.height(), + output.color().has_alpha() + ); + } + } +} diff --git a/server-rs/crates/platform-matting/examples/segment_smoke.rs b/server-rs/crates/platform-matting/examples/segment_smoke.rs new file mode 100644 index 000000000..82d08639d --- /dev/null +++ b/server-rs/crates/platform-matting/examples/segment_smoke.rs @@ -0,0 +1,158 @@ +//! 通用抠图冒烟验证：本地图片 → OSS → SegmentCommonImage → 下载结果。 +//! +//! 运行（在 server-rs 目录下）： +//! cargo run -p platform-matting --example segment_smoke -- "C:\path\to\input.png" +//! +//! 依赖仓库根目录 .env.local（OSS bucket/endpoint）与 .env.secrets.local（AK/SK）。 + +use std::path::{Path, PathBuf}; + +use platform_matting::{ + DEFAULT_IMAGESEG_ENDPOINT, MattingClient, MattingConfig, SegmentCommonImageRequest, + SegmentReturnForm, +}; + +fn load_env_files() { + // example 的工作目录通常是 server-rs，.env 都在仓库根目录。 + for candidate in [ + ".env", + ".env.local", + ".env.secrets.local", + "../.env", + "../.env.local", + "../.env.secrets.local", + ] { + if Path::new(candidate).exists() { + let _ = dotenvy::from_path_override(candidate); + } + } +} + +#[tokio::main] +async fn main() { + load_env_files(); + + let input_path = std::env::args().nth(1).unwrap_or_else(|| { + r"C:\Users\lingh\Downloads\gpt-image-2-elf-archer-2048x2048.png".to_string() + }); + let input_bytes = std::fs::read(&input_path) + .unwrap_or_else(|error| panic!("读取测试图片失败（{input_path}）：{error}")); + println!("[1/5] 已读取测试图片：{input_path}（{} 字节）", input_bytes.len()); + + // SegmentCommonImage 要求分辨率低于 2000x2000，超限先等比缩小。 + const MAX_EDGE: u32 = 1999; + let decoded = image::load_from_memory(&input_bytes).expect("测试图片应可解码"); + let input_bytes = if decoded.width() > MAX_EDGE || decoded.height() > MAX_EDGE { + let resized = decoded.resize( + MAX_EDGE, + MAX_EDGE, + image::imageops::FilterType::CatmullRom, + ); + let mut buffer = std::io::Cursor::new(Vec::new()); + resized + .write_to(&mut buffer, image::ImageFormat::Png) + .expect("缩放后的图片应可编码为 PNG"); + println!( + " 分辨率 {}x{} 超限，已缩放到 {}x{}", + decoded.width(), + decoded.height(), + resized.width(), + resized.height() + ); + buffer.into_inner() + } else { + input_bytes + }; + + let http_client = reqwest::Client::new(); + + // --- 调用通用抠图 --- + // key 优先级：VIAPI 专用 → 官方 SDK 标准命名（#IMAGE_CALL）→ 短信 key 兜底。 + let (matting_key_id, matting_key_secret) = [ + ("ALIYUN_IMAGESEG_ACCESS_KEY_ID", "ALIYUN_IMAGESEG_ACCESS_KEY_SECRET"), + ("ALIBABA_CLOUD_ACCESS_KEY_ID", "ALIBABA_CLOUD_ACCESS_KEY_SECRET"), + ("ALIYUN_SMS_ACCESS_KEY_ID", "ALIYUN_SMS_ACCESS_KEY_SECRET"), + ] + .iter() + .find_map(|(id_name, secret_name)| { + let id = std::env::var(id_name).ok()?; + let secret = std::env::var(secret_name).ok()?; + if id.trim().is_empty() || secret.trim().is_empty() { + return None; + } + println!(" 使用 {id_name} 调用抠图服务"); + Some((id, secret)) + }) + .expect("未找到可用的抠图 AccessKey 环境变量"); + let matting_config = MattingConfig::new( + std::env::var("ALIYUN_IMAGESEG_ENDPOINT") + .unwrap_or_else(|_| DEFAULT_IMAGESEG_ENDPOINT.to_string()), + matting_key_id, + matting_key_secret, + ) + .expect("抠图配置应有效"); + let matting_client = MattingClient::new(matting_config).expect("抠图客户端应可构建"); + + // 本地 OSS 在北京地域，抠图服务要求上海地域，走 VIAPI 官方临时桶上传。 + let temp_url = matting_client + .upload_temp_image(input_bytes, "segment-input.png", "image/png") + .await + .expect("上传 VIAPI 临时桶应成功"); + println!("[2/5] 已上传 VIAPI 临时桶"); + println!("[3/5] 输入图 URL host：{}", host_of(&temp_url)); + + let result = matting_client + .segment_common_image(SegmentCommonImageRequest { + image_url: temp_url, + return_form: Some(SegmentReturnForm::Crop), + }) + .await; + let result = match result { + Ok(result) => result, + Err(error) => { + eprintln!("[4/5] 通用抠图调用失败：{error}"); + std::process::exit(1); + } + }; + println!( + "[4/5] 通用抠图成功，RequestId={}", + result.request_id.as_deref().unwrap_or("unknown") + ); + + // --- 下载结果 --- + let output_bytes = http_client + .get(&result.image_url) + .send() + .await + .expect("下载抠图结果应成功") + .error_for_status() + .expect("抠图结果 URL 应返回 200") + .bytes() + .await + .expect("读取抠图结果字节应成功"); + + let output_path = build_output_path(&input_path); + std::fs::write(&output_path, &output_bytes).expect("写出抠图结果应成功"); + println!( + "[5/5] 抠图结果已保存：{}（{} 字节）", + output_path.display(), + output_bytes.len() + ); +} + +fn build_output_path(input_path: &str) -> PathBuf { + let input = Path::new(input_path); + let stem = input + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or("segment-output"); + input.with_file_name(format!("{stem}-matting.png")) +} + +fn host_of(url: &str) -> String { + url.split("//") + .nth(1) + .and_then(|rest| rest.split('/').next()) + .unwrap_or("unknown") + .to_string() +} diff --git a/server-rs/crates/platform-matting/src/lib.rs b/server-rs/crates/platform-matting/src/lib.rs new file mode 100644 index 000000000..0a61e314d --- /dev/null +++ b/server-rs/crates/platform-matting/src/lib.rs @@ -0,0 +1,724 @@ +//! 阿里云 VIAPI 通用抠图（SegmentCommonImage）客户端。 +//! +//! 官方没有 Rust SDK，这里按 ACS3-HMAC-SHA256 签名协议手搓 HTTP 调用， +//! 签名实现与 platform-auth 的阿里云短信调用保持一致。 +//! 文档：https://help.aliyun.com/zh/viapi/use-cases/general-image-segmentation + +use std::collections::BTreeMap; + +use hmac::{Hmac, Mac}; +use reqwest::{Client, StatusCode}; +use sha2::{Digest, Sha256}; +use time::OffsetDateTime; +use tracing::{info, warn}; + +type HmacSha256 = Hmac; + +pub const DEFAULT_IMAGESEG_ENDPOINT: &str = "imageseg.cn-shanghai.aliyuncs.com"; +const IMAGESEG_API_VERSION: &str = "2019-12-30"; +const SEGMENT_COMMON_IMAGE_ACTION: &str = "SegmentCommonImage"; + +// VIAPI 官方临时上传通道：非上海地域 OSS / 本地文件先传到官方临时桶（1 天过期， +// 全用户共享 QPS，生产建议直接用上海地域自有 OSS）。 +// 文档：https://help.aliyun.com/document_detail/155645.html +const VIAPI_UTILS_ENDPOINT: &str = "viapiutils.cn-shanghai.aliyuncs.com"; +const VIAPI_UTILS_VERSION: &str = "2020-04-01"; +const GET_OSS_STS_TOKEN_ACTION: &str = "GetOssStsToken"; +const VIAPI_TEMP_BUCKET: &str = "viapi-customer-temp"; +const VIAPI_TEMP_OSS_HOST: &str = "viapi-customer-temp.oss-cn-shanghai.aliyuncs.com"; + +pub const DEFAULT_MATTING_REQUEST_TIMEOUT_MS: u64 = 30_000; + +/// SegmentCommonImage 要求每条边小于 2000。 +const MAX_INPUT_EDGE: u32 = 1_999; + +#[derive(Clone, Debug)] +pub struct MattingConfig { + pub endpoint: String, + pub access_key_id: String, + pub access_key_secret: String, + pub request_timeout_ms: u64, +} + +impl MattingConfig { + pub fn new( + endpoint: String, + access_key_id: String, + access_key_secret: String, + ) -> Result { + Self::with_timeout( + endpoint, + access_key_id, + access_key_secret, + DEFAULT_MATTING_REQUEST_TIMEOUT_MS, + ) + } + + pub fn with_timeout( + endpoint: String, + access_key_id: String, + access_key_secret: String, + request_timeout_ms: u64, + ) -> Result { + let endpoint = endpoint + .trim() + .trim_start_matches("https://") + .trim_start_matches("http://") + .trim_matches('/') + .to_string(); + if endpoint.is_empty() { + return Err(MattingError::InvalidConfig( + "抠图服务 endpoint 不能为空".to_string(), + )); + } + let access_key_id = access_key_id.trim().to_string(); + let access_key_secret = access_key_secret.trim().to_string(); + if access_key_id.is_empty() || access_key_secret.is_empty() { + return Err(MattingError::InvalidConfig( + "抠图服务 AccessKeyId/AccessKeySecret 不能为空".to_string(), + )); + } + Ok(Self { + endpoint, + access_key_id, + access_key_secret, + request_timeout_ms: request_timeout_ms.max(1), + }) + } +} + +/// SegmentCommonImage 的 ReturnForm 取值。 +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SegmentReturnForm { + /// 透明背景 PNG（默认，抠像主用途）。 + Crop, + /// 黑白 mask 图。 + Mask, + /// 白底图。 + WhiteBackground, +} + +impl SegmentReturnForm { + fn as_str(&self) -> &'static str { + match self { + Self::Crop => "crop", + Self::Mask => "mask", + Self::WhiteBackground => "whiteBK", + } + } +} + +#[derive(Clone, Debug)] +pub struct SegmentCommonImageRequest { + /// 待抠图图片的公网可访问 URL（推荐上海地域 OSS 签名 URL）。 + pub image_url: String, + /// None 表示不传 ReturnForm，使用服务端默认行为。 + pub return_form: Option, +} + +#[derive(Clone, Debug)] +pub struct SegmentCommonImageResult { + /// 抠图结果图片 URL（阿里云临时地址，30 分钟内有效，需及时转存）。 + pub image_url: String, + pub request_id: Option, +} + +#[derive(Debug)] +pub enum MattingError { + InvalidConfig(String), + InvalidRequest(String), + Sign(String), + Upstream(String), +} + +impl MattingError { + pub fn message(&self) -> &str { + match self { + Self::InvalidConfig(message) + | Self::InvalidRequest(message) + | Self::Sign(message) + | Self::Upstream(message) => message, + } + } +} + +impl std::fmt::Display for MattingError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.message()) + } +} + +impl std::error::Error for MattingError {} + +#[derive(Clone)] +pub struct MattingClient { + config: MattingConfig, + client: Client, +} + +impl std::fmt::Debug for MattingClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MattingClient") + .field("endpoint", &self.config.endpoint) + .finish_non_exhaustive() + } +} + +impl MattingClient { + pub fn new(config: MattingConfig) -> Result { + let client = Client::builder() + .timeout(std::time::Duration::from_millis(config.request_timeout_ms)) + .build() + .map_err(|error| { + MattingError::InvalidConfig(format!("构建 reqwest client 失败：{error}")) + })?; + Ok(Self { config, client }) + } + + pub async fn segment_common_image( + &self, + request: SegmentCommonImageRequest, + ) -> Result { + let image_url = request.image_url.trim().to_string(); + if image_url.is_empty() { + return Err(MattingError::InvalidRequest( + "imageUrl 不能为空".to_string(), + )); + } + + let mut form = BTreeMap::new(); + form.insert("Action".to_string(), SEGMENT_COMMON_IMAGE_ACTION.to_string()); + form.insert("Format".to_string(), "json".to_string()); + form.insert("Version".to_string(), IMAGESEG_API_VERSION.to_string()); + form.insert("ImageURL".to_string(), image_url); + if let Some(return_form) = request.return_form { + form.insert("ReturnForm".to_string(), return_form.as_str().to_string()); + } + + let payload = build_aliyun_form_body(&form); + let headers = self.build_signature_headers(SEGMENT_COMMON_IMAGE_ACTION, &payload)?; + + info!( + provider = "aliyun-imageseg", + endpoint = self.config.endpoint.as_str(), + action = SEGMENT_COMMON_IMAGE_ACTION, + return_form = request + .return_form + .map_or("default", |return_form| return_form.as_str()), + "准备调用阿里云通用抠图接口" + ); + + let response = self + .client + .post(format!("https://{}/", self.config.endpoint)) + .headers(headers) + .header( + reqwest::header::CONTENT_TYPE, + "application/x-www-form-urlencoded", + ) + .body(payload) + .send() + .await + .map_err(|error| MattingError::Upstream(format!("通用抠图请求失败：{error}")))?; + + let http_status = response.status(); + let body_text = response.text().await.map_err(|error| { + MattingError::Upstream(format!("通用抠图响应读取失败：{error}")) + })?; + let body: serde_json::Value = serde_json::from_str(&body_text).map_err(|error| { + MattingError::Upstream(format!( + "通用抠图响应不是合法 JSON：{error}；原始响应：{}", + truncate_for_log(&body_text) + )) + })?; + + let request_id = body + .get("RequestId") + .and_then(|value| value.as_str()) + .map(|value| value.to_string()); + + if http_status != StatusCode::OK { + let code = body.get("Code").and_then(|value| value.as_str()); + let message = body.get("Message").and_then(|value| value.as_str()); + warn!( + provider = "aliyun-imageseg", + http_status = http_status.as_u16(), + provider_code = code.unwrap_or("unknown"), + provider_message = message.unwrap_or("unknown"), + provider_request_id = request_id.as_deref().unwrap_or("unknown"), + "阿里云通用抠图接口返回失败" + ); + return Err(MattingError::Upstream(format!( + "通用抠图接口返回失败（HTTP {}，Code={}）：{}", + http_status.as_u16(), + code.unwrap_or("unknown"), + message.unwrap_or("unknown") + ))); + } + + let result_image_url = body + .get("Data") + .and_then(|data| data.get("ImageURL")) + .and_then(|value| value.as_str()) + .map(|value| value.to_string()) + .ok_or_else(|| { + MattingError::Upstream(format!( + "通用抠图响应缺少 Data.ImageURL；原始响应：{}", + truncate_for_log(&body_text) + )) + })?; + + info!( + provider = "aliyun-imageseg", + provider_request_id = request_id.as_deref().unwrap_or("unknown"), + "阿里云通用抠图接口调用成功" + ); + + Ok(SegmentCommonImageResult { + image_url: result_image_url, + request_id, + }) + } + + /// 图片字节 → 通用抠图 → 原尺寸透明 PNG 字节。 + /// + /// - `accessible_url`：该字节对应的、抠图服务可访问的 URL（上海地域 OSS 签名 URL）。 + /// 为 None 或图片超分辨率上限需要缩图时，自动改走官方临时桶上传。 + /// - 分辨率守卫：任一边 >= 2000 时先等比缩小送抠，抠完只取结果 alpha 上采样回贴 + /// 原图 RGB，保证输出与输入同尺寸、画质无损。 + pub async fn segment_image_to_transparent_png( + &self, + bytes: &[u8], + file_name: &str, + accessible_url: Option, + ) -> Result, MattingError> { + let source = image::load_from_memory(bytes).map_err(|error| { + MattingError::InvalidRequest(format!("解析待抠图图片失败：{error}")) + })?; + let source_rgba = source.to_rgba8(); + let (source_width, source_height) = source_rgba.dimensions(); + + let needs_resize = source_width > MAX_INPUT_EDGE || source_height > MAX_INPUT_EDGE; + let (submit_bytes, submit_url) = if needs_resize { + let resized = source.resize( + MAX_INPUT_EDGE, + MAX_INPUT_EDGE, + image::imageops::FilterType::CatmullRom, + ); + let resized_bytes = encode_rgba_png(&resized.to_rgba8())?; + (Some(resized_bytes), None) + } else { + (None, accessible_url) + }; + + let image_url = match submit_url { + Some(url) => url, + None => { + let upload_bytes = submit_bytes + .clone() + .unwrap_or_else(|| bytes.to_vec()); + self.upload_temp_image(upload_bytes, file_name, "image/png") + .await? + } + }; + + let result = self + .segment_common_image(SegmentCommonImageRequest { + image_url, + // 默认 ReturnForm：原尺寸 + 透明背景，无需本地合成。 + return_form: None, + }) + .await?; + let result_bytes = self.download_result_image(&result.image_url).await?; + let result_image = image::load_from_memory(&result_bytes) + .map_err(|error| { + MattingError::Upstream(format!("解析抠图结果图片失败：{error}")) + })? + .to_rgba8(); + + if !needs_resize { + if result_image.dimensions() != (source_width, source_height) { + return Err(MattingError::Upstream(format!( + "抠图结果尺寸 {}x{} 与输入 {}x{} 不一致", + result_image.width(), + result_image.height(), + source_width, + source_height + ))); + } + return encode_rgba_png(&result_image); + } + + // 缩图送抠的场景：只取结果 alpha，上采样回原尺寸后贴回原图 RGB。 + let alpha_mask = image::DynamicImage::ImageRgba8(result_image).resize_exact( + source_width, + source_height, + image::imageops::FilterType::Triangle, + ); + let alpha_mask = alpha_mask.to_rgba8(); + let mut output = source_rgba; + for (target, mask) in output.pixels_mut().zip(alpha_mask.pixels()) { + target.0[3] = target.0[3].min(mask.0[3]); + } + encode_rgba_png(&output) + } + + async fn download_result_image(&self, url: &str) -> Result, MattingError> { + let response = self + .client + .get(url) + .send() + .await + .map_err(|error| MattingError::Upstream(format!("下载抠图结果失败：{error}")))?; + let status = response.status(); + if !status.is_success() { + return Err(MattingError::Upstream(format!( + "下载抠图结果失败（HTTP {}）", + status.as_u16() + ))); + } + response + .bytes() + .await + .map(|bytes| bytes.to_vec()) + .map_err(|error| MattingError::Upstream(format!("读取抠图结果字节失败：{error}"))) + } + + /// 把本地图片字节上传到 VIAPI 官方临时桶，返回可直接作为 ImageURL 的公网地址。 + /// 适用于本地文件或非上海地域 OSS 的开发/调试场景；生产建议用上海地域自有 OSS。 + pub async fn upload_temp_image( + &self, + bytes: Vec, + file_name: &str, + content_type: &str, + ) -> Result { + if bytes.is_empty() { + return Err(MattingError::InvalidRequest( + "上传内容不能为空".to_string(), + )); + } + let sts = self.get_oss_sts_token().await?; + let file_name = file_name.trim().trim_matches('/'); + if file_name.is_empty() { + return Err(MattingError::InvalidRequest( + "file_name 不能为空".to_string(), + )); + } + let object_key = format!( + "{}/{}/{}", + self.config.access_key_id, + uuid::Uuid::new_v4().simple(), + file_name + ); + let date = httpdate::fmt_http_date(std::time::SystemTime::now()); + // OSS V1 头签名（带 STS security token）。 + let string_to_sign = format!( + "PUT\n\n{content_type}\n{date}\nx-oss-security-token:{}\n/{VIAPI_TEMP_BUCKET}/{object_key}", + sts.security_token + ); + let signature = hmac_sha1_base64(sts.access_key_secret.as_bytes(), string_to_sign.as_bytes())?; + let authorization = format!("OSS {}:{}", sts.access_key_id, signature); + + let target_url = format!("https://{VIAPI_TEMP_OSS_HOST}/{object_key}"); + let response = self + .client + .put(&target_url) + .header(reqwest::header::CONTENT_TYPE, content_type) + .header(reqwest::header::DATE, &date) + .header("x-oss-security-token", &sts.security_token) + .header(reqwest::header::AUTHORIZATION, &authorization) + .body(bytes) + .send() + .await + .map_err(|error| { + MattingError::Upstream(format!("上传 VIAPI 临时桶请求失败：{error}")) + })?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + return Err(MattingError::Upstream(format!( + "上传 VIAPI 临时桶失败（HTTP {}）：{}", + status.as_u16(), + truncate_for_log(&body) + ))); + } + + Ok(target_url) + } + + async fn get_oss_sts_token(&self) -> Result { + let mut form = BTreeMap::new(); + form.insert("Action".to_string(), GET_OSS_STS_TOKEN_ACTION.to_string()); + form.insert("Format".to_string(), "json".to_string()); + form.insert("Version".to_string(), VIAPI_UTILS_VERSION.to_string()); + let payload = build_aliyun_form_body(&form); + let headers = self.build_acs3_headers( + VIAPI_UTILS_ENDPOINT, + GET_OSS_STS_TOKEN_ACTION, + VIAPI_UTILS_VERSION, + &payload, + )?; + + let response = self + .client + .post(format!("https://{VIAPI_UTILS_ENDPOINT}/")) + .headers(headers) + .header( + reqwest::header::CONTENT_TYPE, + "application/x-www-form-urlencoded", + ) + .body(payload) + .send() + .await + .map_err(|error| MattingError::Upstream(format!("GetOssStsToken 请求失败：{error}")))?; + let http_status = response.status(); + let body_text = response.text().await.map_err(|error| { + MattingError::Upstream(format!("GetOssStsToken 响应读取失败：{error}")) + })?; + let body: serde_json::Value = serde_json::from_str(&body_text).map_err(|error| { + MattingError::Upstream(format!( + "GetOssStsToken 响应不是合法 JSON：{error}；原始响应：{}", + truncate_for_log(&body_text) + )) + })?; + if http_status != StatusCode::OK { + return Err(MattingError::Upstream(format!( + "GetOssStsToken 返回失败（HTTP {}，Code={}）：{}", + http_status.as_u16(), + body.get("Code").and_then(|value| value.as_str()).unwrap_or("unknown"), + body.get("Message") + .and_then(|value| value.as_str()) + .unwrap_or("unknown") + ))); + } + let data = body.get("Data").ok_or_else(|| { + MattingError::Upstream(format!( + "GetOssStsToken 响应缺少 Data；原始响应：{}", + truncate_for_log(&body_text) + )) + })?; + let read_field = |name: &str| -> Result { + data.get(name) + .and_then(|value| value.as_str()) + .map(|value| value.to_string()) + .ok_or_else(|| { + MattingError::Upstream(format!("GetOssStsToken 响应缺少 Data.{name}")) + }) + }; + Ok(ViapiStsToken { + access_key_id: read_field("AccessKeyId")?, + access_key_secret: read_field("AccessKeySecret")?, + security_token: read_field("SecurityToken")?, + }) + } + + fn build_signature_headers( + &self, + action: &str, + payload: &str, + ) -> Result { + self.build_acs3_headers(&self.config.endpoint, action, IMAGESEG_API_VERSION, payload) + } + + fn build_acs3_headers( + &self, + endpoint: &str, + action: &str, + version: &str, + payload: &str, + ) -> Result { + let date = current_aliyun_timestamp(); + let nonce = uuid::Uuid::new_v4().simple().to_string(); + let payload_hash = sha256_hex(payload.as_bytes()); + let canonical_headers = format!( + "host:{}\nx-acs-action:{}\nx-acs-content-sha256:{}\nx-acs-date:{}\nx-acs-signature-nonce:{}\nx-acs-version:{}\n", + endpoint, action, payload_hash, date, nonce, version + ); + let signed_headers = + "host;x-acs-action;x-acs-content-sha256;x-acs-date;x-acs-signature-nonce;x-acs-version"; + let canonical_request = format!( + "POST\n/\n\n{}\n{}\n{}", + canonical_headers, signed_headers, payload_hash + ); + let string_to_sign = format!( + "ACS3-HMAC-SHA256\n{}", + sha256_hex(canonical_request.as_bytes()) + ); + let signature = hmac_sha256_hex( + self.config.access_key_secret.as_bytes(), + string_to_sign.as_bytes(), + )?; + let authorization = format!( + "ACS3-HMAC-SHA256 Credential={},SignedHeaders={signed_headers},Signature={signature}", + self.config.access_key_id + ); + let mut headers = reqwest::header::HeaderMap::new(); + insert_header(&mut headers, "x-acs-action", action)?; + insert_header(&mut headers, "x-acs-version", version)?; + insert_header(&mut headers, "x-acs-date", &date)?; + insert_header(&mut headers, "x-acs-signature-nonce", &nonce)?; + insert_header(&mut headers, "x-acs-content-sha256", &payload_hash)?; + insert_header(&mut headers, "authorization", &authorization)?; + + Ok(headers) + } +} + +fn encode_rgba_png(image: &image::RgbaImage) -> Result, MattingError> { + use image::ImageEncoder as _; + let mut encoded = Vec::new(); + image::codecs::png::PngEncoder::new(&mut encoded) + .write_image( + image.as_raw(), + image.width(), + image.height(), + image::ExtendedColorType::Rgba8, + ) + .map_err(|error| MattingError::Upstream(format!("编码抠图结果 PNG 失败：{error}")))?; + Ok(encoded) +} + +struct ViapiStsToken { + access_key_id: String, + access_key_secret: String, + security_token: String, +} + +fn hmac_sha1_base64(key: &[u8], content: &[u8]) -> Result { + use base64::Engine as _; + let mut signer = Hmac::::new_from_slice(key) + .map_err(|error| MattingError::Sign(format!("初始化 OSS V1 签名器失败：{error}")))?; + signer.update(content); + Ok(base64::engine::general_purpose::STANDARD.encode(signer.finalize().into_bytes())) +} + +fn insert_header( + headers: &mut reqwest::header::HeaderMap, + name: &'static str, + value: &str, +) -> Result<(), MattingError> { + let value = reqwest::header::HeaderValue::from_str(value) + .map_err(|error| MattingError::Sign(format!("构造请求头 {name} 失败：{error}")))?; + headers.insert(name, value); + Ok(()) +} + +fn current_aliyun_timestamp() -> String { + // 阿里云 OpenAPI ACS3 签名头 x-acs-date 要求不带小数秒的 UTC ISO 8601 格式 + // （yyyy-MM-dd'T'HH:mm:ss'Z'），Rfc3339 默认会保留纳秒，网关会判定时间格式非法。 + let now = OffsetDateTime::now_utc(); + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", + now.year(), + u8::from(now.month()), + now.day(), + now.hour(), + now.minute(), + now.second() + ) +} + +fn canonicalize_aliyun_form_params(params: &BTreeMap) -> String { + params + .iter() + .map(|(key, value)| { + format!( + "{}={}", + urlencoding_encode(key), + urlencoding_encode(value) + ) + }) + .collect::>() + .join("&") +} + +fn urlencoding_encode(value: &str) -> String { + serde_urlencoded::to_string([("k", value)]) + .map(|encoded| encoded.trim_start_matches("k=").to_string()) + .unwrap_or_else(|_| value.to_string()) +} + +fn build_aliyun_form_body(params: &BTreeMap) -> String { + serde_urlencoded::to_string(params).unwrap_or_else(|_| canonicalize_aliyun_form_params(params)) +} + +fn hmac_sha256_hex(key: &[u8], content: &[u8]) -> Result { + let mut signer = HmacSha256::new_from_slice(key) + .map_err(|error| MattingError::Sign(format!("初始化抠图签名器失败：{error}")))?; + signer.update(content); + Ok(hex::encode(signer.finalize().into_bytes())) +} + +fn sha256_hex(content: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(content); + hex::encode(hasher.finalize()) +} + +fn truncate_for_log(text: &str) -> String { + const LIMIT: usize = 512; + if text.len() <= LIMIT { + text.to_string() + } else { + let mut end = LIMIT; + while !text.is_char_boundary(end) { + end -= 1; + } + format!("{}...(截断)", &text[..end]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn form_body_is_sorted_and_url_encoded() { + let mut params = BTreeMap::new(); + params.insert("ReturnForm".to_string(), "crop".to_string()); + params.insert( + "ImageURL".to_string(), + "https://example.com/a.png?x=1&y=2".to_string(), + ); + params.insert("Action".to_string(), "SegmentCommonImage".to_string()); + + let body = build_aliyun_form_body(¶ms); + assert_eq!( + body, + "Action=SegmentCommonImage&ImageURL=https%3A%2F%2Fexample.com%2Fa.png%3Fx%3D1%26y%3D2&ReturnForm=crop" + ); + } + + #[test] + fn signature_headers_use_acs3_sha256() { + let config = MattingConfig::new( + DEFAULT_IMAGESEG_ENDPOINT.to_string(), + "test-key-id".to_string(), + "test-key-secret".to_string(), + ) + .expect("config should build"); + let client = MattingClient::new(config).expect("client should build"); + let headers = client + .build_signature_headers("SegmentCommonImage", "ImageURL=x") + .expect("headers should build"); + + let authorization = headers + .get("authorization") + .expect("authorization header should exist") + .to_str() + .expect("authorization header should be ascii"); + assert!(authorization.starts_with("ACS3-HMAC-SHA256 Credential=test-key-id,")); + assert!(authorization.contains("SignedHeaders=host;x-acs-action;x-acs-content-sha256;")); + assert_eq!( + headers + .get("x-acs-version") + .and_then(|value| value.to_str().ok()), + Some("2019-12-30") + ); + let date = headers + .get("x-acs-date") + .and_then(|value| value.to_str().ok()) + .expect("x-acs-date should exist"); + assert!(!date.contains('.'), "x-acs-date 不能带小数秒：{date}"); + } +} -- 2.52.0 From ba8662f844fdef77cc36a602ede48eeebe4b0d07 Mon Sep 17 00:00:00 2001 From: Linghong Date: Wed, 8 Jul 2026 03:50:44 +0000 Subject: [PATCH 02/41] =?UTF-8?q?=E5=8A=A8=E4=BD=9C=E8=A7=86=E9=A2=91?= =?UTF-8?q?=E8=83=8C=E6=99=AF=E8=89=B2=E5=A4=9A=E8=89=B2=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E5=86=B3=E7=AD=96=E5=B9=B6=E6=8E=A5=E5=85=A5=E9=98=BF=E9=87=8C?= =?UTF-8?q?=E4=BA=91=E6=8A=A0=E5=B8=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 动画生成请求新增 screenColor，继承生图入口的 auto 决策行为， 提示词从写死绿幕改为纯色背景条款（LLM 决策 11 色） - 后端抽帧抠图改为逐帧阿里云优先（并发 3 保序），单帧失败降级 本地键色算法（使用决策出的背景色） - 清理 legacy 纯绿幕常量与提示词 Co-Authored-By: Claude Fable 5 --- .../src/character_animation_assets.rs | 138 +++++++++++++----- .../api-server/src/editor_green_screen.rs | 10 -- .../src/editor_screen_background_decision.rs | 2 + .../crates/shared-contracts/src/assets.rs | 3 + ...ageCanvasGenerationSubmissionModel.test.ts | 1 + .../ImageCanvasGenerationSubmissionModel.ts | 2 + .../image-editor/editorProjectClient.ts | 2 + 7 files changed, 113 insertions(+), 45 deletions(-) diff --git a/server-rs/crates/api-server/src/character_animation_assets.rs b/server-rs/crates/api-server/src/character_animation_assets.rs index ea44c174b..49eb850fb 100644 --- a/server-rs/crates/api-server/src/character_animation_assets.rs +++ b/server-rs/crates/api-server/src/character_animation_assets.rs @@ -61,10 +61,13 @@ use crate::{ editor_generation_source_entity_id, enqueue_editor_generation_job, }, editor_green_screen::{ - LEGACY_EDITOR_GREEN_SCREEN_CHARACTER_PROMPT_CLAUSE, - legacy_editor_green_screen_background_color, + EditorScreenBackgroundColor, editor_green_screen_character_prompt_clause, remove_editor_generated_green_screen_background, }, + editor_screen_background_decision::{ + EditorScreenBackgroundDecisionInput, EditorScreenBackgroundDecisionKind, + resolve_editor_screen_background_color, + }, editor_project::{ EditorCanvasGeneratedLayerInput, PersistEditorGeneratedAssetRequest, build_editor_canvas_generated_layer_item, complete_editor_canvas_generation_with_items, @@ -564,9 +567,13 @@ pub async fn generate_editor_character_animation( })), ) })?; - let normalized = - normalize_editor_character_animation_request_with_pricing(payload.clone(), &pricing) - .map_err(|error| character_animation_error_response(&request_context, error))?; + // 队列路径只取定价，背景色决策留到实际执行时再做，这里用默认色占位。 + let normalized = normalize_editor_character_animation_request_with_pricing( + payload.clone(), + &pricing, + crate::editor_green_screen::default_editor_screen_background_color(), + ) + .map_err(|error| character_animation_error_response(&request_context, error))?; let source_entity_id = editor_generation_source_entity_id( payload.project_id.as_deref(), payload.source_layer_id.as_str(), @@ -628,9 +635,25 @@ pub(crate) async fn generate_editor_character_animation_for_owner( })), ) })?; - let normalized = - normalize_editor_character_animation_request_with_pricing(payload, &pricing) - .map_err(|error| character_animation_error_response(&request_context, error))?; + // 背景色决策：显式传入 → 继承源角色图（前端从图层 generationInputs 带入）→ LLM 自动决策 → 默认色。 + let screen_background_decision = resolve_editor_screen_background_color( + state.llm_client(), + EditorScreenBackgroundDecisionInput { + kind: EditorScreenBackgroundDecisionKind::CharacterAnimation, + screen_color: payload.screen_color.clone(), + prompt: payload.prompt_text.clone(), + icon_descriptions: Vec::new(), + reference_count: 1, + }, + ) + .await + .map_err(|error| character_animation_error_response(&request_context, error))?; + let normalized = normalize_editor_character_animation_request_with_pricing( + payload, + &pricing, + screen_background_decision.color, + ) + .map_err(|error| character_animation_error_response(&request_context, error))?; let settings = require_editor_character_animation_settings(&state, &normalized) .map_err(|error| character_animation_error_response(&request_context, error))?; let extraction_settings = resolve_backend_frame_extraction_settings(&state); @@ -2075,10 +2098,13 @@ async fn extract_and_persist_editor_character_animation_frames( ) .await?; let finalized_frames = remove_editor_character_animation_frame_backgrounds( + state, finalized_frames, request.frame_width, request.frame_height, - )?; + request.screen_color, + ) + .await?; let mut frame_payloads = Vec::with_capacity(finalized_frames.len()); for (index, frame) in finalized_frames.into_iter().enumerate() { @@ -2147,30 +2173,58 @@ async fn persist_editor_character_animation_green_screen_source_frames( Ok(()) } -fn remove_editor_character_animation_frame_backgrounds( +/// 逐帧抠图：优先阿里云通用抠图，失败降级本地键色（使用生成时的纯色背景色）。 +/// 小并发保序处理，单帧降级不影响其他帧。 +async fn remove_editor_character_animation_frame_backgrounds( + state: &AppState, frames: Vec, frame_width: u32, frame_height: u32, + screen_color: EditorScreenBackgroundColor, ) -> Result, AppError> { - let mut removed_frames = Vec::with_capacity(frames.len()); - for frame in frames { - let removed = remove_editor_generated_green_screen_background( - &DownloadedOpenAiImage { + use futures_util::{StreamExt as _, TryStreamExt as _}; + + const FRAME_MATTING_CONCURRENCY: usize = 3; + + futures_util::stream::iter(frames.into_iter().enumerate().map( + |(frame_index, frame)| async move { + let image = DownloadedOpenAiImage { bytes: frame.bytes, mime_type: frame.mime_type, extension: frame.extension, - }, - legacy_editor_green_screen_background_color(), - )?; - removed_frames.push(finalize_animation_frame_payload( - removed.bytes.as_slice(), - removed.mime_type.as_str(), - frame_width, - frame_height, - false, - )?); - } - Ok(removed_frames) + }; + let removed = match crate::aliyun_matting::segment_image_with_aliyun_matting( + state, + &image, + "editor-animation-frame", + ) + .await + { + Ok(removed) => removed, + Err(error) => { + tracing::warn!( + provider = "aliyun-matting", + frame_index, + screen_color = screen_color.hex, + error = %error, + error_details = ?error.details(), + "editor_animation_frame_aliyun_matting_fallback_to_local" + ); + remove_editor_generated_green_screen_background(&image, screen_color)? + } + }; + finalize_animation_frame_payload( + removed.bytes.as_slice(), + removed.mime_type.as_str(), + frame_width, + frame_height, + false, + ) + }, + )) + .buffered(FRAME_MATTING_CONCURRENCY) + .try_collect::>() + .await } async fn publish_animation_set( @@ -2835,12 +2889,17 @@ fn normalize_editor_character_animation_request( ) -> Result { let pricing = crate::editor_generation_config::load_editor_generation_pricing_from_paths(None) .expect("默认模型定价配置必须合法"); - normalize_editor_character_animation_request_with_pricing(payload, &pricing) + normalize_editor_character_animation_request_with_pricing( + payload, + &pricing, + crate::editor_green_screen::default_editor_screen_background_color(), + ) } fn normalize_editor_character_animation_request_with_pricing( payload: EditorCharacterAnimationGenerateRequest, pricing: &EditorGenerationPricingConfig, + screen_color: EditorScreenBackgroundColor, ) -> Result { let source_layer_id = normalize_required_text(payload.source_layer_id.as_str(), ""); if source_layer_id.is_empty() { @@ -2880,12 +2939,13 @@ fn normalize_editor_character_animation_request_with_pricing( ratio, resolution, ); - let prompt = build_editor_character_animation_prompt(prompt_text.as_str()); + let prompt = build_editor_character_animation_prompt(prompt_text.as_str(), screen_color); Ok(NormalizedEditorCharacterAnimationRequest { source_layer_id, source_image_src, prompt, + screen_color, resolution: resolution.to_string(), ratio: resolve_editor_character_animation_provider_ratio( ratio, @@ -3385,10 +3445,13 @@ fn editor_video_bad_request(message: impl Into) -> AppError { })) } -fn build_editor_character_animation_prompt(prompt_text: &str) -> String { +fn build_editor_character_animation_prompt( + prompt_text: &str, + screen_color: EditorScreenBackgroundColor, +) -> String { format!( "生成游戏角色动画，参考图作为首帧和尾帧，画面中心构图，角色主体完整置于画面中央，禁止镜头透视，禁止特写。{}；禁止出现建筑、室内布景、风景、地面道具、漂浮物、烟雾叙事元素、文字或其他角色以外的场景内容。\n动作描述：\n{}", - LEGACY_EDITOR_GREEN_SCREEN_CHARACTER_PROMPT_CLAUSE, + editor_green_screen_character_prompt_clause(screen_color), prompt_text.trim() ) } @@ -5091,6 +5154,7 @@ struct NormalizedEditorCharacterAnimationRequest { frame_width: u32, frame_height: u32, fps: u32, + screen_color: EditorScreenBackgroundColor, } #[derive(Debug)] @@ -5319,6 +5383,7 @@ mod tests { source_width: 768, source_height: 1024, prompt_text: "待机呼吸，轻微摆动。".to_string(), + screen_color: None, resolution: "720p".to_string(), ratio: "same".to_string(), frame_count: 48, @@ -5357,6 +5422,7 @@ mod tests { source_width: 1024, source_height: 1024, prompt_text: "待机呼吸".to_string(), + screen_color: None, resolution: "480p".to_string(), ratio: "same".to_string(), frame_count: 32, @@ -5384,6 +5450,7 @@ mod tests { source_width: 1024, source_height: 1024, prompt_text: "奔跑".to_string(), + screen_color: None, resolution: "480p".to_string(), ratio: "1:1".to_string(), frame_count: 48, @@ -5400,14 +5467,15 @@ mod tests { } #[test] - fn editor_character_animation_builds_required_green_screen_prompt() { - let prompt = build_editor_character_animation_prompt("行走两步后回到站姿。"); + fn editor_character_animation_builds_required_screen_background_prompt() { + let color = crate::editor_green_screen::default_editor_screen_background_color(); + let prompt = build_editor_character_animation_prompt("行走两步后回到站姿。", color); assert!(prompt.contains("生成游戏角色动画")); assert!(prompt.contains("参考图作为首帧和尾帧")); - assert!(prompt.contains("背景固定为单一纯绿色 #00FF00 / RGB(0,255,0) 绿幕")); - assert!(prompt.contains("绿幕背景必须平整无纹理、无渐变、无阴影")); - assert!(prompt.contains("角色主体不得带绿色描边、绿色投影或绿色反光")); + assert!(prompt.contains(color.hex)); + assert!(prompt.contains("纯色背景必须平整无纹理、无渐变、无阴影")); + assert!(prompt.contains("角色主体不得带与背景色相同或相近的描边、投影或反光")); assert!(prompt.contains("动作描述：\n行走两步后回到站姿。")); } diff --git a/server-rs/crates/api-server/src/editor_green_screen.rs b/server-rs/crates/api-server/src/editor_green_screen.rs index 586b682d1..6ea4be842 100644 --- a/server-rs/crates/api-server/src/editor_green_screen.rs +++ b/server-rs/crates/api-server/src/editor_green_screen.rs @@ -100,15 +100,6 @@ pub(crate) fn default_editor_screen_background_color() -> EditorScreenBackground EDITOR_SCREEN_BACKGROUND_COLORS[0] } -pub(crate) fn legacy_editor_green_screen_background_color() -> EditorScreenBackgroundColor { - EditorScreenBackgroundColor { - label: "纯绿色", - hex: "#00FF00", - red: 0, - green: 255, - blue: 0, - } -} pub(crate) fn parse_editor_screen_background_color( value: Option<&str>, @@ -139,7 +130,6 @@ pub(crate) const EDITOR_GREEN_SCREEN_ASSET_GUARDRAILS: &str = "素材自身不要出现与背景色相同或相近的描边、底板、投影或反光"; pub(crate) const EDITOR_GREEN_SCREEN_CHARACTER_GUARDRAILS: &str = "角色主体不得带与背景色相同或相近的描边、投影或反光"; -pub(crate) const LEGACY_EDITOR_GREEN_SCREEN_CHARACTER_PROMPT_CLAUSE: &str = "背景固定为单一纯绿色 #00FF00 / RGB(0,255,0) 绿幕，只作为抠像底色；绿幕背景必须平整无纹理、无渐变、无阴影、无地面、无环境、无道具；角色主体不得带绿色描边、绿色投影或绿色反光"; fn editor_screen_background_color_prompt(color: EditorScreenBackgroundColor) -> String { if color.hex == "#00FF00" { diff --git a/server-rs/crates/api-server/src/editor_screen_background_decision.rs b/server-rs/crates/api-server/src/editor_screen_background_decision.rs index 021988940..848520451 100644 --- a/server-rs/crates/api-server/src/editor_screen_background_decision.rs +++ b/server-rs/crates/api-server/src/editor_screen_background_decision.rs @@ -31,6 +31,7 @@ pub(crate) struct EditorScreenBackgroundDecision { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum EditorScreenBackgroundDecisionKind { Character, + CharacterAnimation, IconSpritesheet, UiDesignAssetExtraction, } @@ -39,6 +40,7 @@ impl EditorScreenBackgroundDecisionKind { fn label(self) -> &'static str { match self { Self::Character => "角色形象生成", + Self::CharacterAnimation => "角色动作视频生成", Self::IconSpritesheet => "图标素材 spritesheet 生成", Self::UiDesignAssetExtraction => "UI 设计图素材提取", } diff --git a/server-rs/crates/shared-contracts/src/assets.rs b/server-rs/crates/shared-contracts/src/assets.rs index 90a03a889..0251a8f32 100644 --- a/server-rs/crates/shared-contracts/src/assets.rs +++ b/server-rs/crates/shared-contracts/src/assets.rs @@ -314,6 +314,9 @@ pub struct EditorCharacterAnimationGenerateRequest { pub source_width: u32, pub source_height: u32, pub prompt_text: String, + /// 纯色抠像背景色（hex）。None/auto 走自动决策；通常继承源角色图的背景色。 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub screen_color: Option, pub resolution: String, pub ratio: String, pub frame_count: u32, diff --git a/src/components/image-editor/ImageCanvasGenerationSubmissionModel.test.ts b/src/components/image-editor/ImageCanvasGenerationSubmissionModel.test.ts index 7abdb2cf7..85c6078e4 100644 --- a/src/components/image-editor/ImageCanvasGenerationSubmissionModel.test.ts +++ b/src/components/image-editor/ImageCanvasGenerationSubmissionModel.test.ts @@ -726,6 +726,7 @@ describe('ImageCanvasGenerationSubmissionModel', () => { sourceWidth: 960, sourceHeight: 1280, promptText: '循环奔跑动作', + screenColor: 'auto', resolution: '720p', ratio: 'same', frameCount: 48, diff --git a/src/components/image-editor/ImageCanvasGenerationSubmissionModel.ts b/src/components/image-editor/ImageCanvasGenerationSubmissionModel.ts index 4d57e52e9..6f2d1c139 100644 --- a/src/components/image-editor/ImageCanvasGenerationSubmissionModel.ts +++ b/src/components/image-editor/ImageCanvasGenerationSubmissionModel.ts @@ -649,6 +649,8 @@ export function buildCharacterAnimationSubmissionPlan({ sourceWidth: sourceLayer.originalWidth, sourceHeight: sourceLayer.originalHeight, promptText, + // 与生图入口一致：默认 auto，由后端做背景色自动决策。 + screenColor: DEFAULT_EDITOR_GENERATION_BACKGROUND_COLOR, resolution: panel.resolution, ratio: panel.ratio, frameCount: panel.frameCount, diff --git a/src/services/image-editor/editorProjectClient.ts b/src/services/image-editor/editorProjectClient.ts index bc831182b..5c0ef5c53 100644 --- a/src/services/image-editor/editorProjectClient.ts +++ b/src/services/image-editor/editorProjectClient.ts @@ -328,6 +328,8 @@ export type EditorCharacterAnimationGenerationInput = { sourceWidth: number; sourceHeight: number; promptText: string; + /** 纯色抠像背景色（hex 或 'auto'），继承生图的背景色选项行为。 */ + screenColor?: string; resolution: EditorCharacterAnimationResolution; ratio: EditorCharacterAnimationRatio; frameCount: EditorCharacterAnimationFrameCount; -- 2.52.0 From c53d6ffef04f9b27aac6ffa043feb5eeff132c76 Mon Sep 17 00:00:00 2001 From: Linghong Date: Wed, 8 Jul 2026 04:03:10 +0000 Subject: [PATCH 03/41] =?UTF-8?q?=E7=A7=BB=E9=99=A4=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=E6=80=A7=E7=9A=84=E6=8A=A0=E5=9B=BE=E5=B0=BA=E5=AF=B8=E6=8E=A2?= =?UTF-8?q?=E9=92=88=E7=A4=BA=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../examples/crop_size_probe.rs | 104 ------------------ 1 file changed, 104 deletions(-) delete mode 100644 server-rs/crates/platform-matting/examples/crop_size_probe.rs diff --git a/server-rs/crates/platform-matting/examples/crop_size_probe.rs b/server-rs/crates/platform-matting/examples/crop_size_probe.rs deleted file mode 100644 index 21f89a891..000000000 --- a/server-rs/crates/platform-matting/examples/crop_size_probe.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! 探测 SegmentCommonImage 在小尺寸（480p 级）输入下 crop/mask 的输出尺寸行为。 -//! -//! cargo run -p platform-matting --example crop_size_probe - -use std::io::Cursor; -use std::path::Path; - -use image::{DynamicImage, GenericImage, ImageFormat, Rgba, RgbaImage}; -use platform_matting::{ - DEFAULT_IMAGESEG_ENDPOINT, MattingClient, MattingConfig, SegmentCommonImageRequest, - SegmentReturnForm, -}; - -fn load_env_files() { - for candidate in [ - ".env", - ".env.local", - ".env.secrets.local", - "../.env", - "../.env.local", - "../.env.secrets.local", - ] { - if Path::new(candidate).exists() { - let _ = dotenvy::from_path_override(candidate); - } - } -} - -fn encode_png(image: &DynamicImage) -> Vec { - let mut buffer = Cursor::new(Vec::new()); - image - .write_to(&mut buffer, ImageFormat::Png) - .expect("PNG 编码应成功"); - buffer.into_inner() -} - -#[tokio::main] -async fn main() { - load_env_files(); - - let input_path = std::env::args().nth(1).unwrap_or_else(|| { - r"C:\Users\lingh\Downloads\gpt-image-2-elf-archer-2048x2048.png".to_string() - }); - let source = image::open(&input_path).expect("测试图片应可解码"); - - // 变体1：等比缩到 480（模拟 1:1 480p 视频帧，主体居中） - let centered = source.resize(480, 480, image::imageops::FilterType::CatmullRom); - - // 变体2：把 360 尺寸的主体贴到 640x480 画布左上角（模拟横版帧、主体偏移） - let small = source.resize(360, 360, image::imageops::FilterType::CatmullRom); - let corner_pixel = centered.to_rgba8().get_pixel(0, 0).0; - let mut offset_canvas = RgbaImage::from_pixel(640, 480, Rgba(corner_pixel)); - offset_canvas - .copy_from(&small.to_rgba8(), 20, 40) - .expect("贴图应成功"); - let offset = DynamicImage::ImageRgba8(offset_canvas); - - let matting_config = MattingConfig::new( - std::env::var("ALIYUN_IMAGESEG_ENDPOINT") - .unwrap_or_else(|_| DEFAULT_IMAGESEG_ENDPOINT.to_string()), - std::env::var("ALIBABA_CLOUD_ACCESS_KEY_ID").expect("需要 ALIBABA_CLOUD_ACCESS_KEY_ID"), - std::env::var("ALIBABA_CLOUD_ACCESS_KEY_SECRET") - .expect("需要 ALIBABA_CLOUD_ACCESS_KEY_SECRET"), - ) - .expect("抠图配置应有效"); - let client = MattingClient::new(matting_config).expect("客户端应可构建"); - let http_client = reqwest::Client::new(); - - for (label, variant) in [("居中480x480", ¢ered), ("偏移640x480", &offset)] { - let bytes = encode_png(variant); - let temp_url = client - .upload_temp_image(bytes, &format!("probe-{label}.png"), "image/png") - .await - .expect("上传临时桶应成功"); - - for return_form in [None, Some(SegmentReturnForm::Crop), Some(SegmentReturnForm::Mask)] { - let result = client - .segment_common_image(SegmentCommonImageRequest { - image_url: temp_url.clone(), - return_form, - }) - .await - .expect("抠图调用应成功"); - let output_bytes = http_client - .get(&result.image_url) - .send() - .await - .expect("下载结果应成功") - .bytes() - .await - .expect("读取结果字节应成功"); - let output = image::load_from_memory(&output_bytes).expect("结果应可解码"); - println!( - "{label} 输入 {}x{} -> {} 输出 {}x{} (alpha通道: {})", - variant.width(), - variant.height(), - return_form.map_or("默认(不传)".to_string(), |f| format!("{f:?}")), - output.width(), - output.height(), - output.color().has_alpha() - ); - } - } -} -- 2.52.0 From 77d33af6519dac454b684c4cc8165d4100b2227a Mon Sep 17 00:00:00 2001 From: Linghong Date: Wed, 8 Jul 2026 04:10:14 +0000 Subject: [PATCH 04/41] =?UTF-8?q?=E6=B8=85=E7=90=86=E5=89=8D=E7=AB=AF?= =?UTF-8?q?=E6=8A=A0=E7=BB=BF=E4=B8=8E=20qwenSprite=20=E6=AD=BB=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 删除无调用方的前端像素抠绿链（chromaKey.ts 及 characterAssetWorkflowModel 中的视频采帧/抠绿函数） - 删除已被 server-rs 提示词实现取代的 qwenSprite 共享骨架 Co-Authored-By: Claude Fable 5 --- packages/shared/src/assets/chromaKey.test.ts | 67 --- packages/shared/src/assets/chromaKey.ts | 478 ------------------ packages/shared/src/assets/qwenSprite.ts | 1 - packages/shared/src/index.ts | 1 - packages/shared/src/prompts/qwenSprite.ts | 176 ------- .../characterAssetWorkflowModel.ts | 208 -------- 6 files changed, 931 deletions(-) delete mode 100644 packages/shared/src/assets/chromaKey.test.ts delete mode 100644 packages/shared/src/assets/chromaKey.ts delete mode 100644 packages/shared/src/assets/qwenSprite.ts delete mode 100644 packages/shared/src/prompts/qwenSprite.ts diff --git a/packages/shared/src/assets/chromaKey.test.ts b/packages/shared/src/assets/chromaKey.test.ts deleted file mode 100644 index 125bb5393..000000000 --- a/packages/shared/src/assets/chromaKey.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { removeBackgroundFromRgba } from './chromaKey'; - -function createSolidRgbaBuffer({ - width, - height, - color, -}: { - width: number; - height: number; - color: [number, number, number, number]; -}) { - const pixels = new Uint8ClampedArray(width * height * 4); - for (let index = 0; index < width * height; index += 1) { - pixels.set(color, index * 4); - } - return pixels; -} - -function setPixel( - pixels: Uint8ClampedArray, - width: number, - x: number, - y: number, - color: [number, number, number, number], -) { - pixels.set(color, (y * width + x) * 4); -} - -function alphaAt( - pixels: Uint8ClampedArray, - width: number, - x: number, - y: number, -) { - return pixels[(y * width + x) * 4 + 3] ?? 0; -} - -describe('chromaKey', () => { - it('removes near-white canvas background without breaking an enclosed white character', () => { - const width = 9; - const height = 9; - const pixels = createSolidRgbaBuffer({ - width, - height, - color: [250, 250, 250, 255], - }); - - for (let y = 2; y <= 6; y += 1) { - for (let x = 2; x <= 6; x += 1) { - setPixel(pixels, width, x, y, [92, 80, 72, 255]); - } - } - for (let y = 3; y <= 5; y += 1) { - for (let x = 3; x <= 5; x += 1) { - setPixel(pixels, width, x, y, [246, 246, 244, 255]); - } - } - - expect(removeBackgroundFromRgba(pixels, width, height)).toBe(true); - - expect(alphaAt(pixels, width, 0, 0)).toBe(0); - expect(alphaAt(pixels, width, 4, 4)).toBe(255); - expect(alphaAt(pixels, width, 2, 2)).toBeGreaterThan(200); - }); -}); diff --git a/packages/shared/src/assets/chromaKey.ts b/packages/shared/src/assets/chromaKey.ts deleted file mode 100644 index 7b517cc66..000000000 --- a/packages/shared/src/assets/chromaKey.ts +++ /dev/null @@ -1,478 +0,0 @@ -export type MutableRgbaBuffer = Uint8Array | Uint8ClampedArray; - -const SOFT_EDGE_ALPHA_THRESHOLD = 224; -const FOREGROUND_NEIGHBOR_ALPHA_THRESHOLD = 96; - -function clamp01(value: number) { - return Math.max(0, Math.min(1, value)); -} - -function lerp(from: number, to: number, t: number) { - return from + (to - from) * clamp01(t); -} - -function computeGreenBackgroundScore( - red: number, - green: number, - blue: number, - alpha: number, -) { - if (alpha === 0) { - return 1; - } - - const greenLead = green - Math.max(red, blue); - if (green < 52 || greenLead <= 8) { - return 0; - } - - const greenRatio = green / Math.max(1, red + blue); - if (greenRatio <= 0.52) { - return 0; - } - - return clamp01( - ((green - 52) / 168) * 0.22 + - ((greenLead - 8) / 96) * 0.53 + - ((greenRatio - 0.52) / 0.82) * 0.25, - ); -} - -function computeWhiteBackgroundScore( - red: number, - green: number, - blue: number, - alpha: number, -) { - if (alpha === 0) { - return 1; - } - - const maxChannel = Math.max(red, green, blue); - const minChannel = Math.min(red, green, blue); - const average = (red + green + blue) / 3; - if (average < 188 || minChannel < 168) { - return 0; - } - - const spread = maxChannel - minChannel; - const neutrality = 1 - clamp01((spread - 6) / 34); - const brightness = clamp01((average - 188) / 55); - const floor = clamp01((minChannel - 168) / 60); - return clamp01(neutrality * (brightness * 0.85 + floor * 0.15)); -} - -function collectForegroundNeighborColor( - pixels: MutableRgbaBuffer, - width: number, - height: number, - x: number, - y: number, - backgroundMask: Uint8Array, - backgroundHints: Float32Array, -) { - let totalWeight = 0; - let totalRed = 0; - let totalGreen = 0; - let totalBlue = 0; - - for (let offsetY = -2; offsetY <= 2; offsetY += 1) { - for (let offsetX = -2; offsetX <= 2; offsetX += 1) { - if (offsetX === 0 && offsetY === 0) { - continue; - } - - const nextX = x + offsetX; - const nextY = y + offsetY; - if (nextX < 0 || nextX >= width || nextY < 0 || nextY >= height) { - continue; - } - - const nextPixelIndex = nextY * width + nextX; - if (backgroundMask[nextPixelIndex]) { - continue; - } - - if ((backgroundHints[nextPixelIndex] ?? 0) >= 0.18) { - continue; - } - - const nextOffset = nextPixelIndex * 4; - const nextAlpha = pixels[nextOffset + 3] ?? 0; - if (nextAlpha < FOREGROUND_NEIGHBOR_ALPHA_THRESHOLD) { - continue; - } - - const distance = Math.abs(offsetX) + Math.abs(offsetY); - const weight = - (nextAlpha / 255) * - (distance <= 1 ? 1.8 : distance === 2 ? 1.2 : 0.7); - - totalWeight += weight; - totalRed += (pixels[nextOffset] ?? 0) * weight; - totalGreen += (pixels[nextOffset + 1] ?? 0) * weight; - totalBlue += (pixels[nextOffset + 2] ?? 0) * weight; - } - } - - if (totalWeight <= 0) { - return null; - } - - return { - red: Math.round(totalRed / totalWeight), - green: Math.round(totalGreen / totalWeight), - blue: Math.round(totalBlue / totalWeight), - }; -} - -export function removeBackgroundFromRgba( - pixels: MutableRgbaBuffer, - width: number, - height: number, -) { - const pixelCount = width * height; - if (pixelCount <= 0) { - return false; - } - - const backgroundMask = new Uint8Array(pixelCount); - const greenScores = new Float32Array(pixelCount); - const whiteScores = new Float32Array(pixelCount); - const backgroundHints = new Float32Array(pixelCount); - const queue: number[] = []; - let queueIndex = 0; - let changed = false; - - for (let pixelIndex = 0; pixelIndex < pixelCount; pixelIndex += 1) { - const offset = pixelIndex * 4; - const red = pixels[offset] ?? 0; - const green = pixels[offset + 1] ?? 0; - const blue = pixels[offset + 2] ?? 0; - const alpha = pixels[offset + 3] ?? 0; - const greenScore = computeGreenBackgroundScore(red, green, blue, alpha); - const whiteScore = computeWhiteBackgroundScore(red, green, blue, alpha); - const transparencyHint = clamp01((56 - alpha) / 56) * 0.75; - - greenScores[pixelIndex] = greenScore; - whiteScores[pixelIndex] = whiteScore; - backgroundHints[pixelIndex] = Math.max( - greenScore, - whiteScore, - transparencyHint, - ); - } - - const trySeedBackground = (pixelIndex: number) => { - if (backgroundMask[pixelIndex]) { - return; - } - - const offset = pixelIndex * 4; - const alpha = pixels[offset + 3] ?? 0; - const strongCandidate = - alpha < 40 || - (greenScores[pixelIndex] ?? 0) > 0.12 || - (whiteScores[pixelIndex] ?? 0) > 0.32; - - if (!strongCandidate) { - return; - } - - backgroundMask[pixelIndex] = 1; - queue.push(pixelIndex); - }; - - for (let x = 0; x < width; x += 1) { - trySeedBackground(x); - trySeedBackground((height - 1) * width + x); - } - - for (let y = 1; y < height - 1; y += 1) { - trySeedBackground(y * width); - trySeedBackground(y * width + width - 1); - } - - while (queueIndex < queue.length) { - const pixelIndex = queue[queueIndex]!; - queueIndex += 1; - - const x = pixelIndex % width; - const y = Math.floor(pixelIndex / width); - const neighborIndexes = [ - x > 0 ? pixelIndex - 1 : -1, - x + 1 < width ? pixelIndex + 1 : -1, - y > 0 ? pixelIndex - width : -1, - y + 1 < height ? pixelIndex + width : -1, - ]; - - for (const nextPixelIndex of neighborIndexes) { - if (nextPixelIndex < 0 || backgroundMask[nextPixelIndex]) { - continue; - } - - const nextOffset = nextPixelIndex * 4; - const nextAlpha = pixels[nextOffset + 3] ?? 0; - const nextGreenScore = greenScores[nextPixelIndex] ?? 0; - const nextWhiteScore = whiteScores[nextPixelIndex] ?? 0; - const nextHint = backgroundHints[nextPixelIndex] ?? 0; - const reachableSoftEdge = - nextHint > 0.08 && - nextAlpha < SOFT_EDGE_ALPHA_THRESHOLD && - (nextGreenScore > 0.04 || nextWhiteScore > 0.08 || nextAlpha < 180); - - if ( - nextAlpha < 40 || - nextGreenScore > 0.12 || - nextWhiteScore > 0.32 || - reachableSoftEdge - ) { - backgroundMask[nextPixelIndex] = 1; - queue.push(nextPixelIndex); - } - } - } - - for (let iteration = 0; iteration < 2; iteration += 1) { - const expandedMask = new Uint8Array(backgroundMask); - - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - const pixelIndex = y * width + x; - if (expandedMask[pixelIndex]) { - continue; - } - - const alpha = pixels[pixelIndex * 4 + 3] ?? 0; - const hint = backgroundHints[pixelIndex] ?? 0; - if (alpha >= SOFT_EDGE_ALPHA_THRESHOLD || hint <= 0.06) { - continue; - } - - let adjacentBackgroundCount = 0; - for (let offsetY = -1; offsetY <= 1; offsetY += 1) { - for (let offsetX = -1; offsetX <= 1; offsetX += 1) { - if (offsetX === 0 && offsetY === 0) { - continue; - } - - const nextX = x + offsetX; - const nextY = y + offsetY; - if (nextX < 0 || nextX >= width || nextY < 0 || nextY >= height) { - continue; - } - - if (backgroundMask[nextY * width + nextX]) { - adjacentBackgroundCount += 1; - } - } - } - - if ( - adjacentBackgroundCount >= 2 || - (adjacentBackgroundCount >= 1 && hint > 0.18) - ) { - expandedMask[pixelIndex] = 1; - } - } - } - - backgroundMask.set(expandedMask); - } - - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - const pixelIndex = y * width + x; - if (!backgroundMask[pixelIndex]) { - continue; - } - - const offset = pixelIndex * 4; - const alpha = pixels[offset + 3] ?? 0; - if (alpha === 0) { - continue; - } - - const matteScore = Math.max( - backgroundHints[pixelIndex] ?? 0, - greenScores[pixelIndex] ?? 0, - whiteScores[pixelIndex] ?? 0, - ); - - let foregroundSupport = 0; - for (let offsetY = -1; offsetY <= 1; offsetY += 1) { - for (let offsetX = -1; offsetX <= 1; offsetX += 1) { - if (offsetX === 0 && offsetY === 0) { - continue; - } - - const nextX = x + offsetX; - const nextY = y + offsetY; - if (nextX < 0 || nextX >= width || nextY < 0 || nextY >= height) { - continue; - } - - const nextPixelIndex = nextY * width + nextX; - if (backgroundMask[nextPixelIndex]) { - continue; - } - - const nextAlpha = pixels[nextPixelIndex * 4 + 3] ?? 0; - if (nextAlpha >= FOREGROUND_NEIGHBOR_ALPHA_THRESHOLD) { - foregroundSupport += 1; - } - } - } - - let nextAlpha = alpha; - if (matteScore > 0.9 || foregroundSupport === 0) { - nextAlpha = 0; - } else if (matteScore > 0.72 && foregroundSupport <= 1) { - nextAlpha = Math.min(alpha, Math.round(alpha * 0.08)); - } else { - nextAlpha = Math.min( - alpha, - Math.round(alpha * Math.max(0.08, 1 - matteScore * 0.95)), - ); - } - - if (foregroundSupport >= 3 && matteScore < 0.55) { - nextAlpha = Math.max(nextAlpha, Math.round(alpha * 0.22)); - } - - if (nextAlpha < 10) { - nextAlpha = 0; - } - - if (nextAlpha !== alpha) { - pixels[offset + 3] = nextAlpha; - changed = true; - } - } - } - - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - const pixelIndex = y * width + x; - const offset = pixelIndex * 4; - const alpha = pixels[offset + 3] ?? 0; - if (alpha === 0) { - continue; - } - - let touchesTransparentEdge = false; - for (let offsetY = -1; offsetY <= 1; offsetY += 1) { - for (let offsetX = -1; offsetX <= 1; offsetX += 1) { - if (offsetX === 0 && offsetY === 0) { - continue; - } - - const nextX = x + offsetX; - const nextY = y + offsetY; - if (nextX < 0 || nextX >= width || nextY < 0 || nextY >= height) { - touchesTransparentEdge = true; - continue; - } - - const nextPixelIndex = nextY * width + nextX; - if ( - backgroundMask[nextPixelIndex] || - (pixels[nextPixelIndex * 4 + 3] ?? 0) < 16 - ) { - touchesTransparentEdge = true; - } - } - } - - if (!touchesTransparentEdge) { - continue; - } - - const greenScore = greenScores[pixelIndex] ?? 0; - const whiteScore = whiteScores[pixelIndex] ?? 0; - const contamination = Math.max( - greenScore, - whiteScore, - backgroundMask[pixelIndex] ? 0.35 : 0, - alpha < 220 ? ((220 - alpha) / 220) * 0.25 : 0, - ); - - if (contamination < 0.06) { - continue; - } - - let red = pixels[offset] ?? 0; - let green = pixels[offset + 1] ?? 0; - let blue = pixels[offset + 2] ?? 0; - const sample = collectForegroundNeighborColor( - pixels, - width, - height, - x, - y, - backgroundMask, - backgroundHints, - ); - const blend = clamp01( - Math.max(contamination * 0.82, touchesTransparentEdge ? 0.22 : 0), - ); - - if (sample) { - red = Math.round(lerp(red, sample.red, blend)); - green = Math.round(lerp(green, sample.green, blend)); - blue = Math.round(lerp(blue, sample.blue, blend)); - - if (greenScore > 0.04) { - green = Math.min(green, sample.green + 18); - } - - if (whiteScore > 0.1) { - red = Math.min(red, sample.red + 26); - green = Math.min(green, sample.green + 26); - blue = Math.min(blue, sample.blue + 26); - } - } else { - if (greenScore > 0.04) { - green = Math.max( - Math.max(red, blue), - Math.round(green - (green - Math.max(red, blue)) * 0.78), - ); - } - - if (whiteScore > 0.12) { - const spread = Math.max(red, green, blue) - Math.min(red, green, blue); - if (spread < 20) { - const tonedValue = Math.round(((red + green + blue) / 3) * 0.88); - red = Math.min(red, tonedValue); - green = Math.min(green, tonedValue); - blue = Math.min(blue, tonedValue); - } - } - } - - let nextAlpha = alpha; - const edgeFade = Math.max(greenScore * 0.35, whiteScore * 0.28); - if (edgeFade > 0.08) { - nextAlpha = Math.min(alpha, Math.round(alpha * (1 - edgeFade))); - if (nextAlpha < 10) { - nextAlpha = 0; - } - } - - if ( - red !== (pixels[offset] ?? 0) || - green !== (pixels[offset + 1] ?? 0) || - blue !== (pixels[offset + 2] ?? 0) || - nextAlpha !== alpha - ) { - pixels[offset] = red; - pixels[offset + 1] = green; - pixels[offset + 2] = blue; - pixels[offset + 3] = nextAlpha; - changed = true; - } - } - } - - return changed; -} diff --git a/packages/shared/src/assets/qwenSprite.ts b/packages/shared/src/assets/qwenSprite.ts deleted file mode 100644 index 67957218c..000000000 --- a/packages/shared/src/assets/qwenSprite.ts +++ /dev/null @@ -1 +0,0 @@ -export * from '../prompts/qwenSprite.js'; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index cc84c6f11..23bbb0c45 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1,4 +1,3 @@ -export * from './assets/qwenSprite'; export * from './contracts/auth'; export type * from './contracts/bigFish'; export * from './contracts/common'; diff --git a/packages/shared/src/prompts/qwenSprite.ts b/packages/shared/src/prompts/qwenSprite.ts deleted file mode 100644 index 560a22b45..000000000 --- a/packages/shared/src/prompts/qwenSprite.ts +++ /dev/null @@ -1,176 +0,0 @@ -/** - * 共享 sprite / 角色资产正式 prompt 模板。 - * - * 这份脚本属于“正式模型 prompt 模板层”，不负责从角色卡里挑默认文本。 - * 它的定位是： - * - 给后端角色主图生成链路提供标准主图 prompt 骨架 - * - 给后端角色动作视频生成链路提供标准动作 prompt 骨架 - * - * 当前角色资产主链中的关系是： - * 1. 前端或 Rust 后端先拿到一段较短的描述文本 - * 2. 当前角色资产链路调用本文件 buildMasterPrompt / buildVideoActionPrompt - * 把短描述扩成正式给模型吃的 prompt - * - * 因此本文件不要承载“角色卡字段挑选”或“UI 默认值”职责， - * 只维护共享的正式 prompt 骨架与动作模板。 - */ -export type QwenSpriteActionTemplateId = - | 'idle' - | 'run' - | 'attack_slash' - | 'hurt' - | 'die'; - -export type QwenSpriteActionTemplate = { - id: QwenSpriteActionTemplateId; - label: string; - loop: boolean; - defaultFps: number; - bodyTravel: string; - weaponRule: string; - sequenceLines: [string, string, string, string]; - ending: string; -}; - -export const QWEN_SPRITE_ACTION_TEMPLATES: QwenSpriteActionTemplate[] = [ - { - id: 'idle', - label: '待机循环', - loop: true, - defaultFps: 8, - bodyTravel: '原地', - weaponRule: '武器始终在主手，位置稳定', - sequenceLines: [ - '1-4 帧：稳定站姿，轻微呼吸起伏', - '5-8 帧：胸腔与肩膀轻微抬起，衣摆极轻微变化', - '9-12 帧：呼气回落，重心恢复', - '13-16 帧：逐渐回到与首帧接近的站姿', - ], - ending: '第 16 帧自然衔接第 1 帧', - }, - { - id: 'run', - label: '奔跑循环', - loop: true, - defaultFps: 12, - bodyTravel: '小幅前移但角色中心基本固定', - weaponRule: '武器始终在主手，不换手', - sequenceLines: [ - '1-4 帧：右腿前摆，左腿后蹬，身体略前倾', - '5-8 帧：双腿交叉经过身体下方，手臂反向摆动', - '9-12 帧：左腿前摆，右腿后蹬，继续前倾', - '13-16 帧：完成另一半跑步循环并回到可接第 1 帧的状态', - ], - ending: '第 16 帧能无缝接回第 1 帧', - }, - { - id: 'attack_slash', - label: '横斩攻击', - loop: false, - defaultFps: 12, - bodyTravel: '中幅前探', - weaponRule: '右手持武器，始终右手，不换手', - sequenceLines: [ - '1-4 帧：轻微收身蓄力，武器向后收', - '5-8 帧：重心前压，挥击开始', - '9-12 帧：斩击达到最大幅度，动作力量最强', - '13-16 帧：顺势收招，回到可接下一动作的稳定姿态', - ], - ending: '第 16 帧停在收招后稳定姿态', - }, - { - id: 'hurt', - label: '受击后仰', - loop: false, - defaultFps: 10, - bodyTravel: '原地或极小后仰', - weaponRule: '武器不要脱手，不要换手', - sequenceLines: [ - '1-4 帧：突然受击，头肩后仰', - '5-8 帧：身体失衡最明显', - '9-12 帧：手臂和武器随惯性摆动', - '13-16 帧：逐渐恢复到勉强站稳的姿态', - ], - ending: '第 16 帧能接回 idle 或下一个动作', - }, - { - id: 'die', - label: '倒地死亡', - loop: false, - defaultFps: 8, - bodyTravel: '明显倒地位移', - weaponRule: '武器不可瞬间消失', - sequenceLines: [ - '1-4 帧：受创失衡，重心被打断', - '5-8 帧：身体明显下坠或后仰', - '9-12 帧：倒地过程完成，动作幅度最大', - '13-16 帧：停在清晰的终止姿态', - ], - ending: '第 16 帧停在死亡结束姿态，不需要循环', - }, -]; - -const BODY_RATIO_TEXT = - '横版像素动作角色体型，头身比优先控制在 3 到 4 头身，头部只允许略大于写实比例，保留清楚的头、躯干、双臂和双腿轮廓，不要退化成软萌 Q版大头贴或儿童绘本比例。'; -const PIXEL_STYLE_TEXT = - '明确的像素动作角色设定稿气质，整体按像素游戏角色设计方向组织，使用深色清楚轮廓、稳定剪影、有限大色块和硬朗边缘，不要柔和厚涂插画感，发型、服装、配饰优先形成醒目可读的像素级识别点，身体始终朝右，适合横版动作 sprite 资产。'; - -export function getActionTemplateById(id: QwenSpriteActionTemplateId) { - return ( - QWEN_SPRITE_ACTION_TEMPLATES.find((template) => template.id === id) ?? - QWEN_SPRITE_ACTION_TEMPLATES[0] - ); -} - -/** - * 正式角色主图 prompt 骨架。 - * - * 输入应该是一段已经整理好的角色摘要或视觉描述， - * 这里会把它嵌进统一的 sprite 资产约束中， - * 输出真正发给图像模型的完整 prompt。 - */ -export function buildMasterPrompt(characterBrief: string) { - return [ - '单人，2D 横版游戏角色标准设定图，主体完整可见，底部轮廓完整，身体比例稳定，轮廓清楚，适合后续制作 sprite sheet 动画。', - `视角要求：角色采用横版动作素材常用的右向斜侧身站姿，身体整体朝右，但保留少量正面信息，能读到面部轮廓与胸肩结构，不是完全 90 度纯右视图，也不是正面立绘。`, - `主体要求：画面中只保留单个角色主体，不要额外人物、动物、召唤物、载具或陪体。`, - `画面要求：1:1 正方形画布，画面中心构图，角色主体完整置于画面中央，不要裁切主体顶部和底部，不要镜头透视，不要特写。背景固定为单一纯绿色 #00FF00 / RGB(0,255,0) 绿幕，只作为抠像底色，不出现建筑、室内布景、风景、地面道具、漂浮物、烟雾叙事元素或其他角色以外的场景内容。`, - `风格要求：${BODY_RATIO_TEXT} ${PIXEL_STYLE_TEXT} 高可读性游戏角色设定图，形体清晰，服装层次明确，优先体现像素动作角色感而不是软萌 Q版插画感，便于后续连续动作生成。`, - '请先拆解设定中的“身份词、主题词、身体结构词”。如果文字设定没有明确要求非人身体结构，默认优先使用参考图对应的人类或类人动作角色骨架，保持清楚的头、躯干、手臂和双腿轮廓，只有当文字设定明确要求非人结构时，才改为对应非人身体。', - '主题词默认只作用在角色自身的服装剪裁、材质、纹样、饰品、发光细节上，不要把主题词自动扩写成背景建筑、自然场景、漂浮装饰或额外环境物件。', - '视觉优先级应当是：身体结构词第一，身份词第二，主题词第三。没有明确身体结构词时，默认用人形拟人化表现，再把主题词转译成服装和装饰。', - characterBrief.trim(), - ] - .filter(Boolean) - .join('\n'); -} - -/** - * 正式动作视频 prompt 骨架。 - * - * 输入应该是已经整理好的动作细节与角色摘要， - * 这里负责统一拼装成 sprite 动作生成所需的正式 prompt， - * 包括视角、像素风格、动作模板、绿幕约束等。 - */ -export function buildVideoActionPrompt(options: { - actionTemplate: QwenSpriteActionTemplate; - actionDetailText: string; - useChromaKey: boolean; - characterBrief: string; -}) { - return [ - `单人全身角色动作视频，动作英文名是 ${options.actionTemplate.id}。`, - `角色固定为图1同一角色，保持右向斜侧身动作视角，镜头稳定，轮廓清晰，不要退化成完全 90 度纯右视图。`, - `视角要求：角色采用横版动作素材常用的右向斜侧身站姿，身体整体朝右，但保留少量正面信息，能读到面部轮廓与胸肩结构，不是完全 90 度纯右视图，也不是正面立绘。`, - `主体要求：画面中只保留单个角色主体，不要额外人物、动物、召唤物、载具或陪体。`, - `画面要求：1:1 正方形画布，画面中心构图，角色主体完整置于画面中央，不要裁切主体顶部和底部，不要镜头透视，不要特写。背景固定为单一纯绿色 #00FF00 / RGB(0,255,0) 绿幕，只作为抠像底色，不出现建筑、室内布景、风景、地面道具、漂浮物、烟雾叙事元素或其他角色以外的场景内容。`, - `风格要求：${BODY_RATIO_TEXT} ${PIXEL_STYLE_TEXT} 高可读性游戏角色设定图，偏像素动画前置设计稿，形体清晰，服装层次明确，道具/权杖/武器如有则存在关系合理，优先保证像素动作角色感，不要退化成只剩 Q 版比例的普通插画，便于后续连续动作生成。`, - `动作结构：${options.actionTemplate.sequenceLines.join('；')}。结尾要求：${options.actionTemplate.ending}。`, - options.useChromaKey - ? '背景为单一纯绿色 #00FF00 / RGB(0,255,0) 绿幕，无其他人物和场景元素，方便后期抽帧与抠像。' - : '背景简洁纯净，无复杂场景。', - `动作补充细节：${options.actionDetailText.trim() || '保持动作清晰、节奏明确、适合后续抽帧为 sprite sheet。'}`, - `角色设定：${options.characterBrief.trim()}`, - '目标是后续抽帧为横版动作游戏精灵表，因此不要镜头切换，不要景别变化，不要角色漂移。', - ].join(' '); -} diff --git a/src/components/asset-studio/characterAssetWorkflowModel.ts b/src/components/asset-studio/characterAssetWorkflowModel.ts index a37239d2c..b9e945bd2 100644 --- a/src/components/asset-studio/characterAssetWorkflowModel.ts +++ b/src/components/asset-studio/characterAssetWorkflowModel.ts @@ -1,4 +1,3 @@ -import { removeBackgroundFromRgba } from '../../../packages/shared/src/assets/chromaKey'; import { AnimationState, type Character, @@ -453,19 +452,6 @@ export function loadImageFromSource(source: string) { }); } -function loadVideoFromSource(source: string) { - return new Promise((resolve, reject) => { - const video = document.createElement('video'); - video.crossOrigin = 'anonymous'; - video.preload = 'auto'; - video.muted = true; - video.playsInline = true; - video.onloadeddata = () => resolve(video); - video.onerror = () => reject(new Error(`加载视频失败：${source}`)); - video.src = source; - }); -} - function createCanvas(width: number, height: number) { const canvas = document.createElement('canvas'); canvas.width = width; @@ -708,200 +694,6 @@ export async function buildAnimationClipFromMaster( } satisfies DraftAnimationClip; } -function applyGreenScreenAlpha( - context: CanvasRenderingContext2D, - width: number, - height: number, -) { - const imageData = context.getImageData(0, 0, width, height); - removeBackgroundFromRgba(imageData.data, width, height); - - context.putImageData(imageData, 0, 0); -} - -async function normalizeFrameSourceToDataUrl( - frameSource: string, - options: { - frameWidth: number; - frameHeight: number; - applyChromaKey: boolean; - }, -) { - const image = await loadImageFromSource(frameSource); - const { canvas, context } = createCanvas( - options.frameWidth, - options.frameHeight, - ); - context.clearRect(0, 0, canvas.width, canvas.height); - drawContainedImage(context, image, { - width: canvas.width, - height: canvas.height, - }); - - if (options.applyChromaKey) { - applyGreenScreenAlpha(context, canvas.width, canvas.height); - } - - return canvas.toDataURL('image/png'); -} - -export async function normalizeMasterVisualSourceToDataUrl( - source: string, - options: { - applyChromaKey?: boolean; - } = {}, -) { - const image = await loadImageFromSource(source); - const { canvas, context } = createCanvas( - MASTER_VISUAL_WIDTH, - MASTER_VISUAL_HEIGHT, - ); - context.clearRect(0, 0, canvas.width, canvas.height); - drawContainedImage(context, image, { - width: canvas.width, - height: canvas.height, - }); - - if (options.applyChromaKey !== false) { - applyGreenScreenAlpha(context, canvas.width, canvas.height); - } - - return { - dataUrl: canvas.toDataURL('image/png'), - width: canvas.width, - height: canvas.height, - }; -} - -function seekVideo(video: HTMLVideoElement, targetTime: number) { - return new Promise((resolve, reject) => { - if (Math.abs(video.currentTime - targetTime) < 0.001) { - window.requestAnimationFrame(() => resolve()); - return; - } - - const handleSeeked = () => { - cleanup(); - resolve(); - }; - const handleError = () => { - cleanup(); - reject(new Error('视频定位失败')); - }; - const cleanup = () => { - video.removeEventListener('seeked', handleSeeked); - video.removeEventListener('error', handleError); - }; - - video.addEventListener('seeked', handleSeeked, { once: true }); - video.addEventListener('error', handleError, { once: true }); - video.currentTime = Math.max(0, targetTime); - }); -} - -export async function buildAnimationClipFromImageSources( - sources: string[], - options: { - animation: AnimationState; - fps: number; - loop: boolean; - frameWidth?: number; - frameHeight?: number; - applyChromaKey?: boolean; - }, -) { - const frameWidth = options.frameWidth ?? GENERATED_FRAME_WIDTH; - const frameHeight = options.frameHeight ?? GENERATED_FRAME_HEIGHT; - const frames = await Promise.all( - sources.map((source) => - normalizeFrameSourceToDataUrl(source, { - frameWidth, - frameHeight, - applyChromaKey: options.applyChromaKey ?? false, - }), - ), - ); - - return { - animation: options.animation, - frames, - fps: Math.max(1, options.fps), - loop: options.loop, - frameWidth, - frameHeight, - } satisfies DraftAnimationClip; -} - -export async function buildAnimationClipFromVideoSource( - videoSource: string, - options: { - animation: AnimationState; - fps: number; - loop: boolean; - frameCount?: number; - frameWidth?: number; - frameHeight?: number; - applyChromaKey?: boolean; - sampleStartRatio?: number; - sampleEndRatio?: number; - }, -) { - const video = await loadVideoFromSource(videoSource); - const frameWidth = options.frameWidth ?? GENERATED_FRAME_WIDTH; - const frameHeight = options.frameHeight ?? GENERATED_FRAME_HEIGHT; - const duration = - Number.isFinite(video.duration) && video.duration > 0 ? video.duration : 1; - const derivedFrameCount = Math.max( - 2, - options.frameCount ?? Math.round(duration * Math.max(1, options.fps)), - ); - const sampleStartRatio = Math.min( - 0.85, - Math.max(0, options.sampleStartRatio ?? 0), - ); - const sampleEndRatio = Math.min( - 1, - Math.max(sampleStartRatio + 0.05, options.sampleEndRatio ?? 1), - ); - const sampleWindowDuration = duration * (sampleEndRatio - sampleStartRatio); - const { canvas, context } = createCanvas(frameWidth, frameHeight); - const frames: string[] = []; - - for (let frameIndex = 0; frameIndex < derivedFrameCount; frameIndex += 1) { - const progress = options.loop - ? frameIndex / derivedFrameCount - : frameIndex / Math.max(1, derivedFrameCount - 1); - const targetTime = Math.min( - duration - 0.001, - duration * sampleStartRatio + sampleWindowDuration * progress, - ); - - await seekVideo(video, targetTime); - - context.clearRect(0, 0, canvas.width, canvas.height); - drawContainedSource(context, video, video.videoWidth, video.videoHeight, { - width: canvas.width, - height: canvas.height, - }); - - if (options.applyChromaKey) { - applyGreenScreenAlpha(context, canvas.width, canvas.height); - } - - frames.push(canvas.toDataURL('image/png')); - } - - return { - animation: options.animation, - frames, - fps: Math.max(1, options.fps), - loop: options.loop, - frameWidth, - frameHeight, - previewVideoPath: videoSource, - } satisfies DraftAnimationClip; -} - async function buildReferenceVideoFromFrameSources( frameSources: string[], options: { -- 2.52.0 From 923993ea6cfa2a26c086fa92353890f961c7d53c Mon Sep 17 00:00:00 2001 From: Linghong Date: Wed, 8 Jul 2026 13:37:44 +0000 Subject: [PATCH 05/41] =?UTF-8?q?=E8=83=8C=E6=99=AF=E8=89=B2=E5=86=B3?= =?UTF-8?q?=E7=AD=96=E5=8D=87=E7=BA=A7=EF=BC=9A=E8=A7=86=E8=A7=89=E9=80=89?= =?UTF-8?q?=E8=89=B2=20+=20=E7=9A=AE=E8=82=A4=E7=A1=AC=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=20+=20=E6=BA=90=E5=9B=BE=E5=90=88=E6=88=90=20+=20=E7=BB=93?= =?UTF-8?q?=E6=9E=84=E5=8C=96=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复动作视频背景色的三类真实事故并加确定性防线： 1. 视觉背景色决策：让 LLM 看参考图选背景色（gpt-4o-mini 视觉模型， 路由到 VectorEngine 视觉客户端），修复原纯文本盲选导致的"蓝撞蓝"。 2. 背景色候选硬过滤器（新增 editor_screen_background_filter）： 按参考图前景配色算 Lab 危险质量 + 皮肤专属三判据（ΔE 距离、 色调投影、RGB 分离）剔除撞色候选，LLM 只在安全集合里审美选。 露肤角色自动收敛到冷区。新增"灰竹绿"候选补齐安全弧绿色段。 3. 源图合成到背景色：透明源图先填成选定实色再发给 Ark，视频背景 确定性等于抠图键色，不再赌模型服从提示词（修复背景变白）。 4. screenColorHex 结构化记录：四条链路统一把实际背景色写进 generation_inputs，并修复生图链路"抠图背景色"字段从未生效的旧逻辑。 Co-Authored-By: Claude Opus 4.8 --- .../src/character_animation_assets.rs | 157 ++++- .../api-server/src/editor_green_screen.rs | 15 +- .../crates/api-server/src/editor_project.rs | 101 ++- .../src/editor_screen_background_decision.rs | 233 ++++++- .../src/editor_screen_background_filter.rs | 611 ++++++++++++++++++ .../api-server/src/llm_model_routing.rs | 2 + server-rs/crates/api-server/src/main.rs | 1 + 7 files changed, 1075 insertions(+), 45 deletions(-) create mode 100644 server-rs/crates/api-server/src/editor_screen_background_filter.rs diff --git a/server-rs/crates/api-server/src/character_animation_assets.rs b/server-rs/crates/api-server/src/character_animation_assets.rs index 49eb850fb..dd7899953 100644 --- a/server-rs/crates/api-server/src/character_animation_assets.rs +++ b/server-rs/crates/api-server/src/character_animation_assets.rs @@ -70,6 +70,7 @@ use crate::{ }, editor_project::{ EditorCanvasGeneratedLayerInput, PersistEditorGeneratedAssetRequest, + apply_editor_screen_background_decision_to_generation_inputs, build_editor_canvas_generated_layer_item, complete_editor_canvas_generation_with_items, persist_editor_generated_media_asset, }, @@ -108,6 +109,7 @@ const FIXED_ARK_CHARACTER_VIDEO_RESOLUTION: &str = "480p"; const FIXED_ARK_CHARACTER_VIDEO_RATIO: &str = "1:1"; const FIXED_ARK_CHARACTER_VIDEO_DURATION_SECONDS: u32 = 4; const ARK_VIDEO_TASK_POLL_INTERVAL_MS: u64 = 5_000; +const SOURCE_IMAGE_RESOLVE_TIMEOUT_MS: u64 = 30_000; const EDITOR_CHARACTER_ANIMATION_MODEL: &str = "seedance2.0-fast"; const EDITOR_CHARACTER_ANIMATION_ASSET_KIND: &str = "editor_character_animation"; const EDITOR_GREEN_SCREEN_SOURCE_ASSET_KIND: &str = "editor_green_screen_source"; @@ -635,19 +637,37 @@ pub(crate) async fn generate_editor_character_animation_for_owner( })), ) })?; - // 背景色决策：显式传入 → 继承源角色图（前端从图层 generationInputs 带入）→ LLM 自动决策 → 默认色。 + // 先解析源角色图：图生视频的主体颜色完全由源图决定，背景色决策必须看到这张图。 + let source_resolve_client = build_upstream_http_client(SOURCE_IMAGE_RESOLVE_TIMEOUT_MS) + .map_err(|error| character_animation_error_response(&request_context, error))?; + let source_data_url = resolve_media_source_as_data_url( + &state, + &source_resolve_client, + payload.source_image_src.trim(), + "sourceImageSrc", + ) + .await + .map_err(|error| character_animation_error_response(&request_context, error))?; + // 背景色决策：显式传入 → 视觉 LLM 按源图主体配色自动决策 → 默认色。 let screen_background_decision = resolve_editor_screen_background_color( state.llm_client(), + state.creative_agent_gpt5_client(), EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::CharacterAnimation, screen_color: payload.screen_color.clone(), prompt: payload.prompt_text.clone(), icon_descriptions: Vec::new(), reference_count: 1, + source_image_data_url: Some(source_data_url.clone()), }, ) .await .map_err(|error| character_animation_error_response(&request_context, error))?; + // 把实际选定的背景色落进资产记录，供后续以该资产为参考图时反查旧背景色。 + let generation_inputs = apply_editor_screen_background_decision_to_generation_inputs( + generation_inputs, + Some(&screen_background_decision), + ); let normalized = normalize_editor_character_animation_request_with_pricing( payload, &pricing, @@ -656,18 +676,16 @@ pub(crate) async fn generate_editor_character_animation_for_owner( .map_err(|error| character_animation_error_response(&request_context, error))?; let settings = require_editor_character_animation_settings(&state, &normalized) .map_err(|error| character_animation_error_response(&request_context, error))?; + // 透明源图先合成到选定背景色：让视频背景确定性地等于抠图键色，而不是赌模型服从提示词。 + let source_data_url = composite_source_image_onto_screen_color( + source_data_url.as_str(), + screen_background_decision.color, + ) + .unwrap_or(source_data_url); let extraction_settings = resolve_backend_frame_extraction_settings(&state); let http_client = build_upstream_http_client(settings.ark.request_timeout_ms) .map_err(|error| character_animation_error_response(&request_context, error))?; let task_id = generate_ai_task_id(current_utc_micros()); - let source_data_url = resolve_media_source_as_data_url( - &state, - &http_client, - normalized.source_image_src.as_str(), - "sourceImageSrc", - ) - .await - .map_err(|error| character_animation_error_response(&request_context, error))?; let result = execute_billable_asset_operation_with_cost( &state, @@ -2907,7 +2925,8 @@ fn normalize_editor_character_animation_request_with_pricing( "sourceLayerId 不能为空。", )); } - let source_image_src = trim_optional_text(Some(payload.source_image_src.as_str())) + // 源图在 handler 里已提前解析（决策要看图）；这里保留空值校验，队列路径入队前也走这份校验。 + trim_optional_text(Some(payload.source_image_src.as_str())) .ok_or_else(|| editor_character_animation_bad_request("sourceImageSrc 不能为空。"))?; let prompt_text = payload .prompt_text @@ -2943,7 +2962,6 @@ fn normalize_editor_character_animation_request_with_pricing( Ok(NormalizedEditorCharacterAnimationRequest { source_layer_id, - source_image_src, prompt, screen_color, resolution: resolution.to_string(), @@ -4067,6 +4085,68 @@ fn parse_media_data_url(value: &str) -> Option { }) } +/// 把带透明通道的源图合成到选定背景色上，返回新的 PNG data URL。 +/// 视频模型不支持 alpha：透明源图会被按白底理解，且图生视频里源图背景的视觉证据 +/// 权重高于文字提示，导致提示词指定的背景色被无视。先填成实色让源图和提示词一致， +/// 视频背景就是抠图键色。源图本身不透明或无法解码时返回 None（沿用原图）。 +fn composite_source_image_onto_screen_color( + source_data_url: &str, + screen_color: EditorScreenBackgroundColor, +) -> Option { + let payload = parse_media_data_url(source_data_url)?; + if !payload.mime_type.starts_with("image/") { + return None; + } + let Ok(image) = image::load_from_memory(payload.bytes.as_slice()) else { + tracing::warn!("character_animation_source_composite_skip_undecodable_image"); + return None; + }; + let rgba = image.to_rgba8(); + if rgba.pixels().all(|pixel| pixel.0[3] == u8::MAX) { + return None; + } + let background = [ + f32::from(screen_color.red), + f32::from(screen_color.green), + f32::from(screen_color.blue), + ]; + let mut composited = RgbaImage::new(rgba.width(), rgba.height()); + for (x, y, pixel) in rgba.enumerate_pixels() { + let [red, green, blue, alpha] = pixel.0; + let opacity = f32::from(alpha) / 255.0; + let blend = |source: u8, background: f32| -> u8 { + (f32::from(source) * opacity + background * (1.0 - opacity)).round() as u8 + }; + composited.put_pixel( + x, + y, + Rgba([ + blend(red, background[0]), + blend(green, background[1]), + blend(blue, background[2]), + u8::MAX, + ]), + ); + } + let mut bytes = Vec::new(); + if PngEncoder::new(&mut bytes) + .write_image( + composited.as_raw(), + composited.width(), + composited.height(), + ColorType::Rgba8.into(), + ) + .is_err() + { + tracing::warn!("character_animation_source_composite_skip_encode_failed"); + return None; + } + Some(format!( + "data:image/png;base64,{}", + encode_base64(bytes.as_slice()) + )) +} + fn resolve_object_key_from_legacy_path(value: &str, field: &str) -> Result { let trimmed = value.trim(); if trimmed.is_empty() { @@ -5144,7 +5224,6 @@ struct EditorVideoSettings { #[derive(Debug)] struct NormalizedEditorCharacterAnimationRequest { source_layer_id: String, - source_image_src: String, prompt: String, resolution: String, ratio: String, @@ -5247,6 +5326,56 @@ mod tests { assert!(parse_video_data_url("data:image/png;base64,aGVsbG8=").is_none()); } + fn encode_rgba_png_data_url(image: &RgbaImage) -> String { + let mut bytes = Vec::new(); + PngEncoder::new(&mut bytes) + .write_image( + image.as_raw(), + image.width(), + image.height(), + ColorType::Rgba8.into(), + ) + .expect("test image should encode"); + format!("data:image/png;base64,{}", encode_base64(bytes.as_slice())) + } + + #[test] + fn composites_transparent_source_onto_screen_color() { + let mut image = RgbaImage::from_pixel(8, 8, Rgba([0, 0, 0, 0])); + image.put_pixel(4, 4, Rgba([10, 20, 30, 255])); + let screen_color = crate::editor_green_screen::EDITOR_SCREEN_BACKGROUND_COLORS[0]; + + let composited = + composite_source_image_onto_screen_color(&encode_rgba_png_data_url(&image), screen_color) + .expect("透明源图应被合成"); + let payload = parse_media_data_url(&composited).expect("合成结果应是图片 data URL"); + let output = image::load_from_memory(payload.bytes.as_slice()) + .expect("合成结果应可解码") + .to_rgba8(); + + assert_eq!( + output.get_pixel(0, 0).0, + [screen_color.red, screen_color.green, screen_color.blue, 255], + "透明像素应填成背景色" + ); + assert_eq!(output.get_pixel(4, 4).0, [10, 20, 30, 255], "不透明像素应保持原色"); + } + + #[test] + fn opaque_source_image_skips_compositing() { + let image = RgbaImage::from_pixel(4, 4, Rgba([1, 2, 3, 255])); + let screen_color = crate::editor_green_screen::EDITOR_SCREEN_BACKGROUND_COLORS[0]; + + assert!( + composite_source_image_onto_screen_color( + &encode_rgba_png_data_url(&image), + screen_color + ) + .is_none(), + "不透明源图应沿用原图" + ); + } + #[test] fn sanitize_storage_segment_falls_back_for_chinese_label() { assert_eq!( @@ -5475,7 +5604,9 @@ mod tests { assert!(prompt.contains("参考图作为首帧和尾帧")); assert!(prompt.contains(color.hex)); assert!(prompt.contains("纯色背景必须平整无纹理、无渐变、无阴影")); - assert!(prompt.contains("角色主体不得带与背景色相同或相近的描边、投影或反光")); + assert!( + prompt.contains("角色主体及其服饰、道具的颜色必须与背景色明显区分，不得带与背景色相同或相近的描边、投影或反光") + ); assert!(prompt.contains("动作描述：\n行走两步后回到站姿。")); } diff --git a/server-rs/crates/api-server/src/editor_green_screen.rs b/server-rs/crates/api-server/src/editor_green_screen.rs index 6ea4be842..34fa243f0 100644 --- a/server-rs/crates/api-server/src/editor_green_screen.rs +++ b/server-rs/crates/api-server/src/editor_green_screen.rs @@ -16,7 +16,7 @@ pub(crate) struct EditorScreenBackgroundColor { pub(crate) blue: u8, } -pub(crate) const EDITOR_SCREEN_BACKGROUND_COLORS: [EditorScreenBackgroundColor; 11] = [ +pub(crate) const EDITOR_SCREEN_BACKGROUND_COLORS: [EditorScreenBackgroundColor; 12] = [ EditorScreenBackgroundColor { label: "浅雾蓝", hex: "#CFEFFF", @@ -94,6 +94,15 @@ pub(crate) const EDITOR_SCREEN_BACKGROUND_COLORS: [EditorScreenBackgroundColor; green: 247, blue: 240, }, + // 中明度低饱和柔雾绿：填补安全弧的绿色段。浅粉彩绿（淡薄荷绿）会被露肤角色的皮肤规则剔除， + // 故绿色改用中明度以过 RGB 分离（Rule 3）；色相远离皮肤橙调，过色调投影（Rule 2）。 + EditorScreenBackgroundColor { + label: "灰竹绿", + hex: "#A0BBA0", + red: 160, + green: 187, + blue: 160, + }, ]; pub(crate) fn default_editor_screen_background_color() -> EditorScreenBackgroundColor { @@ -127,9 +136,9 @@ pub(crate) fn parse_editor_screen_background_color( pub(crate) const EDITOR_GREEN_SCREEN_BACKGROUND_GUARDRAILS: &str = "纯色背景必须平整无纹理、无渐变、无阴影、无地面、无环境、无道具"; pub(crate) const EDITOR_GREEN_SCREEN_ASSET_GUARDRAILS: &str = - "素材自身不要出现与背景色相同或相近的描边、底板、投影或反光"; + "素材主体及其配色必须与背景色明显区分，不要出现与背景色相同或相近的描边、底板、投影或反光"; pub(crate) const EDITOR_GREEN_SCREEN_CHARACTER_GUARDRAILS: &str = - "角色主体不得带与背景色相同或相近的描边、投影或反光"; + "角色主体及其服饰、道具的颜色必须与背景色明显区分，不得带与背景色相同或相近的描边、投影或反光"; fn editor_screen_background_color_prompt(color: EditorScreenBackgroundColor) -> String { if color.hex == "#00FF00" { diff --git a/server-rs/crates/api-server/src/editor_project.rs b/server-rs/crates/api-server/src/editor_project.rs index 1e7d1dd66..9558c3f6d 100644 --- a/server-rs/crates/api-server/src/editor_project.rs +++ b/server-rs/crates/api-server/src/editor_project.rs @@ -1348,12 +1348,14 @@ pub(crate) async fn generate_editor_image_for_owner( Some( resolve_editor_screen_background_color( state.llm_client(), + state.creative_agent_gpt5_client(), EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::Character, screen_color: payload.screen_color.clone(), prompt: role_setting.clone(), icon_descriptions: Vec::new(), reference_count: payload.reference_image_srcs.as_ref().map_or(0, Vec::len), + source_image_data_url: None, }, ) .await?, @@ -1636,25 +1638,31 @@ fn editor_image_generation_billing_asset_kind(normalized_kind: Option<&str>) -> } } -fn apply_editor_screen_background_decision_to_generation_inputs( +pub(crate) fn apply_editor_screen_background_decision_to_generation_inputs( generation_inputs: Option, decision: Option<&EditorScreenBackgroundDecision>, ) -> Option { let Some(decision) = decision else { return generation_inputs; }; - let mut value = generation_inputs?; - let Some(fields) = value.get_mut("fields").and_then(Value::as_array_mut) else { - return Some(value); + let mut value = match generation_inputs { + Some(value) if value.is_object() => value, + Some(other) => return Some(other), + None => json!({ "fields": [], "references": [] }), }; let resolved_value = format_editor_screen_background_decision_input(decision); - for field in fields { - let title = field.get("title").and_then(Value::as_str); - if title == Some("抠图背景色") { + if let Some(fields) = value.get_mut("fields").and_then(Value::as_array_mut) { + if let Some(field) = fields + .iter_mut() + .find(|field| field.get("title").and_then(Value::as_str) == Some("抠图背景色")) + { field["value"] = Value::String(resolved_value); - break; + } else { + fields.push(json!({ "title": "抠图背景色", "value": resolved_value })); } } + // 结构化背景色，供旧背景色先验等程序化消费；展示字段的中文格式不可作解析依据。 + value["screenColorHex"] = Value::String(decision.color.hex.to_string()); Some(value) } @@ -2727,12 +2735,14 @@ pub(crate) async fn generate_editor_icon_spritesheet_for_owner( let icon_descriptions = normalize_icon_descriptions(payload.icon_descriptions)?; let screen_background_decision = resolve_editor_screen_background_color( state.llm_client(), + state.creative_agent_gpt5_client(), EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::IconSpritesheet, screen_color: payload.screen_color.clone(), prompt: icon_descriptions.join("\n"), icon_descriptions: icon_descriptions.clone(), reference_count: 1 + payload.reference_image_srcs.as_ref().map_or(0, Vec::len), + source_image_data_url: None, }, ) .await?; @@ -2994,6 +3004,7 @@ pub(crate) async fn extract_editor_ui_design_assets_for_owner( ) -> Result, AppError> { let screen_background_decision = resolve_editor_screen_background_color( state.llm_client(), + state.creative_agent_gpt5_client(), EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::UiDesignAssetExtraction, screen_color: payload.screen_color.clone(), @@ -3003,6 +3014,7 @@ pub(crate) async fn extract_editor_ui_design_assets_for_owner( ), icon_descriptions: Vec::new(), reference_count: 1 + payload.reference_image_srcs.as_ref().map_or(0, Vec::len), + source_image_data_url: None, }, ) .await?; @@ -5525,6 +5537,69 @@ mod tests { ); } + fn manual_screen_background_decision(hex: &str) -> EditorScreenBackgroundDecision { + EditorScreenBackgroundDecision { + color: parse_editor_screen_background_color(Some(hex)) + .expect("hex should be a known background color candidate"), + mode: crate::editor_screen_background_decision::EditorScreenBackgroundDecisionMode::Manual, + attempts: 0, + fallback: false, + } + } + + #[test] + fn screen_background_decision_generation_inputs_record_structured_hex() { + let decision = manual_screen_background_decision("#B0C2E0"); + + let updated = apply_editor_screen_background_decision_to_generation_inputs( + Some(json!({ + "fields": [{ "title": "角色设定", "value": "红发骑士" }], + "references": [], + })), + Some(&decision), + ) + .expect("generation inputs should be preserved"); + assert_eq!(updated["screenColorHex"], json!("#B0C2E0")); + assert_eq!( + updated["fields"], + json!([ + { "title": "角色设定", "value": "红发骑士" }, + { "title": "抠图背景色", "value": "浅钢蓝 #B0C2E0" }, + ]), + ); + + let created = + apply_editor_screen_background_decision_to_generation_inputs(None, Some(&decision)) + .expect("missing generation inputs should still record the decision"); + assert_eq!( + created, + json!({ + "fields": [{ "title": "抠图背景色", "value": "浅钢蓝 #B0C2E0" }], + "references": [], + "screenColorHex": "#B0C2E0", + }), + ); + } + + #[test] + fn screen_background_decision_generation_inputs_overwrite_existing_display_field() { + let decision = manual_screen_background_decision("#CFEFFF"); + + let updated = apply_editor_screen_background_decision_to_generation_inputs( + Some(json!({ + "fields": [{ "title": "抠图背景色", "value": "旧值" }], + "references": [], + })), + Some(&decision), + ) + .expect("generation inputs should be preserved"); + assert_eq!( + updated["fields"], + json!([{ "title": "抠图背景色", "value": "浅雾蓝 #CFEFFF" }]), + ); + assert_eq!(updated["screenColorHex"], json!("#CFEFFF")); + } + fn apply_editor_canvas_generation_completion( layers: Value, completion: &EditorCanvasGenerationCompletionRequest, @@ -6195,7 +6270,9 @@ mod tests { assert!(prompt.contains("生成游戏角色立绘")); assert!(prompt.contains("背景固定为单一纯色背景 暖浅桃色 #FFD6C2 / RGB(255,214,194)")); assert!(prompt.contains("纯色背景必须平整无纹理、无渐变、无阴影")); - assert!(prompt.contains("角色主体不得带与背景色相同或相近的描边、投影或反光")); + assert!( + prompt.contains("角色主体及其服饰、道具的颜色必须与背景色明显区分，不得带与背景色相同或相近的描边、投影或反光") + ); assert!(prompt.contains("禁止镜头透视")); assert!(prompt.contains("角色设定：菜市场卖菜大妈")); } @@ -6737,7 +6814,9 @@ mod tests { assert!(prompt.contains("参考图1的图标素材规范")); assert!(prompt.contains("背景必须是单一纯色背景 中度天蓝 #7FB3FF / RGB(127,179,255)")); assert!(prompt.contains("平整无纹理、无渐变、无阴影")); - assert!(prompt.contains("素材自身不要出现与背景色相同或相近的描边、底板、投影或反光")); + assert!( + prompt.contains("素材主体及其配色必须与背景色明显区分，不要出现与背景色相同或相近的描边、底板、投影或反光") + ); assert!(prompt.contains("返回按钮、设置按钮、下一关按钮")); } @@ -6759,7 +6838,7 @@ mod tests { parse_editor_screen_background_color(None) .expect("default screen color should parse") ), - "仅提取被红色框框选的素材并整理成spritesheet，图集背景必须使用单一纯色背景 浅雾蓝 #CFEFFF / RGB(207,239,255)。纯色背景必须平整无纹理、无渐变、无阴影、无地面、无环境、无道具，方便后续扣除背景；素材自身不要出现与背景色相同或相近的描边、底板、投影或反光。" + "仅提取被红色框框选的素材并整理成spritesheet，图集背景必须使用单一纯色背景 浅雾蓝 #CFEFFF / RGB(207,239,255)。纯色背景必须平整无纹理、无渐变、无阴影、无地面、无环境、无道具，方便后续扣除背景；素材主体及其配色必须与背景色明显区分，不要出现与背景色相同或相近的描边、底板、投影或反光。" ); } diff --git a/server-rs/crates/api-server/src/editor_screen_background_decision.rs b/server-rs/crates/api-server/src/editor_screen_background_decision.rs index 848520451..d2f59a7bd 100644 --- a/server-rs/crates/api-server/src/editor_screen_background_decision.rs +++ b/server-rs/crates/api-server/src/editor_screen_background_decision.rs @@ -1,4 +1,4 @@ -use platform_llm::{LlmClient, LlmTextRequest}; +use platform_llm::{LlmClient, LlmMessage, LlmTextRequest}; use serde_json::json; use tracing::{info, warn}; @@ -7,6 +7,10 @@ use crate::{ EDITOR_SCREEN_BACKGROUND_COLORS, EditorScreenBackgroundColor, default_editor_screen_background_color, parse_editor_screen_background_color, }, + editor_screen_background_filter::{ + ScreenBackgroundColorSafetyReport, decode_image_data_url, + filter_editor_screen_background_colors, safety_report_log_json, + }, http_error::AppError, }; @@ -54,11 +58,15 @@ pub(crate) struct EditorScreenBackgroundDecisionInput { pub(crate) prompt: String, pub(crate) icon_descriptions: Vec, pub(crate) reference_count: usize, + /// 主体参考图（data URL）。图生视频等主体颜色由参考图决定的场景必须传， + /// 让视觉 LLM 按主体实际配色避开撞色背景；纯文生图场景可为 None。 + pub(crate) source_image_data_url: Option, } pub(crate) async fn resolve_editor_screen_background_color( llm_client: Option<&LlmClient>, - input: EditorScreenBackgroundDecisionInput, + vision_llm_client: Option<&LlmClient>, + mut input: EditorScreenBackgroundDecisionInput, ) -> Result { if !is_auto_screen_background_color(input.screen_color.as_deref()) { return Ok(EditorScreenBackgroundDecision { @@ -69,7 +77,50 @@ pub(crate) async fn resolve_editor_screen_background_color( }); } - let fallback_color = default_editor_screen_background_color(); + let has_source_image = input + .source_image_data_url + .as_deref() + .map(str::trim) + .is_some_and(|value| !value.is_empty()); + // 硬过滤先于 LLM：按参考图前景配色剔除撞色候选，LLM 只在安全集合里做审美选择。 + // 分析失败（图片不可解码、前景不足）只降级为不过滤，不阻断生成。 + let safety_report = if has_source_image { + resolve_screen_background_safety_report(&input) + } else { + None + }; + let candidate_colors: Vec = safety_report + .as_ref() + .map(|report| report.allowed.clone()) + .unwrap_or_else(|| EDITOR_SCREEN_BACKGROUND_COLORS.to_vec()); + + // 默认 llm_client 是纯文本模型（Ark），收到图片分片会被上游 400 拒绝； + // 带图决策必须走 VectorEngine 视觉客户端，没有视觉客户端时降级为纯文本决策。 + let (llm_client, vision_model) = if has_source_image { + match vision_llm_client { + Some(client) => ( + Some(client), + Some(crate::llm_model_routing::EDITOR_SCREEN_BACKGROUND_VISION_LLM_MODEL), + ), + None => { + warn!( + kind = input.kind.label(), + "editor_screen_background_vision_client_missing_downgrade_to_text" + ); + input.source_image_data_url = None; + (llm_client, None) + } + } + } else { + (llm_client, None) + }; + + // 默认兜底色被过滤掉时，改用危险度最小的候选兜底。 + let default_color = default_editor_screen_background_color(); + let fallback_color = match safety_report.as_ref() { + Some(report) if !report.contains(default_color.hex) => report.safest, + _ => default_color, + }; let Some(llm_client) = llm_client else { warn!( kind = input.kind.label(), @@ -85,15 +136,33 @@ pub(crate) async fn resolve_editor_screen_background_color( }; let system_prompt = editor_screen_background_decision_system_prompt(); - let user_prompt = editor_screen_background_decision_user_prompt(&input); + let user_prompt = + editor_screen_background_decision_user_prompt(&input, &candidate_colors, fallback_color); + let source_image_data_url = input + .source_image_data_url + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()); let mut last_error: Option = None; for attempt in 1..=EDITOR_SCREEN_BACKGROUND_DECISION_MAX_ATTEMPTS { - let request = LlmTextRequest::single_turn(system_prompt, user_prompt.as_str()) + let user_message = match source_image_data_url { + Some(image_url) => { + LlmMessage::user(user_prompt.as_str()).with_image_url(image_url) + } + None => LlmMessage::user(user_prompt.as_str()), + }; + let mut request = LlmTextRequest::new(vec![LlmMessage::system(system_prompt), user_message]) .with_max_tokens(96) .with_request_timeout_ms(EDITOR_SCREEN_BACKGROUND_DECISION_TIMEOUT_MS); + if let Some(vision_model) = vision_model { + request = request.with_model(vision_model); + } match llm_client.request_text(request).await { Ok(response) => { - match parse_editor_screen_background_decision_response(response.content.as_str()) { + match parse_editor_screen_background_decision_response( + response.content.as_str(), + &candidate_colors, + ) { Some(color) => { info!( kind = input.kind.label(), @@ -148,6 +217,51 @@ pub(crate) async fn resolve_editor_screen_background_color( }) } +/// 解出参考图并做候选色硬过滤；任何一步失败都返回 None（降级为不过滤）。 +fn resolve_screen_background_safety_report( + input: &EditorScreenBackgroundDecisionInput, +) -> Option { + let data_url = input.source_image_data_url.as_deref()?.trim(); + let Some(image_bytes) = decode_image_data_url(data_url) else { + warn!( + kind = input.kind.label(), + "editor_screen_background_filter_skip_undecodable_data_url" + ); + return None; + }; + // 角色/动作类启用皮肤专属硬否决（皮肤恒暖恒在，是约束最紧的前景）；图标/UI 类只跑统计过滤。 + let apply_skin_veto = matches!( + input.kind, + EditorScreenBackgroundDecisionKind::Character + | EditorScreenBackgroundDecisionKind::CharacterAnimation + ); + match filter_editor_screen_background_colors(&image_bytes, apply_skin_veto) { + Ok(report) => { + if report.excluded.is_empty() { + info!( + kind = input.kind.label(), + "editor_screen_background_filter_no_exclusion" + ); + } else { + info!( + kind = input.kind.label(), + result = %safety_report_log_json(&report), + "editor_screen_background_filter_excluded_colliding_colors" + ); + } + Some(report) + } + Err(reason) => { + warn!( + kind = input.kind.label(), + reason = reason.as_str(), + "editor_screen_background_filter_skip_analysis_failed" + ); + None + } + } +} + pub(crate) fn is_auto_screen_background_color(value: Option<&str>) -> bool { value .map(str::trim) @@ -174,13 +288,15 @@ pub(crate) fn format_editor_screen_background_decision_input( } fn editor_screen_background_decision_system_prompt() -> &'static str { - "你是游戏素材生图的抠图背景色决策器。根据任务和用户输入，从候选列表中选择一个最不容易与主体混淆、最适合后续扣除的纯色背景。只能返回 JSON：{\"hex\":\"#RRGGBB\"}。不要解释，不要返回候选列表外的颜色。" + "你是游戏素材生图的抠图背景色决策器。根据任务和用户输入，从候选列表中选择一个最不容易与主体混淆、最适合后续扣除的纯色背景。如果附带了主体参考图，必须以图中主体（含服饰、道具、光效）的实际配色为准，选择色相距离最远的候选色，忽略图中已有的背景或透明区域。只能返回 JSON：{\"hex\":\"#RRGGBB\"}。不要解释，不要返回候选列表外的颜色。" } fn editor_screen_background_decision_user_prompt( input: &EditorScreenBackgroundDecisionInput, + candidate_colors: &[EditorScreenBackgroundColor], + fallback_color: EditorScreenBackgroundColor, ) -> String { - let colors = EDITOR_SCREEN_BACKGROUND_COLORS + let colors = candidate_colors .iter() .map(|color| { format!( @@ -190,16 +306,23 @@ fn editor_screen_background_decision_user_prompt( }) .collect::>() .join("\n"); + let has_source_image = input + .source_image_data_url + .as_deref() + .map(str::trim) + .is_some_and(|value| !value.is_empty()); let context = json!({ "task": input.kind.label(), "userPrompt": input.prompt, "iconDescriptions": input.icon_descriptions, "referenceCount": input.reference_count, + "hasSourceImage": has_source_image, "selectionCriteria": [ "优先选择与主体、主题色、UI 面板底色、发光效果和描边明显区分的背景色", "避免选择可能出现在主体、服饰、道具、按钮、图标或 UI 面板里的颜色", + "有主体参考图时以图中主体实际配色为准，文本描述只作辅助", "角色类优先低干扰浅色，图标和 UI 提取类优先更高对比但仍保持纯净", - "信息不足时选择浅雾蓝 #CFEFFF" + format!("信息不足时选择{} {}", fallback_color.label, fallback_color.hex) ], }); format!("候选背景色：\n{colors}\n\n任务上下文 JSON：\n{context}") @@ -207,27 +330,32 @@ fn editor_screen_background_decision_user_prompt( fn parse_editor_screen_background_decision_response( content: &str, + candidate_colors: &[EditorScreenBackgroundColor], ) -> Option { let trimmed = content.trim(); let json_source = extract_json_object(trimmed).unwrap_or(trimmed); if let Ok(value) = serde_json::from_str::(json_source) { if let Some(hex) = value.get("hex").and_then(serde_json::Value::as_str) { - return find_editor_screen_background_color(hex); + return find_editor_screen_background_color(candidate_colors, hex); } if let Some(hex) = value .get("screenColor") .or_else(|| value.get("screen_color")) .and_then(serde_json::Value::as_str) { - return find_editor_screen_background_color(hex); + return find_editor_screen_background_color(candidate_colors, hex); } } - find_hex_in_text(trimmed).and_then(find_editor_screen_background_color) + find_hex_in_text(trimmed) + .and_then(|hex| find_editor_screen_background_color(candidate_colors, hex)) } -fn find_editor_screen_background_color(value: &str) -> Option { +fn find_editor_screen_background_color( + candidate_colors: &[EditorScreenBackgroundColor], + value: &str, +) -> Option { let normalized = value.trim().to_ascii_uppercase(); - EDITOR_SCREEN_BACKGROUND_COLORS + candidate_colors .iter() .copied() .find(|color| color.hex == normalized) @@ -266,6 +394,7 @@ mod tests { #[tokio::test] async fn manual_screen_background_decision_keeps_selected_color() { let decision = resolve_editor_screen_background_color( + None, None, EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::Character, @@ -273,6 +402,7 @@ mod tests { prompt: "绿色史莱姆".to_string(), icon_descriptions: Vec::new(), reference_count: 1, + source_image_data_url: None, }, ) .await @@ -286,6 +416,7 @@ mod tests { #[tokio::test] async fn auto_screen_background_decision_falls_back_without_llm() { let decision = resolve_editor_screen_background_color( + None, None, EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::IconSpritesheet, @@ -293,6 +424,7 @@ mod tests { prompt: String::new(), icon_descriptions: vec!["金币按钮".to_string()], reference_count: 2, + source_image_data_url: None, }, ) .await @@ -306,6 +438,7 @@ mod tests { #[tokio::test] async fn missing_screen_background_decision_uses_auto_default() { let decision = resolve_editor_screen_background_color( + None, None, EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::UiDesignAssetExtraction, @@ -313,6 +446,7 @@ mod tests { prompt: "提取 UI 按钮".to_string(), icon_descriptions: Vec::new(), reference_count: 1, + source_image_data_url: None, }, ) .await @@ -325,9 +459,11 @@ mod tests { #[test] fn parses_llm_json_background_decision() { - let color = - parse_editor_screen_background_decision_response("```json\n{\"hex\":\"#a8f7f0\"}\n```") - .expect("known color should parse"); + let color = parse_editor_screen_background_decision_response( + "```json\n{\"hex\":\"#a8f7f0\"}\n```", + &EDITOR_SCREEN_BACKGROUND_COLORS, + ) + .expect("known color should parse"); assert_eq!(color.label, "高对比浅青"); } @@ -335,7 +471,68 @@ mod tests { #[test] fn rejects_unknown_llm_background_decision() { assert!( - parse_editor_screen_background_decision_response("{\"hex\":\"#123456\"}").is_none() + parse_editor_screen_background_decision_response( + "{\"hex\":\"#123456\"}", + &EDITOR_SCREEN_BACKGROUND_COLORS, + ) + .is_none() + ); + } + + #[test] + fn rejects_filtered_out_llm_background_decision() { + // LLM 返回了被硬过滤剔除的颜色（不在候选集里）时按无效响应处理。 + let allowed = [EDITOR_SCREEN_BACKGROUND_COLORS[0]]; + assert!( + parse_editor_screen_background_decision_response("{\"hex\":\"#7FB3FF\"}", &allowed) + .is_none() + ); + } + + #[tokio::test] + async fn auto_decision_fallback_avoids_colliding_default_color() { + use base64::Engine as _; + use image::{Rgba, RgbaImage}; + + // 前景是与默认兜底色（浅雾蓝 #CFEFFF）同色的实心块： + // 无 LLM 时兜底不能再落在被硬过滤剔除的默认色上。 + let mut image = RgbaImage::from_pixel(64, 64, Rgba([0, 0, 0, 0])); + for y in 16..48 { + for x in 16..48 { + image.put_pixel(x, y, Rgba([207, 239, 255, 255])); + } + } + let mut bytes = Vec::new(); + image::DynamicImage::ImageRgba8(image) + .write_to( + &mut std::io::Cursor::new(&mut bytes), + image::ImageFormat::Png, + ) + .expect("test image should encode"); + let data_url = format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(&bytes) + ); + + let decision = resolve_editor_screen_background_color( + None, + None, + EditorScreenBackgroundDecisionInput { + kind: EditorScreenBackgroundDecisionKind::CharacterAnimation, + screen_color: Some("auto".to_string()), + prompt: "起跳动作".to_string(), + icon_descriptions: Vec::new(), + reference_count: 1, + source_image_data_url: Some(data_url), + }, + ) + .await + .expect("decision should fall back"); + + assert!(decision.fallback); + assert_ne!( + decision.color.hex, "#CFEFFF", + "默认兜底色与前景撞色时应改用危险度最小的候选" ); } } diff --git a/server-rs/crates/api-server/src/editor_screen_background_filter.rs b/server-rs/crates/api-server/src/editor_screen_background_filter.rs new file mode 100644 index 000000000..f5e85b6d9 --- /dev/null +++ b/server-rs/crates/api-server/src/editor_screen_background_filter.rs @@ -0,0 +1,611 @@ +//! 背景色候选硬过滤：统计参考图前景配色，剔除会与前景（尤其皮肤）撞色的候选背景色。 +//! +//! 抠图管线对像素的判定由 ΔE（Lab 距离）和色调投影（像素色度在背景色调方向上的投影） +//! 两个量决定：ΔE ≤ 6.7 的前景近乎必被抹除；ΔE ≤ 28 且投影 ≥ 4（同色调壳层）时命运交给 +//! 后端 matting，同色调恰是其最不可靠的区间。经验上危险质量占前景 0.5% 以上就会出现 +//! 可见破损，故以此为剔除阈值。 +//! +//! 角色/动作类另加皮肤专属硬否决（三判据取并集）：动漫皮肤恒暖且恒在，是约束最紧的前景。 +//! Rule 1 由危险质量的 ΔE 门（28）承载；Rule 2 皮肤色度在候选色调方向投影 < 4（同色调否决， +//! 覆盖 trimap hue-split 与 cross-check 削薄）；Rule 3 候选与最亮皮肤的 RGB 合成距离 ≥ 52 +//! （两个都很浅的暖色在 RGB 天然接近，Lab ΔE 防不住，是第二条判死路径）。 +//! 三条的净效果是把角色候选压进冷区（蓝/青/紫），LLM 只在安全候选里做审美选择。 + +use std::collections::HashMap; + +use base64::Engine as _; +use image::GenericImageView; +use serde_json::json; + +use crate::editor_green_screen::{EDITOR_SCREEN_BACKGROUND_COLORS, EditorScreenBackgroundColor}; + +/// 危险格 ΔE 上限：超过它的前景像素被管线稳定判为前景。实测第 13 帧损伤发生在 ΔE 22–30 +/// （前景最近分位 p0.5≈28），故取 28 覆盖真实杀伤半径；proj 门保证异色调 bin 不被误伤。 +const DANGER_DELTA_E: f64 = 28.0; +/// 无条件危险 ΔE 上限：低于它的前景像素不看色调也近乎必被抹除。 +const SURE_LOSS_DELTA_E: f64 = 6.7; +/// 色调投影阈值：投影低于它（色度与背景色调近正交）的像素会被规则救回。 +const DANGER_HUE_PROJECTION: f64 = 4.0; +/// 危险质量占比达到该值即剔除候选色。 +const DANGER_MASS_EXCLUSION_RATIO: f64 = 0.005; +/// 单个颜色桶低于该质量占比视为噪声（残留背景、孤立杂色），不参与危险统计。 +const BIN_MASS_FLOOR_RATIO: f64 = 0.001; +/// 分析用缩略图最长边。 +const ANALYSIS_MAX_EDGE: u32 = 192; +/// 不透明图边框背景估计：主色覆盖率达到该值才认为边框是平坦背景。 +const BORDER_BACKGROUND_MIN_COVERAGE: f64 = 0.5; +/// 不透明图中与边框背景色 ΔE 不超过该值的像素按背景剔除。 +const BORDER_BACKGROUND_DELTA_E: f64 = 12.0; + +// 皮肤 bin 判定：高 L*、a*/b* 双正的暖色，且质量足够。 +const SKIN_MIN_LIGHTNESS: f64 = 55.0; +const SKIN_MIN_A: f64 = 3.0; +const SKIN_MIN_B: f64 = 8.0; +/// 皮肤 bin 质量下限：低到能保护小脸（远景角色脸可能只占前景零点几个百分点）。 +const SKIN_BIN_MASS_FLOOR_RATIO: f64 = 0.001; +/// Rule 2：皮肤色度在候选色调方向的投影达到该值即同色调撞色，否决。 +const SKIN_HUE_PROJECTION_LIMIT: f64 = 4.0; +/// Rule 3：候选与最亮皮肤的 RGB 合成距离（0–255）低于该值即过近，否决。取 52 是管线 RGB 通路 +/// bg_conf 冲到 0.5 的反解并留余量；不取完全沉默半径 68，否则会误杀已验证安全的 #CFEFFF（L2≈66）。 +const SKIN_MIN_RGB_DISTANCE: f64 = 52.0; + +#[derive(Clone, Debug)] +pub(crate) struct ScreenBackgroundColorSafetyReport { + /// 过滤后仍可用的候选色（保持调色板顺序），保证非空。 + pub(crate) allowed: Vec, + /// 被剔除的候选色及其危险质量占比。 + pub(crate) excluded: Vec<(EditorScreenBackgroundColor, f64)>, + /// 兜底色：皮肤安全候选里危险质量最小者（无皮肤安全候选时退回全局最小）。 + pub(crate) safest: EditorScreenBackgroundColor, +} + +impl ScreenBackgroundColorSafetyReport { + pub(crate) fn contains(&self, hex: &str) -> bool { + self.allowed.iter().any(|color| color.hex == hex) + } + + #[cfg(test)] + pub(crate) fn excluded_summary(&self) -> String { + self.excluded + .iter() + .map(|(color, ratio)| format!("{} {}({:.1}‰)", color.label, color.hex, ratio * 1000.0)) + .collect::>() + .join(", ") + } +} + +/// 从 image data URL 解出图片字节；非图片或无法解码时返回 None。 +pub(crate) fn decode_image_data_url(value: &str) -> Option> { + let body = value.trim().strip_prefix("data:")?; + let (mime_type, data) = body.split_once(";base64,")?; + if !mime_type.trim().starts_with("image/") { + return None; + } + base64::engine::general_purpose::STANDARD + .decode(data.trim()) + .ok() + .filter(|bytes| !bytes.is_empty()) +} + +/// 按参考图前景配色过滤候选背景色。`apply_skin_veto` 为 true（角色/动作类）时，检测到皮肤 bin +/// 还会启用 Rule 2/3 皮肤硬否决。失败（图片不可解码、前景不足）返回原因字符串， +/// 调用方应降级为不过滤，而不是阻断生成。 +pub(crate) fn filter_editor_screen_background_colors( + image_bytes: &[u8], + apply_skin_veto: bool, +) -> Result { + let image = image::load_from_memory(image_bytes) + .map_err(|error| format!("参考图无法解码：{error}"))?; + let (width, height) = image.dimensions(); + if width == 0 || height == 0 { + return Err("参考图尺寸为空".to_string()); + } + let image = if width.max(height) > ANALYSIS_MAX_EDGE { + // 最近邻缩放不混色：双线性会把前景像素和透明/背景像素调和出不存在的颜色。 + image.resize( + ANALYSIS_MAX_EDGE, + ANALYSIS_MAX_EDGE, + image::imageops::FilterType::Nearest, + ) + } else { + image + }; + let rgba = image.to_rgba8(); + + let histogram = build_foreground_histogram(&rgba)?; + let total_mass = histogram.total_mass; + // 皮肤约束基准：最亮的皮肤 bin（RGB 上最贴近浅色背景，是判据最紧点）。 + let skin = if apply_skin_veto { + histogram.brightest_skin_bin() + } else { + None + }; + + let mut allowed = Vec::new(); + let mut excluded = Vec::new(); + let mut safest = EDITOR_SCREEN_BACKGROUND_COLORS[0]; + let mut safest_ratio = f64::INFINITY; + for color in EDITOR_SCREEN_BACKGROUND_COLORS { + let danger_ratio = danger_mass(&histogram, color) / total_mass; + let skin_unsafe = skin + .as_ref() + .is_some_and(|reference| candidate_collides_with_skin(reference, color)); + // 兜底只在皮肤安全候选里挑危险质量最小者，避免把撞肤色（可能危险质量很低）兜回来。 + if !skin_unsafe && danger_ratio < safest_ratio { + safest_ratio = danger_ratio; + safest = color; + } + if danger_ratio >= DANGER_MASS_EXCLUSION_RATIO || skin_unsafe { + excluded.push((color, danger_ratio)); + } else { + allowed.push(color); + } + } + if safest_ratio.is_infinite() { + // 所有候选都皮肤不安全：退回全局危险质量最小者。 + for color in EDITOR_SCREEN_BACKGROUND_COLORS { + let danger_ratio = danger_mass(&histogram, color) / total_mass; + if danger_ratio < safest_ratio { + safest_ratio = danger_ratio; + safest = color; + } + } + } + if allowed.is_empty() { + allowed.push(safest); + excluded.retain(|(color, _)| color != &safest); + } + Ok(ScreenBackgroundColorSafetyReport { + allowed, + excluded, + safest, + }) +} + +/// 皮肤约束基准：最亮皮肤 bin 的 Lab 色度与 RGB 均值。 +struct SkinReference { + lab: [f64; 3], + rgb: [f64; 3], +} + +struct ForegroundHistogram { + bins: HashMap<(i32, i32, i32), HistogramBin>, + total_mass: f64, +} + +impl ForegroundHistogram { + /// 取最亮的皮肤 bin 作为约束基准；没有合格皮肤 bin（纯蓝机器人等）时返回 None。 + fn brightest_skin_bin(&self) -> Option { + let mass_floor = self.total_mass * SKIN_BIN_MASS_FLOOR_RATIO; + self.bins + .values() + .filter(|bin| bin.mass >= mass_floor) + .filter_map(|bin| { + let lab = bin.mean(); + (lab[0] > SKIN_MIN_LIGHTNESS && lab[1] > SKIN_MIN_A && lab[2] > SKIN_MIN_B) + .then(|| SkinReference { + lab, + rgb: bin.mean_rgb(), + }) + }) + .max_by(|left, right| { + left.lab[0] + .partial_cmp(&right.lab[0]) + .unwrap_or(std::cmp::Ordering::Equal) + }) + } +} + +#[derive(Default)] +struct HistogramBin { + mass: f64, + sum_l: f64, + sum_a: f64, + sum_b: f64, + sum_red: f64, + sum_green: f64, + sum_blue: f64, +} + +impl HistogramBin { + fn mean(&self) -> [f64; 3] { + [ + self.sum_l / self.mass, + self.sum_a / self.mass, + self.sum_b / self.mass, + ] + } + + fn mean_rgb(&self) -> [f64; 3] { + [ + self.sum_red / self.mass, + self.sum_green / self.mass, + self.sum_blue / self.mass, + ] + } +} + +const BIN_L_STEP: f64 = 8.0; +const BIN_AB_STEP: f64 = 6.0; + +fn bin_key(lab: [f64; 3]) -> (i32, i32, i32) { + ( + (lab[0] / BIN_L_STEP).floor() as i32, + (lab[1] / BIN_AB_STEP).floor() as i32, + (lab[2] / BIN_AB_STEP).floor() as i32, + ) +} + +fn build_foreground_histogram(rgba: &image::RgbaImage) -> Result { + let has_transparency = rgba.pixels().any(|pixel| pixel.0[3] < 250); + let mut bins: HashMap<(i32, i32, i32), HistogramBin> = HashMap::new(); + let mut total_mass = 0.0; + let mut add = |lab: [f64; 3], rgb: [f64; 3], weight: f64| { + let bin = bins.entry(bin_key(lab)).or_default(); + bin.mass += weight; + bin.sum_l += lab[0] * weight; + bin.sum_a += lab[1] * weight; + bin.sum_b += lab[2] * weight; + bin.sum_red += rgb[0] * weight; + bin.sum_green += rgb[1] * weight; + bin.sum_blue += rgb[2] * weight; + total_mass += weight; + }; + + if has_transparency { + // alpha² 加权：半透明羽化带（残留背景最集中的位置）权重自然趋零， + // 实心小区域（如角色的脸）质量不受损。 + for pixel in rgba.pixels() { + let [red, green, blue, alpha] = pixel.0; + if alpha == 0 { + continue; + } + let opacity = f64::from(alpha) / 255.0; + add( + srgb_to_lab(red, green, blue), + [f64::from(red), f64::from(green), f64::from(blue)], + opacity * opacity, + ); + } + } else { + // 不透明图：用边框主色估计原背景，剔除与其相近的像素；边框不平坦则全图算前景。 + let border_background = estimate_border_background(rgba); + for pixel in rgba.pixels() { + let [red, green, blue, _] = pixel.0; + let lab = srgb_to_lab(red, green, blue); + if let Some(background_lab) = border_background { + if delta_e(lab, background_lab) <= BORDER_BACKGROUND_DELTA_E { + continue; + } + } + add( + lab, + [f64::from(red), f64::from(green), f64::from(blue)], + 1.0, + ); + } + } + + let pixel_count = f64::from(rgba.width()) * f64::from(rgba.height()); + if total_mass < pixel_count * 0.01 { + return Err("参考图前景像素不足，无法做背景色安全分析".to_string()); + } + Ok(ForegroundHistogram { bins, total_mass }) +} + +/// 估计不透明图的边框背景色：边框环里主导颜色桶覆盖率足够高才认为存在平坦背景。 +fn estimate_border_background(rgba: &image::RgbaImage) -> Option<[f64; 3]> { + let (width, height) = rgba.dimensions(); + let ring = (width.min(height) / 32).max(1); + let mut bins: HashMap<(i32, i32, i32), HistogramBin> = HashMap::new(); + let mut total = 0.0; + for (x, y, pixel) in rgba.enumerate_pixels() { + let on_border = x < ring + || y < ring + || x >= width.saturating_sub(ring) + || y >= height.saturating_sub(ring); + if !on_border { + continue; + } + let [red, green, blue, _] = pixel.0; + let lab = srgb_to_lab(red, green, blue); + let bin = bins.entry(bin_key(lab)).or_default(); + bin.mass += 1.0; + bin.sum_l += lab[0]; + bin.sum_a += lab[1]; + bin.sum_b += lab[2]; + total += 1.0; + } + let dominant = bins.values().max_by(|left, right| { + left.mass + .partial_cmp(&right.mass) + .unwrap_or(std::cmp::Ordering::Equal) + })?; + (dominant.mass / total >= BORDER_BACKGROUND_MIN_COVERAGE).then(|| dominant.mean()) +} + +/// 候选背景色的危险质量：落入「ΔE ≤ 6.7 无条件」或「ΔE ≤ 28 且同色调（投影 ≥ 4）」的前景质量之和。 +fn danger_mass(histogram: &ForegroundHistogram, color: EditorScreenBackgroundColor) -> f64 { + let background_lab = srgb_to_lab(color.red, color.green, color.blue); + let chroma_norm = background_lab[1].hypot(background_lab[2]); + let mass_floor = histogram.total_mass * BIN_MASS_FLOOR_RATIO; + let mut danger = 0.0; + for bin in histogram.bins.values() { + if bin.mass < mass_floor { + continue; + } + let mean = bin.mean(); + let distance = delta_e(mean, background_lab); + if distance > DANGER_DELTA_E { + continue; + } + let projection = if chroma_norm > 1e-6 { + (mean[1] * background_lab[1] + mean[2] * background_lab[2]) / chroma_norm + } else { + mean[1].hypot(mean[2]) + }; + if distance <= SURE_LOSS_DELTA_E || projection >= DANGER_HUE_PROJECTION { + danger += bin.mass; + } + } + danger +} + +/// 皮肤专属硬否决（Rule 2 + Rule 3）：皮肤色度在候选色调方向投影 ≥ 4（同色调）， +/// 或候选与最亮皮肤的 RGB 合成距离 < 52（两个浅暖色天然接近）。任一命中即否决。 +fn candidate_collides_with_skin(skin: &SkinReference, color: EditorScreenBackgroundColor) -> bool { + let background_lab = srgb_to_lab(color.red, color.green, color.blue); + let chroma_norm = background_lab[1].hypot(background_lab[2]); + // Rule 2：近中性灰候选无色调可撞（chroma≈0），投影记 0，交给 Rule 3 的亮度分离判定。 + let projection = if chroma_norm > 1e-6 { + (skin.lab[1] * background_lab[1] + skin.lab[2] * background_lab[2]) / chroma_norm + } else { + 0.0 + }; + if projection >= SKIN_HUE_PROJECTION_LIMIT { + return true; + } + let delta_red = f64::from(color.red) - skin.rgb[0]; + let delta_green = f64::from(color.green) - skin.rgb[1]; + let delta_blue = f64::from(color.blue) - skin.rgb[2]; + let rgb_distance = + (delta_red * delta_red + delta_green * delta_green + delta_blue * delta_blue).sqrt(); + rgb_distance < SKIN_MIN_RGB_DISTANCE +} + +fn delta_e(left: [f64; 3], right: [f64; 3]) -> f64 { + let dl = left[0] - right[0]; + let da = left[1] - right[1]; + let db = left[2] - right[2]; + (dl * dl + da * da + db * db).sqrt() +} + +fn srgb_to_lab(red: u8, green: u8, blue: u8) -> [f64; 3] { + fn linearize(channel: u8) -> f64 { + let value = f64::from(channel) / 255.0; + if value <= 0.04045 { + value / 12.92 + } else { + ((value + 0.055) / 1.055).powf(2.4) + } + } + let (r, g, b) = (linearize(red), linearize(green), linearize(blue)); + let x = r * 0.4124564 + g * 0.3575761 + b * 0.1804375; + let y = r * 0.2126729 + g * 0.7151522 + b * 0.0721750; + let z = r * 0.0193339 + g * 0.1191920 + b * 0.9503041; + fn transfer(t: f64) -> f64 { + if t > 0.008856 { + t.cbrt() + } else { + 7.787 * t + 16.0 / 116.0 + } + } + let (fx, fy, fz) = ( + transfer(x / 0.95047), + transfer(y / 1.0), + transfer(z / 1.08883), + ); + [116.0 * fy - 16.0, 500.0 * (fx - fy), 200.0 * (fy - fz)] +} + +/// 供日志使用的过滤结果 JSON。 +pub(crate) fn safety_report_log_json( + report: &ScreenBackgroundColorSafetyReport, +) -> serde_json::Value { + json!({ + "allowed": report.allowed.iter().map(|color| color.hex).collect::>(), + "excluded": report + .excluded + .iter() + .map(|(color, ratio)| json!({ "hex": color.hex, "dangerPermille": (ratio * 1000.0 * 10.0).round() / 10.0 })) + .collect::>(), + "safest": report.safest.hex, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use image::{Rgba, RgbaImage}; + + fn encode_png(image: &RgbaImage) -> Vec { + let mut bytes = Vec::new(); + image::DynamicImage::ImageRgba8(image.clone()) + .write_to( + &mut std::io::Cursor::new(&mut bytes), + image::ImageFormat::Png, + ) + .expect("test image should encode"); + bytes + } + + fn transparent_image_with_center_block(color: [u8; 3]) -> RgbaImage { + let mut image = RgbaImage::from_pixel(64, 64, Rgba([0, 0, 0, 0])); + for y in 16..48 { + for x in 16..48 { + image.put_pixel(x, y, Rgba([color[0], color[1], color[2], 255])); + } + } + image + } + + fn report_for(image: &RgbaImage, apply_skin_veto: bool) -> ScreenBackgroundColorSafetyReport { + filter_editor_screen_background_colors(&encode_png(image), apply_skin_veto) + .expect("filter should analyze the test image") + } + + #[test] + fn skin_bearing_character_excludes_warm_keeps_cool() { + // 亮暖肤块：Rule 2/3 应同时剔除暖桃与淡黄（两次真实事故的撞色），保留冷色。 + let report = report_for(&transparent_image_with_center_block([250, 224, 200]), true); + + assert!( + report.excluded.iter().any(|(color, _)| color.hex == "#FFD6C2"), + "肤色前景应剔除暖浅桃色，excluded: {}", + report.excluded_summary() + ); + assert!( + report.excluded.iter().any(|(color, _)| color.hex == "#FFF2A8"), + "肤色前景应剔除淡黄（Rule 2/3 关键新覆盖），excluded: {}", + report.excluded_summary() + ); + assert!(report.contains("#CFEFFF"), "冷色浅雾蓝应保留"); + assert_ne!(report.safest.hex, "#FFD6C2"); + assert_ne!(report.safest.hex, "#FFF2A8"); + } + + #[test] + fn skin_veto_gating_only_matters_for_minority_skin() { + // 大面积暗中性前景 + 极小肤块（占前景 ~0.3%，低于危险质量阈值 0.5%）： + // 关闭皮肤否决时统计过滤抓不到淡黄；开启时皮肤规则应抓到。这正是「小脸」场景。 + let mut image = RgbaImage::from_pixel(64, 64, Rgba([0, 0, 0, 0])); + for y in 0..56 { + for x in 0..54 { + image.put_pixel(x, y, Rgba([40, 40, 40, 255])); // 3024 px 暗中性，不撞任何浅候选 + } + } + for y in 60..63 { + for x in 60..63 { + image.put_pixel(x, y, Rgba([250, 224, 200, 255])); // 9 px 肤块 ≈ 0.3% 前景 + } + } + + let disabled = report_for(&image, false); + assert!( + disabled.contains("#FFF2A8"), + "关闭皮肤否决时小肤块不足以统计剔除淡黄，excluded: {}", + disabled.excluded_summary() + ); + + let enabled = report_for(&image, true); + assert!( + enabled.excluded.iter().any(|(color, _)| color.hex == "#FFF2A8"), + "开启皮肤否决时淡黄应被剔除，excluded: {}", + enabled.excluded_summary() + ); + } + + #[test] + fn no_skin_detected_skips_skin_veto() { + // 无皮肤 bin（纯青前景）即使 apply_skin_veto=true 也不否决暖色。 + let report = report_for(&transparent_image_with_center_block([0, 180, 180]), true); + assert!( + report.contains("#FFD6C2"), + "无皮肤时暖桃不应被皮肤规则剔除，excluded: {}", + report.excluded_summary() + ); + } + + #[test] + fn blue_foreground_excludes_blue_candidate() { + let report = report_for(&transparent_image_with_center_block([127, 179, 255]), false); + + assert!( + report.excluded.iter().any(|(color, _)| color.hex == "#7FB3FF"), + "蓝色前景应剔除中度天蓝，excluded: {}", + report.excluded_summary() + ); + assert!(report.contains("#FFD6C2"), "暖浅桃色应保留"); + } + + #[test] + fn semi_transparent_fringe_does_not_exclude_candidates() { + // 前景是红色实心块，另有 100 个 alpha=32 的中度天蓝残留像素（模拟没滤净的旧背景羽化带）。 + let mut image = transparent_image_with_center_block([200, 30, 30]); + let mut placed = 0u32; + 'outer: for y in 0..16u32 { + for x in 0..64u32 { + if placed >= 100 { + break 'outer; + } + image.put_pixel(x, y, Rgba([127, 179, 255, 32])); + placed += 1; + } + } + + let report = report_for(&image, false); + assert!( + report.contains("#7FB3FF"), + "半透明残留不应剔除候选色，excluded: {}", + report.excluded_summary() + ); + } + + #[test] + fn opaque_flat_background_is_not_counted_as_foreground() { + // 不透明图：浅雾蓝底 + 深红主体。若边框背景估计失效，浅雾蓝会被误判撞色。 + let mut image = RgbaImage::from_pixel(64, 64, Rgba([207, 239, 255, 255])); + for y in 24..40 { + for x in 24..40 { + image.put_pixel(x, y, Rgba([139, 0, 0, 255])); + } + } + + let report = report_for(&image, false); + assert!( + report.contains("#CFEFFF"), + "平坦背景应被剔除出前景统计，excluded: {}", + report.excluded_summary() + ); + } + + #[test] + fn all_candidates_dangerous_keeps_safest_only() { + // 前景铺满全部候选色，任何候选都撞色 → 只保留危险度最小的一个。 + let stripe = 6u32; + let width = EDITOR_SCREEN_BACKGROUND_COLORS.len() as u32 * stripe; + let mut image = RgbaImage::from_pixel(width, 64, Rgba([0, 0, 0, 0])); + for (index, color) in EDITOR_SCREEN_BACKGROUND_COLORS.iter().enumerate() { + let x_start = index as u32 * stripe; + for y in 0..64 { + for x in x_start..x_start + stripe { + image.put_pixel(x, y, Rgba([color.red, color.green, color.blue, 255])); + } + } + } + + let report = report_for(&image, false); + assert_eq!(report.allowed.len(), 1); + assert_eq!(report.allowed[0].hex, report.safest.hex); + } + + #[test] + fn fully_transparent_image_fails_analysis() { + let image = RgbaImage::from_pixel(32, 32, Rgba([0, 0, 0, 0])); + assert!(filter_editor_screen_background_colors(&encode_png(&image), true).is_err()); + } + + #[test] + fn decodes_image_data_url() { + let image = transparent_image_with_center_block([10, 20, 30]); + let bytes = encode_png(&image); + let data_url = format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(&bytes) + ); + + assert_eq!(decode_image_data_url(&data_url), Some(bytes)); + assert_eq!(decode_image_data_url("data:video/mp4;base64,AAAA"), None); + assert_eq!(decode_image_data_url("not a data url"), None); + } +} diff --git a/server-rs/crates/api-server/src/llm_model_routing.rs b/server-rs/crates/api-server/src/llm_model_routing.rs index 4fd986fde..16cd06539 100644 --- a/server-rs/crates/api-server/src/llm_model_routing.rs +++ b/server-rs/crates/api-server/src/llm_model_routing.rs @@ -1,3 +1,5 @@ pub(crate) const RPG_STORY_LLM_MODEL: &str = "doubao-seed-character-251128"; pub(crate) const CREATION_TEMPLATE_LLM_MODEL: &str = "deepseek-v3-2-251201"; pub(crate) const PUZZLE_LEVEL_NAME_VISION_LLM_MODEL: &str = "gpt-4o-mini"; +// 抠图背景色决策的视觉模型：只需看图选色，用低成本视觉模型即可。 +pub(crate) const EDITOR_SCREEN_BACKGROUND_VISION_LLM_MODEL: &str = "gpt-4o-mini"; diff --git a/server-rs/crates/api-server/src/main.rs b/server-rs/crates/api-server/src/main.rs index 06b145b50..142c6bff3 100644 --- a/server-rs/crates/api-server/src/main.rs +++ b/server-rs/crates/api-server/src/main.rs @@ -43,6 +43,7 @@ mod editor_generation_queue; mod editor_green_screen; mod editor_project; mod editor_screen_background_decision; +mod editor_screen_background_filter; mod edutainment_baby_drawing; mod edutainment_baby_object; mod error_middleware; -- 2.52.0 From 13411eb5959c3e03c2c39495b859ce5b742c47cc Mon Sep 17 00:00:00 2001 From: Linghong Date: Wed, 8 Jul 2026 14:25:17 +0000 Subject: [PATCH 06/41] =?UTF-8?q?=E8=83=8C=E6=99=AF=E8=89=B2=E8=A7=86?= =?UTF-8?q?=E8=A7=89=E5=86=B3=E7=AD=96=E6=94=B9=E7=94=A8=20gpt-5-mini?= =?UTF-8?q?=EF=BC=88Responses=20+=20low=20=E6=8E=A8=E7=90=86=E6=A1=A3?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 视觉模型 gpt-4o-mini → gpt-5-mini。 - gpt-5-mini 是推理模型：视觉决策走 Responses 协议并设 reasoning_effort=low， 否则默认档会把 token 预算全烧在推理上、返回空答案；纯文本路径仍走 ChatCompletions。 - max_tokens 96 → 768，给 low 档推理（实测 320~384 token）留足余量。 已真机验证：/responses 端点网关可用，暖肤色主体正确选中冷色背景。 Co-Authored-By: Claude Opus 4.8 --- .../src/editor_screen_background_decision.rs | 13 ++++++++++--- .../crates/api-server/src/llm_model_routing.rs | 5 +++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/server-rs/crates/api-server/src/editor_screen_background_decision.rs b/server-rs/crates/api-server/src/editor_screen_background_decision.rs index d2f59a7bd..1762e3e92 100644 --- a/server-rs/crates/api-server/src/editor_screen_background_decision.rs +++ b/server-rs/crates/api-server/src/editor_screen_background_decision.rs @@ -1,4 +1,4 @@ -use platform_llm::{LlmClient, LlmMessage, LlmTextRequest}; +use platform_llm::{LlmClient, LlmMessage, LlmResponseReasoningEffort, LlmTextRequest}; use serde_json::json; use tracing::{info, warn}; @@ -151,11 +151,18 @@ pub(crate) async fn resolve_editor_screen_background_color( } None => LlmMessage::user(user_prompt.as_str()), }; + // 预算要够推理模型（如 gpt-5-mini）先花几百 token 推理、再吐 JSON 答案； + // 实测 low 档推理约 320~384 token，取 768 留足余量。非推理模型遇 stop 提前结束，不会多花。 let mut request = LlmTextRequest::new(vec![LlmMessage::system(system_prompt), user_message]) - .with_max_tokens(96) + .with_max_tokens(768) .with_request_timeout_ms(EDITOR_SCREEN_BACKGROUND_DECISION_TIMEOUT_MS); if let Some(vision_model) = vision_model { - request = request.with_model(vision_model); + // 视觉模型 gpt-5-mini 是推理模型：走 Responses 协议并压到 low 推理档， + // 否则默认档会把预算全烧在推理上、返回空答案。 + request = request + .with_model(vision_model) + .with_responses_api() + .with_response_reasoning_effort(LlmResponseReasoningEffort::Low); } match llm_client.request_text(request).await { Ok(response) => { diff --git a/server-rs/crates/api-server/src/llm_model_routing.rs b/server-rs/crates/api-server/src/llm_model_routing.rs index 16cd06539..49f8e6740 100644 --- a/server-rs/crates/api-server/src/llm_model_routing.rs +++ b/server-rs/crates/api-server/src/llm_model_routing.rs @@ -1,5 +1,6 @@ pub(crate) const RPG_STORY_LLM_MODEL: &str = "doubao-seed-character-251128"; pub(crate) const CREATION_TEMPLATE_LLM_MODEL: &str = "deepseek-v3-2-251201"; pub(crate) const PUZZLE_LEVEL_NAME_VISION_LLM_MODEL: &str = "gpt-4o-mini"; -// 抠图背景色决策的视觉模型：只需看图选色，用低成本视觉模型即可。 -pub(crate) const EDITOR_SCREEN_BACKGROUND_VISION_LLM_MODEL: &str = "gpt-4o-mini"; +// 抠图背景色决策的视觉模型。gpt-5-mini 是推理模型，会先花若干 token 做推理， +// 故决策请求的 max_tokens 需留足推理开销（见 editor_screen_background_decision）。 +pub(crate) const EDITOR_SCREEN_BACKGROUND_VISION_LLM_MODEL: &str = "gpt-5-mini"; -- 2.52.0 From 1c1db172152f32be2428b5643dcead96d2ec8e0e Mon Sep 17 00:00:00 2001 From: Linghong Date: Wed, 8 Jul 2026 14:39:44 +0000 Subject: [PATCH 07/41] =?UTF-8?q?=E8=83=8C=E6=99=AF=E8=89=B2=E5=86=B3?= =?UTF-8?q?=E7=AD=96=20max=5Ftokens=20768=20=E2=86=92=201024?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 给 gpt-5-mini low 推理档再留余量（实测推理 320~384 token）， 降低复杂图触顶后走空响应兜底的概率。 Co-Authored-By: Claude Opus 4.8 --- .../api-server/src/editor_screen_background_decision.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server-rs/crates/api-server/src/editor_screen_background_decision.rs b/server-rs/crates/api-server/src/editor_screen_background_decision.rs index 1762e3e92..48c47c27f 100644 --- a/server-rs/crates/api-server/src/editor_screen_background_decision.rs +++ b/server-rs/crates/api-server/src/editor_screen_background_decision.rs @@ -152,9 +152,9 @@ pub(crate) async fn resolve_editor_screen_background_color( None => LlmMessage::user(user_prompt.as_str()), }; // 预算要够推理模型（如 gpt-5-mini）先花几百 token 推理、再吐 JSON 答案； - // 实测 low 档推理约 320~384 token，取 768 留足余量。非推理模型遇 stop 提前结束，不会多花。 + // 实测 low 档推理约 320~384 token，取 1024 留足余量。非推理模型遇 stop 提前结束，不会多花。 let mut request = LlmTextRequest::new(vec![LlmMessage::system(system_prompt), user_message]) - .with_max_tokens(768) + .with_max_tokens(1024) .with_request_timeout_ms(EDITOR_SCREEN_BACKGROUND_DECISION_TIMEOUT_MS); if let Some(vision_model) = vision_model { // 视觉模型 gpt-5-mini 是推理模型：走 Responses 协议并压到 low 推理档， -- 2.52.0 From c78f199f4e5697e2d1136f42928aabb14420054b Mon Sep 17 00:00:00 2001 From: Linghong Date: Thu, 9 Jul 2026 03:09:57 +0000 Subject: [PATCH 08/41] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20shared-contracts=20?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=9E=84=E5=BB=BA=EF=BC=9A=E8=A1=A5=20screen?= =?UTF-8?q?=5Fcolor=20=E5=AD=97=E6=AE=B5=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EditorCharacterAnimationGenerateRequest 新增 screen_color 后， shared-contracts 的 camelCase 契约测试初始化漏补该字段，导致 `cargo test -p shared-contracts --no-run` 编译失败（P0，阻塞 PR 合入）。 Co-Authored-By: Claude Opus 4.8 --- server-rs/crates/shared-contracts/src/assets.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/server-rs/crates/shared-contracts/src/assets.rs b/server-rs/crates/shared-contracts/src/assets.rs index 0251a8f32..c30598dc1 100644 --- a/server-rs/crates/shared-contracts/src/assets.rs +++ b/server-rs/crates/shared-contracts/src/assets.rs @@ -1078,6 +1078,7 @@ mod tests { source_width: 768, source_height: 1024, prompt_text: "待机呼吸".to_string(), + screen_color: None, resolution: "720p".to_string(), ratio: "same".to_string(), frame_count: 48, -- 2.52.0 From ef8ac943953c2bd1b2a3efb2f069e26ed3a6b1cd Mon Sep 17 00:00:00 2001 From: Linghong Date: Thu, 9 Jul 2026 03:13:55 +0000 Subject: [PATCH 09/41] =?UTF-8?q?BgFilter=20=E8=AF=B7=E6=B1=82=E8=B6=85?= =?UTF-8?q?=E6=97=B6=E4=BF=9D=E6=8C=81=20120s=EF=BC=88=E5=90=88=E5=B9=B6?= =?UTF-8?q?=20web/master=20=E6=97=B6=E8=AF=AF=E5=8F=96=2045s=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BgFilter 目前跑 CPU 推理，单次抠图耗时较长，超时必须 120s（见文档）。 合并 web/master 的「收紧超时保护」时该常量被误解成 45s，改回 120s。 web/master 新增的熔断器配置（失败阈值/冷却）保留不变。 Co-Authored-By: Claude Opus 4.8 --- server-rs/crates/api-server/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server-rs/crates/api-server/src/config.rs b/server-rs/crates/api-server/src/config.rs index 50b1fc06e..6c6d55526 100644 --- a/server-rs/crates/api-server/src/config.rs +++ b/server-rs/crates/api-server/src/config.rs @@ -20,7 +20,7 @@ pub(crate) const DEFAULT_VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS: u64 = 1_000_000 const DEFAULT_EDITOR_BACKGROUND_REMOVAL_BASE_URL: &str = "http://58.87.105.82"; const DEFAULT_EDITOR_BACKGROUND_REMOVAL_REQUEST_TIMEOUT_MS: u64 = 120_000; const DEFAULT_EDITOR_BGFILTER_BASE_URL: &str = "http://58.87.105.82/bgfilter"; -const DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS: u64 = 45_000; +const DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS: u64 = 120_000; const DEFAULT_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD: u32 = 3; const DEFAULT_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS: u64 = 300; const DEFAULT_ALIYUN_MATTING_ENDPOINT: &str = "imageseg.cn-shanghai.aliyuncs.com"; -- 2.52.0 From 31117ffdb977c8f382837c007f1219982cdae563 Mon Sep 17 00:00:00 2001 From: Linghong Date: Thu, 9 Jul 2026 03:38:00 +0000 Subject: [PATCH 10/41] =?UTF-8?q?BgFilter/=E9=98=BF=E9=87=8C=E4=BA=91?= =?UTF-8?q?=E5=85=9C=E5=BA=95=E6=8A=A0=E5=9B=BE=E5=A4=B1=E8=B4=A5=E6=8E=A5?= =?UTF-8?q?=E5=85=A5=E5=A4=96=E9=83=A8=20API=20=E5=A4=B1=E8=B4=A5=E5=AE=A1?= =?UTF-8?q?=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 架构要求外部供应商调用失败必须进 OTLP + tracking_event external_api_call_failure，但 BgFilter 和阿里云通用抠图失败原来只 warn! 后静默兜底，生产故障在审计链路里不可见（P1）。 - external_api_audit：新增 ExternalApiAuditContext 与统一入口 record_matting_external_api_failure（复用 record_external_api_failure， 自动填 status_class/retryable）；is_retryable_external_api_failure 去掉 cfg(test) 门供生产复用，内部 StatusCode 换字面量避开测试专用导入。 - 三处失败点接入：BgFilter 请求失败、生图链路阿里云兜底失败、 动作视频逐帧阿里云兜底失败（带 frame_index）。 - user/profile/request_id 从各入口 caller / owner+project+request_context 透传进兜底函数，沿用现有 external_api_audit 上下文映射。 Co-Authored-By: Claude Opus 4.8 --- .../src/character_animation_assets.rs | 22 ++++++++ .../crates/api-server/src/editor_project.rs | 52 ++++++++++++++++++ .../api-server/src/external_api_audit.rs | 55 +++++++++++++++++-- 3 files changed, 123 insertions(+), 6 deletions(-) diff --git a/server-rs/crates/api-server/src/character_animation_assets.rs b/server-rs/crates/api-server/src/character_animation_assets.rs index dd7899953..1cf3bdb5d 100644 --- a/server-rs/crates/api-server/src/character_animation_assets.rs +++ b/server-rs/crates/api-server/src/character_animation_assets.rs @@ -686,6 +686,12 @@ pub(crate) async fn generate_editor_character_animation_for_owner( let http_client = build_upstream_http_client(settings.ark.request_timeout_ms) .map_err(|error| character_animation_error_response(&request_context, error))?; let task_id = generate_ai_task_id(current_utc_micros()); + // 抠图供应商失败审计上下文：逐帧阿里云抠图失败即使兜底成功，也要进 OTLP + tracking_event。 + let matting_audit = crate::external_api_audit::ExternalApiAuditContext { + user_id: Some(owner_user_id.clone()), + profile_id: project_id.clone(), + request_id: Some(request_context.request_id().to_string()), + }; let result = execute_billable_asset_operation_with_cost( &state, @@ -713,6 +719,7 @@ pub(crate) async fn generate_editor_character_animation_for_owner( generated.preview_video_path.as_str(), &normalized, &extraction_settings, + &matting_audit, ) .await?; @@ -2090,6 +2097,7 @@ async fn extract_and_persist_editor_character_animation_frames( preview_video_path: &str, request: &NormalizedEditorCharacterAnimationRequest, extraction_settings: &BackendFrameExtractionSettings, + audit: &crate::external_api_audit::ExternalApiAuditContext, ) -> Result, AppError> { let plan = AnimationFrameExtractionPlan { frame_count: request.frame_count, @@ -2121,6 +2129,7 @@ async fn extract_and_persist_editor_character_animation_frames( request.frame_width, request.frame_height, request.screen_color, + audit, ) .await?; @@ -2199,6 +2208,7 @@ async fn remove_editor_character_animation_frame_backgrounds( frame_width: u32, frame_height: u32, screen_color: EditorScreenBackgroundColor, + audit: &crate::external_api_audit::ExternalApiAuditContext, ) -> Result, AppError> { use futures_util::{StreamExt as _, TryStreamExt as _}; @@ -2228,6 +2238,18 @@ async fn remove_editor_character_animation_frame_backgrounds( error_details = ?error.details(), "editor_animation_frame_aliyun_matting_fallback_to_local" ); + crate::external_api_audit::record_matting_external_api_failure( + state, + audit, + "aliyun-matting", + state.config.aliyun_matting_endpoint.clone(), + "editor-character-animation-frame-matting", + "aliyun_segment", + error.status_code().as_u16(), + error.message().to_string(), + Some(format!("frame_index={frame_index}")), + ) + .await; remove_editor_generated_green_screen_background(&image, screen_color)? } }; diff --git a/server-rs/crates/api-server/src/editor_project.rs b/server-rs/crates/api-server/src/editor_project.rs index c8e207289..eea999ee6 100644 --- a/server-rs/crates/api-server/src/editor_project.rs +++ b/server-rs/crates/api-server/src/editor_project.rs @@ -1554,11 +1554,20 @@ pub(crate) async fn generate_editor_image_for_owner( "character-image", ) .await?; + let matting_audit = crate::external_api_audit::ExternalApiAuditContext { + user_id: caller.audit_subject_user_id.clone(), + profile_id: caller + .audit_project_id + .clone() + .or_else(|| payload.project_id.clone()), + request_id: Some(request_context.request_id().to_string()), + }; image = remove_editor_generated_screen_background_with_bgfilter( state, &image, screen_color.expect("character generation should have screen color"), seg_model.expect("character generation should have BgFilter seg model"), + &matting_audit, ) .await?; } @@ -2226,6 +2235,7 @@ async fn remove_editor_generated_screen_background_with_bgfilter( image: &DownloadedOpenAiImage, screen_color: EditorScreenBackgroundColor, seg_model: &str, + audit: &crate::external_api_audit::ExternalApiAuditContext, ) -> Result { if let Some(remaining) = editor_bgfilter_circuit_open_remaining(state) { tracing::warn!( @@ -2260,6 +2270,18 @@ async fn remove_editor_generated_screen_background_with_bgfilter( error_details = ?error.details(), "editor_bgfilter_fallback_to_aliyun_matting" ); + crate::external_api_audit::record_matting_external_api_failure( + state, + audit, + "bgfilter", + state.config.editor_bgfilter_base_url.clone(), + "editor-screen-background-removal", + "bgfilter_segment", + error.status_code().as_u16(), + error.message().to_string(), + None, + ) + .await; match crate::aliyun_matting::segment_image_with_aliyun_matting( state, image, @@ -2276,6 +2298,18 @@ async fn remove_editor_generated_screen_background_with_bgfilter( error_details = ?error.details(), "editor_aliyun_matting_fallback_to_local_screen_background_removal" ); + crate::external_api_audit::record_matting_external_api_failure( + state, + audit, + "aliyun-matting", + state.config.aliyun_matting_endpoint.clone(), + "editor-screen-background-removal", + "aliyun_segment", + error.status_code().as_u16(), + error.message().to_string(), + None, + ) + .await; remove_editor_generated_green_screen_background(image, screen_color) } } @@ -2936,11 +2970,20 @@ pub(crate) async fn generate_editor_icon_spritesheet_for_owner( "spritesheet", ) .await?; + let matting_audit = crate::external_api_audit::ExternalApiAuditContext { + user_id: caller.audit_subject_user_id.clone(), + profile_id: caller + .audit_project_id + .clone() + .or_else(|| payload.project_id.clone()), + request_id: Some(request_context.request_id().to_string()), + }; let image = remove_editor_generated_screen_background_with_bgfilter( state, &image, screen_color, seg_model, + &matting_audit, ) .await?; let (spritesheet_width, spritesheet_height) = image::load_from_memory(image.bytes.as_slice()) @@ -3204,11 +3247,20 @@ pub(crate) async fn extract_editor_ui_design_assets_for_owner( "spritesheet", ) .await?; + let matting_audit = crate::external_api_audit::ExternalApiAuditContext { + user_id: caller.audit_subject_user_id.clone(), + profile_id: caller + .audit_project_id + .clone() + .or_else(|| payload.project_id.clone()), + request_id: Some(request_context.request_id().to_string()), + }; let image = remove_editor_generated_screen_background_with_bgfilter( state, &image, screen_color, seg_model, + &matting_audit, ) .await?; let (spritesheet_width, spritesheet_height) = image::load_from_memory(image.bytes.as_slice()) diff --git a/server-rs/crates/api-server/src/external_api_audit.rs b/server-rs/crates/api-server/src/external_api_audit.rs index 11a104d57..cc0dc824b 100644 --- a/server-rs/crates/api-server/src/external_api_audit.rs +++ b/server-rs/crates/api-server/src/external_api_audit.rs @@ -130,6 +130,53 @@ impl ExternalApiFailureDraft { self.request_id = request_id; self } + + pub(crate) fn with_audit_context(mut self, context: &ExternalApiAuditContext) -> Self { + self.user_id = context.user_id.clone(); + self.profile_id = context.profile_id.clone(); + self.request_id = context.request_id.clone(); + self + } +} + +/// 外部 API 失败审计的调用方上下文（用户 / 档案 / 请求 id）。 +#[derive(Clone, Debug, Default)] +pub(crate) struct ExternalApiAuditContext { + pub(crate) user_id: Option, + pub(crate) profile_id: Option, + pub(crate) request_id: Option, +} + +/// 抠图供应商（BgFilter / 阿里云通用抠图）调用失败的统一失败审计入口。 +/// 即使随后兜底成功，供应商故障也必须进入 OTLP + tracking_event，不能只 warn! 后静默。 +pub(crate) async fn record_matting_external_api_failure( + state: &AppState, + context: &ExternalApiAuditContext, + provider: &'static str, + endpoint: String, + operation: &'static str, + failure_stage: &'static str, + status_code: u16, + error_message: String, + raw_excerpt: Option, +) { + let draft = ExternalApiFailureDraft::new( + provider, + endpoint, + operation, + failure_stage, + error_message, + ) + .with_status_code(Some(status_code)) + .with_optional_status_class(Some(status_class(Some(status_code)))) + .with_retryable(is_retryable_external_api_failure( + Some(status_code), + false, + false, + )) + .with_raw_excerpt(raw_excerpt) + .with_audit_context(context); + record_external_api_failure(state, draft).await; } pub(crate) fn build_external_api_failure_draft_from_platform_image_audit( @@ -306,19 +353,15 @@ fn build_external_api_failure_metadata(failure: &ExternalApiFailureDraft) -> Val metadata } -#[cfg(test)] pub(crate) fn is_retryable_external_api_failure( status_code: Option, timeout: bool, connect: bool, ) -> bool { + // 429 Too Many Requests / 408 Request Timeout / 5xx 视为可重试。 timeout || connect - || status_code.is_some_and(|status| { - status == StatusCode::TOO_MANY_REQUESTS.as_u16() - || status == StatusCode::REQUEST_TIMEOUT.as_u16() - || status >= 500 - }) + || status_code.is_some_and(|status| status == 429 || status == 408 || status >= 500) } fn record_external_api_failure_otlp(failure: &ExternalApiFailureDraft) { -- 2.52.0 From 578313050e475f8beb1c46f50deb2a54239a3bc4 Mon Sep 17 00:00:00 2001 From: Linghong Date: Thu, 9 Jul 2026 03:44:50 +0000 Subject: [PATCH 11/41] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E8=A7=92=E8=89=B2?= =?UTF-8?q?=E5=8A=A8=E4=BD=9C=E8=A7=86=E9=A2=91=E8=83=8C=E6=99=AF=E8=89=B2?= =?UTF-8?q?=E6=96=B0=E5=A5=91=E7=BA=A6=E5=88=B0=E6=9D=83=E5=A8=81=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E4=B8=8E=E5=86=B3=E7=AD=96=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 角色动作抽帧的正式契约已改（screenColor=auto 视觉决策 + 硬过滤 + 源图合成到选定背景色 + 抽帧优先阿里云、失败降级本地键色），但权威 文档仍写着 legacy #00FF00 + 本地 editor_green_screen（P1 文档失同步）。 - 后端数据契约文档、前端图片画布 MVP 文档：更新角色动作抽帧描述到 新契约；顺带把 BgFilter 默认超时 45000ms 修正为 120000ms（与代码 一致，CPU 推理必须留足）。 - decision-log 追加 2026-07-09 决策条目记录本次契约变更。 Co-Authored-By: Claude Opus 4.8 --- docs/project-memory/shared-memory/decision-log.md | 8 ++++++++ .../【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md | 2 +- ...后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index aee4e1725..9062221b7 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -16,6 +16,14 @@ --- +## 2026-07-09 角色动作视频生成背景色统一为多色自动决策 + 阿里云抠帧 + +- 背景：角色动作视频抽帧过去固定 legacy `#00FF00` 绿幕 + 本地 `editor_green_screen`，与生图链路的多色自动决策不一致；实测出现背景色与前景 / 皮肤撞色（蓝撞蓝、桃 / 黄撞肤色）以及图生视频背景变白的问题。 +- 决策：角色动作视频背景色与生图统一。`screenColor=auto` 时由视觉 LLM（`gpt-5-mini`，Responses 协议、`reasoning_effort=low`，`max_tokens=1024`）读源角色图自动决策，并经硬过滤器（Lab 危险质量 + 皮肤专属三判据：ΔE 距离 / 色调投影 / RGB 分离）剔除与前景及皮肤撞色的候选，手动 hex 仍尊重用户选择；透明源角色图在提交 Ark 图生视频前先合成到选定背景色实色，使视频背景确定性等于抠图键色。抽帧后逐帧优先走阿里云通用抠图，失败降级本地 `editor_green_screen` 键色兜底（按生成时选定的背景色，而非固定 `#00FF00`）。BgFilter 与阿里云抠图失败均写入 `external_api_call_failure` 失败审计。调色板新增中明度低饱和「灰竹绿 `#A0BBA0`」补齐冷区绿色段。 +- 影响范围：`server-rs/crates/api-server/src/character_animation_assets.rs`、`editor_screen_background_decision.rs`、`editor_screen_background_filter.rs`（新增硬过滤模块）、`editor_green_screen.rs`（调色板）、`external_api_audit.rs`、`llm_model_routing.rs`、图片画布 MVP 与后端数据契约文档。 +- 验证方式：`cargo test -p api-server editor_screen_background character_animation --manifest-path server-rs/Cargo.toml`、`cargo check -p api-server --manifest-path server-rs/Cargo.toml`、真机对源角色图跑视觉决策与候选危险度表、抽帧后采样序列帧背景色确认落在冷区安全集。 +- 关联文档：`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`、`docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md`。 + ## 2026-07-02 图片画布生成抠图背景色使用 screenColor 传递 - 背景：画布角色、图标和 UI 素材生成过去固定要求 `#00FF00` 绿幕，后续 BGfilter 服务需要按生成时背景色做去背景，不能继续把背景色写死在 prompt 或后处理里。 diff --git a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md index 1b1ac5a2d..bb126820a 100644 --- a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md +++ b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md @@ -21,7 +21,7 @@ - 生成资源右上角显示元数据按钮，点击打开独立元数据窗口。图片信息页不展示后端组装后的生图 Prompt，也不提供复制 Prompt；只展示该图片生成时用户在面板里提交的输入快照，包括普通生成提示词、规范表单字段、角色设定、图标素材描述、快速编辑提示词、重绘提示词，以及角色规范 / 常规参考图 / 图标规范 / 编辑参考图等参考图卡片，并提供“复制信息”复制当前可见字段。参考图输入快照只保存 `refType/refId` 行引用，其中 `refType="project-resource"` 指向 `editor_project_resource.resourceId`，`refType="asset"` 指向 `editor_asset.assetId`；不得把图片 Data URL、普通 URL 或 `objectKey` 写入 `generationInputs.references`。旧数据或上传图片没有输入快照时显示 `-`，禁止回退展示内部 Prompt。 - 对生成资源执行重绘时，在右侧创建新的生成结果图层，并自动调整视图显示原图和新图；重绘面板不因提交成功自动关闭，便于连续改提示词。重绘 / 改造输入框只允许从 `generationInputs.fields` 中恢复用户可见输入快照，例如普通生成提示词、视频描述、音效 `prompt`、背景音乐 `gpt_description_prompt`、角色设定、UI 用户输入、图标素材描述、规范表单和宣发素材字段；禁止回退展示资源 `prompt` / `actualPrompt` 中的后端拼接 Prompt、固定生成模板或模型默认提示词。没有用户输入快照的旧图层打开改造时保持空输入，等待用户重新填写。 - 图片生成 / 修改统一经 api-server BFF 接入 VectorEngine。普通生成、生成规范和重绘保留既有 `gpt-image-2` 路径；图片快速编辑统一打开框选区域 + 单提示词 + 模型选择面板，默认沿用原图模型，不展示参考图或比例 / 尺寸控件；其中生成规范类图片固定 `16:9`、`2K`、`gpt-image-2`，面板底部用与可编辑面板一致的比例 / 尺寸 / 模型胶囊按钮展示固定参数，但按钮为禁用态，不允许在该面板改比例、尺寸或模型。`生成角色形象` 与 `生成图标素材` 支持 `nanobanana2`（`gemini-3.1-flash-image-preview`）和 `gpt-image-2`，默认 `nanobanana2`，并在两类面板之间沿用用户上次选择的模型；两类面板不展示抠图背景色或抠图模型选择；前端用户路径固定提交 `screenColor=auto` 和 `segModel=birefnet`，由后端自动决策具体抠图背景色，`anime-seg` 作为内部保留能力不在用户界面暴露。`nanobanana2` 走 `/v1beta/models/{model}:generateContent`，请求体写入 `generationConfig.imageConfig.aspectRatio/imageSize`；`gpt-image-2` 走 `/v1/images/generations` 或 `/v1/images/edits`，请求体按 VectorEngine 文档映射 `size`。宣发素材三个工作流（游戏首图、详情五图、运营海报）固定使用 `gpt-image-2`，面板模型胶囊为禁用态，不提供 `nanobanana2` 入口；前端按 workflow 同时提交 `outputSize`、`aspectRatio` 和 `imageSize`，其中游戏首图为 `720x540 / 4:3`、详情单图为 `720x1280 / 9:16`、运营海报为 `1280x720 / 16:9`；后端收到 `kind: "publication-material"` 时也强制归一为 `gpt-image-2` 生成和计费，生成回填图层优先使用生成占位的 `originalWidth/originalHeight`，即使上游回包尺寸漂移也不得把宣发素材卡片变成随机 `1:1` 或 `4:3`。纯文本生成走 `/api/editor/images/generations`，重绘在前端读入当前图层图片 Data URL 后走同一图片生成 BFF，并在原图右侧生成一张新图；普通图层重绘作为 `quick-edit` 参考图提交，角色图层重绘必须按 `kind: "character"` 提交，继续套用角色生成器提示词限定、透明 PNG 后处理和角色资产持久化。`生成视频` 走 `/api/editor/videos/generations`，前端模型入口仅展示 Seedance 2.0 Fast / Seedance 2.0 / Kling 3.0 / Kling 3.0 Omni，不展示 Veo 入口，默认 Seedance 2.0 Fast；视频参数按当前正式面板支持的比例、时长、清晰度和声音开关提交，且 Seedance Fast 与 Seedance 标准版必须按各自真实模型 ID 独立映射，不得混用。生成结果以视频图层加入画布。纯文本生成入口采用 Lovart 式画布内占位图 + 锚定生成输入框：点击生成图片后以当前视口世界中心为目标，经统一 placement 避让后创建选中的灰色占位框，输入框跟随占位框显示；待生成、生成中和失败后保留的占位图都必须继续支持拖动，生成完成时真实生成图或视频落在最新占位框位置，输入框继续跟随新生成图层；占位图失焦时隐藏高亮边框、左上角生成器名称和右上角原始尺寸，重新聚焦时再显示，且名称 / 尺寸在画布缩小时按 viewport 反向缩放保持屏幕尺寸稳定；点击所有图片 / 视频生成入口并确认请求开始后，必须隐藏对应设置面板，只保留画布内占位图或原图预览，并在预览上显示 Lovart 式生成中遮罩，避免“面板仍占屏”或“预览一起消失”。图片快速编辑和重绘在调用图片 BFF 前必须把当前图层图片源读取为图片 Data URL；视频素材快速编辑走视频生成 BFF，不允许走图片模型；角色动作的 `生成动画` 仍固定使用 `seedance2.0-fast` 动作 / 视频模型，角色动作素材的 `快速编辑` 按当前帧图片走图片编辑。前端不持有 provider 密钥；上游失败或配置缺失时恢复当前生成设置面板展示失败，不创建 mock 成功图。 -- 图片画布抠图分两类：手动去除背景面向用户任意图片，走登录态同源 BFF `POST /api/editor/images/background-removals` 并转发远端 BiRefNet；编辑器自己生成的标准纯色背景抠图资产在保存源图后统一调用独立 BgFilter 服务 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL/remove-background`，默认 `http://58.87.105.82/bgfilter/remove-background`，默认请求超时 `45000ms`。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的前端用户路径都固定把 `screenColor=auto` 注入请求体，但用户可见 `generationInputs.fields` 不再记录 `抠图背景色` 或 `抠图模型`；api-server 在组装 prompt 前调用背景决策模块，从 11 个候选色中选择具体 hex，最多重试 3 次，失败后兜底 `#CFEFFF`。后端仍保留手动 hex 解析能力供内部兼容。最终生图 prompt 和 BgFilter `screen_color` multipart 字段只接收解析后的具体 hex，不透传 `auto`。三条 BgFilter 路径还必须固定把默认 `segModel=birefnet` 传为 `seg_model`；后端仍保留识别 `anime-seg` 的内部兼容能力，但前端用户入口不展示也不提交该值。这里的 `birefnet` 只是 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。后端在调用 BgFilter 前必须先把带纯色背景 / 绿幕源图写入 OSS；若 BgFilter 请求失败、返回非成功状态、空图片或非法图片，api-server 对这些标准纯色背景生成图使用本地 `editor_green_screen` 按同一 `screenColor` 兜底去背；连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD=3` 后，冷却 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS=300` 秒内直接使用本地兜底。角色动作抽帧后的序列帧暂不接用户背景色选择，仍沿用 legacy `#00FF00` 绿幕和本地 `editor_green_screen` 透明化，不依赖 BiRefNet 或 BgFilter。BiRefNet 手动去背景服务地址为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL/remove-background`，默认 `http://58.87.105.82/remove-background`；BgFilter 可选访问令牌来自 `GENARRATIVE_EDITOR_BGFILTER_TOKEN`，未配置时复用 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN`，所有令牌都只在服务端注入，前端不持有令牌。api-server 对上游结果做响应字节和图片尺寸上限保护，并先落 OSS / asset object，再返回 `imageSrc/objectKey/assetObjectId/taskId`；queue 模式下手动去背景进入 SpacetimeDB 外部生成队列，画布任务侧栏只展示服务器任务阶段，生成中才显示耗时，不显示百分比；有项目上下文时前端同时创建去背景生成占位并把 `canvasCompletion` 交给后端，完成后由后端写入结果图层和最新项目快照。 +- 图片画布抠图分两类：手动去除背景面向用户任意图片，走登录态同源 BFF `POST /api/editor/images/background-removals` 并转发远端 BiRefNet；编辑器自己生成的标准纯色背景抠图资产在保存源图后统一调用独立 BgFilter 服务 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL/remove-background`，默认 `http://58.87.105.82/bgfilter/remove-background`，默认请求超时 `120000ms`（BgFilter CPU 推理）。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的前端用户路径都固定把 `screenColor=auto` 注入请求体，但用户可见 `generationInputs.fields` 不再记录 `抠图背景色` 或 `抠图模型`；api-server 在组装 prompt 前调用背景决策模块，从 11 个候选色中选择具体 hex，最多重试 3 次，失败后兜底 `#CFEFFF`。后端仍保留手动 hex 解析能力供内部兼容。最终生图 prompt 和 BgFilter `screen_color` multipart 字段只接收解析后的具体 hex，不透传 `auto`。三条 BgFilter 路径还必须固定把默认 `segModel=birefnet` 传为 `seg_model`；后端仍保留识别 `anime-seg` 的内部兼容能力，但前端用户入口不展示也不提交该值。这里的 `birefnet` 只是 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。后端在调用 BgFilter 前必须先把带纯色背景 / 绿幕源图写入 OSS；若 BgFilter 请求失败、返回非成功状态、空图片或非法图片，api-server 对这些标准纯色背景生成图使用本地 `editor_green_screen` 按同一 `screenColor` 兜底去背；连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD=3` 后，冷却 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS=300` 秒内直接使用本地兜底。角色动作生成的序列帧背景色已与生图统一：前端固定提交 `screenColor=auto`，后端视觉决策出具体 hex 并把源角色图合成到该背景色后再图生视频；抽帧后逐帧优先阿里云通用抠图，失败降级本地 `editor_green_screen`（按选定背景色，而非固定 `#00FF00`）。BiRefNet 手动去背景服务地址为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL/remove-background`，默认 `http://58.87.105.82/remove-background`；BgFilter 可选访问令牌来自 `GENARRATIVE_EDITOR_BGFILTER_TOKEN`，未配置时复用 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN`，所有令牌都只在服务端注入，前端不持有令牌。api-server 对上游结果做响应字节和图片尺寸上限保护，并先落 OSS / asset object，再返回 `imageSrc/objectKey/assetObjectId/taskId`；queue 模式下手动去背景进入 SpacetimeDB 外部生成队列，画布任务侧栏只展示服务器任务阶段，生成中才显示耗时，不显示百分比；有项目上下文时前端同时创建去背景生成占位并把 `canvasCompletion` 交给后端，完成后由后端写入结果图层和最新项目快照。 - 图片快速编辑面板只保留一个提示词输入框和模型选择，不展示额外参考图或比例 / 尺寸控件；原图 / 原素材作为 `/api/editor/images/edits` 的 `sourceImageSrc` 直接提交，不作为 `referenceImageSrcs`。打开快速编辑时画布必须自动平移缩放，让原素材完整落在可视区上半部分，底部面板固定出现在素材下方且不遮挡内容，竖屏 UI 素材也必须完整展示。快速编辑右侧显示矩形、椭圆、画笔框选工具，但进入时不默认启用；点击工具后显示选中态，再点同一工具取消启用。完成框选后，画布红色细框显示连续序号，提示词可按这些编号填写每个区域怎么改。点击 `修改` 后仍停留在当前快速编辑面板显示修改中，不创建独立 `Quick Edit Generator` 画布占位；生成成功后直接用结果覆盖原图图层，失败时保留当前面板并显示错误。 - 底部生成类按钮每次点击都必须创建独立的画布生成对象；新建规范、角色形象或图标素材时，只切换当前编辑面板，不得销毁此前尚未生成或已生成后的其它生成对象状态。归档为非当前编辑对象的生成占位仍可拖动、删除和等待异步完成，完成 / 失败回写必须按生成对象 ID 读取最新占位状态，不能使用提交瞬间的旧快照。 - 画布右上角提供自动隐藏任务侧栏。列表为空且侧栏关闭时只保留图标开关；生成或去背景任务进入时默认打开；用户可手动切换开关状态。 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index 39d2e59f4..c323c1e7a 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -220,7 +220,7 @@ npm run check:server-rs-ddd - LLM：通用 LLM 门面继续使用 `GENARRATIVE_LLM_*`；创意 Agent `gpt-5.4-mini` Chat Completions 文本链路已于 2026-06 从 APIMart 迁移到 VectorEngine，使用 `VECTOR_ENGINE_BASE_URL` / `VECTOR_ENGINE_API_KEY` 构造 OpenAI-compatible client，`api-server` 会把未带 `/v1` 的 VectorEngine base URL 规范化到 `/v1` 后请求 `/chat/completions`。通用 `/api/llm/chat/completions` 代理使用 `GENARRATIVE_LLM_PROVIDER=openai-compatible`、`GENARRATIVE_LLM_BASE_URL=https://api.vectorengine.cn/v1`、`GENARRATIVE_LLM_MODEL=gpt-5.4-mini`；未单独配置 `GENARRATIVE_LLM_API_KEY` 时可复用 `VECTOR_ENGINE_API_KEY`。`APIMART_BASE_URL` / `APIMART_API_KEY` 只作为历史残留，不再作为创意 Agent gpt-5.4-mini 客户端来源；后续排障时优先确认 VectorEngine `/v1/models`、`/v1/chat/completions` 和 `/v1/responses` 可用性。 - 图片生成：VectorEngine `gpt-image-2` 图片 provider 归属 `platform-image`，密钥只在后端环境变量中；`api-server` 内的 `openai_image_generation.rs` 只是兼容调用面和外部失败审计桥接，不再承载 provider 协议实现。实际外部生成运行记录统一落 `tracking_event`，`event_key = external_generation_run`，metadata 记录开始 / 结束时间、耗时、状态、成功标记、失败原因、provider task id 和结果摘要，不再写回过时的 `ai_task`。DashScope 只按仍在使用的历史能力单独处理，不作为 GPT-image-2 兜底。VectorEngine `/v1/images/generations` 和 `/v1/images/edits` 上游 POST 使用 `libcurl` 发送；`reqwest` 只保留给参考图 URL 下载和响应中图片 URL 下载。`/v1/images/edits` 的 multipart 参考图必须作为 libcurl 文件上传 part 发送，字段名为 `image`，实现上使用 `Form::buffer(file_name, bytes)` 并设置 `Content-Type`；不能只用 `contents(...).filename(...)`，否则上游会把请求转码为缺少图片并返回 `image is required`。`request_send` 阶段的 curl timeout / connect error 按可重试传输错误处理，最多尝试 5 次，并使用指数退避加短抖动；排障时优先看 `attempt`、`max_attempts`、`retry_delay_ms`、`reference_image_bytes_total` 和 `request_params`，不要把 `SendRequest` 当成上游业务错误。 -- 编辑器抠图服务：手动 `POST /api/editor/images/background-removals` 继续代理独立 BiRefNet 服务，配置为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL`、`GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN` 和 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_REQUEST_TIMEOUT_MS`。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的生成后纯色背景透明化改走独立 BgFilter 服务，配置为 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL`、`GENARRATIVE_EDITOR_BGFILTER_TOKEN` 和 `GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS`，默认 base URL 为 `http://58.87.105.82/bgfilter`，默认请求超时为 `45000ms`，token 未配置时复用 BiRefNet token。BgFilter 请求必须显式传 `screen_color=` 和 `seg_model=`；前端用户路径不展示抠图模型选择并固定提交默认 `birefnet`，后端仍识别内部保留的 `anime-seg`，其中 `birefnet` 只表示 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。BgFilter 连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD`（默认 `3`）后，会在 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS`（默认 `300`）内直接使用本地 `editor_green_screen` 兜底，避免上游故障占住 worker。角色动作抽帧仍沿用 legacy `#00FF00` 和本地 `editor_green_screen` 透明化。 +- 编辑器抠图服务：手动 `POST /api/editor/images/background-removals` 继续代理独立 BiRefNet 服务，配置为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL`、`GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN` 和 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_REQUEST_TIMEOUT_MS`。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的生成后纯色背景透明化改走独立 BgFilter 服务，配置为 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL`、`GENARRATIVE_EDITOR_BGFILTER_TOKEN` 和 `GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS`，默认 base URL 为 `http://58.87.105.82/bgfilter`，默认请求超时为 `120000ms`（BgFilter 当前为 CPU 推理，单次抠图较慢，必须留足超时），token 未配置时复用 BiRefNet token。BgFilter 请求必须显式传 `screen_color=` 和 `seg_model=`；前端用户路径不展示抠图模型选择并固定提交默认 `birefnet`，后端仍识别内部保留的 `anime-seg`，其中 `birefnet` 只表示 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。BgFilter 连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD`（默认 `3`）后，会在 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS`（默认 `300`）内直接使用本地 `editor_green_screen` 兜底，避免上游故障占住 worker。角色动作视频生成的背景色已与生图链路统一：`screenColor=auto` 时由视觉 LLM（`gpt-5-mini`，Responses 协议、low 推理档）读源角色图自动决策，并经硬过滤器剔除与前景 / 皮肤撞色的候选，手动 hex 则尊重用户选择；透明源角色图在提交 Ark 图生视频前先合成到选定背景色实色，使视频背景等于抠图键色。抽帧后逐帧优先走阿里云通用抠图，失败时降级本地 `editor_green_screen` 键色兜底（按生成时选定的背景色，而非固定 `#00FF00`）；BgFilter 与阿里云抠图失败都写入 `external_api_call_failure` 审计。 - Match3D 物品 sheet：关卡整图完成后走 VectorEngine `/v1/images/edits` multipart `image`，模型为 `gpt-image-2`，`2K 1:1` 输出 `10*10` spritesheet；物品 sheet prompt 固定要求单一纯绿色 `#00FF00 / RGB(0,255,0)` 绿幕背景，后端上传 OSS 前必须把绿幕扣成透明 PNG，并把透明整图写入 `itemSpritesheetImageSrc/itemSpritesheetImageObjectKey`。后端优先按透明 alpha 连通域从该 sheet 识别真实素材矩形并持久化 20 个物品、每个 5 个形态；识别数量不足时才回退 `10*10` 固定网格。通用系列素材图集的行列索引按每行 2 个物品计算，必须落在 `1..=10`，难度只决定运行态加载 3 / 9 / 15 / 20 种。 - Match3D UI spritesheet 和背景派生图：关卡整图作为参考图并发生成 `1K 1:1` UI spritesheet 与 `1K 9:16` 背景图，模型均为 `gpt-image-2`。UI spritesheet prompt 固定要求单一纯绿色 `#00FF00 / RGB(0,255,0)` 绿幕背景，后端上传 OSS 前必须把绿幕扣成透明 PNG；背景图必须合成为全画幅不透明 PNG。 - Match3D 1:1 容器 UI：VectorEngine `/v1/images/edits` multipart 参考图。该容器参考图是后端生图协议输入，必须通过 `include_bytes!` 随 `api-server` 编译进二进制，避免 API 单独发布或运行目录缺少 `public/` 时生成失败。 -- 2.52.0 From b8c2efaf2496293d1ef04f4432833a14acebad09 Mon Sep 17 00:00:00 2001 From: Linghong Date: Thu, 9 Jul 2026 03:52:43 +0000 Subject: [PATCH 12/41] =?UTF-8?q?=E6=8A=A0=E5=9B=BE=E8=83=8C=E6=99=AF?= =?UTF-8?q?=E8=89=B2=E5=8F=AA=E4=BD=9C=E5=86=85=E9=83=A8=20screenColorHex?= =?UTF-8?q?=20=E5=85=83=E6=95=B0=E6=8D=AE=EF=BC=8C=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E6=9A=B4=E9=9C=B2=E8=BF=9B=E7=94=A8=E6=88=B7=E5=8F=AF=E8=A7=81?= =?UTF-8?q?=20fields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apply_editor_screen_background_decision_to_generation_inputs 之前会往 generationInputs.fields 追加「抠图背景色」条目，把内部自动决策/兜底细节 暴露给用户，违反「用户可见输入快照不记录抠图背景色/抠图模型」的产品契约（P2）。 - 去掉 fields 注入，只保留顶层内部可追踪元数据 screenColorHex（不进用户快照）。 - 删除因此变成死代码的 format_editor_screen_background_decision_input 及其 import。 - 重写两个单测：断言 fields 不被注入/不被修改，只写 screenColorHex。 Co-Authored-By: Claude Opus 4.8 --- .../crates/api-server/src/editor_project.rs | 33 +++++++------------ .../src/editor_screen_background_decision.rs | 16 --------- 2 files changed, 11 insertions(+), 38 deletions(-) diff --git a/server-rs/crates/api-server/src/editor_project.rs b/server-rs/crates/api-server/src/editor_project.rs index eea999ee6..029e9ff85 100644 --- a/server-rs/crates/api-server/src/editor_project.rs +++ b/server-rs/crates/api-server/src/editor_project.rs @@ -64,8 +64,7 @@ use crate::{ }, editor_screen_background_decision::{ EditorScreenBackgroundDecision, EditorScreenBackgroundDecisionInput, - EditorScreenBackgroundDecisionKind, format_editor_screen_background_decision_input, - resolve_editor_screen_background_color, + EditorScreenBackgroundDecisionKind, resolve_editor_screen_background_color, }, generated_image_assets::{ GeneratedImageAssetAdapter, GeneratedImageAssetDataUrl, @@ -1673,18 +1672,8 @@ pub(crate) fn apply_editor_screen_background_decision_to_generation_inputs( Some(other) => return Some(other), None => json!({ "fields": [], "references": [] }), }; - let resolved_value = format_editor_screen_background_decision_input(decision); - if let Some(fields) = value.get_mut("fields").and_then(Value::as_array_mut) { - if let Some(field) = fields - .iter_mut() - .find(|field| field.get("title").and_then(Value::as_str) == Some("抠图背景色")) - { - field["value"] = Value::String(resolved_value); - } else { - fields.push(json!({ "title": "抠图背景色", "value": resolved_value })); - } - } - // 结构化背景色，供旧背景色先验等程序化消费；展示字段的中文格式不可作解析依据。 + // 抠图背景色是内部自动决策 / 兜底细节，产品契约要求不进用户可见的 generationInputs.fields； + // 只写顶层内部可追踪元数据 screenColorHex（不属于用户输入快照，前端展示不读取）。 value["screenColorHex"] = Value::String(decision.color.hex.to_string()); Some(value) } @@ -5680,9 +5669,10 @@ mod tests { } #[test] - fn screen_background_decision_generation_inputs_record_structured_hex() { + fn screen_background_decision_records_hex_without_touching_visible_fields() { let decision = manual_screen_background_decision("#B0C2E0"); + // 用户可见的 fields 不被注入抠图背景色，只写内部 screenColorHex。 let updated = apply_editor_screen_background_decision_to_generation_inputs( Some(json!({ "fields": [{ "title": "角色设定", "value": "红发骑士" }], @@ -5694,19 +5684,17 @@ mod tests { assert_eq!(updated["screenColorHex"], json!("#B0C2E0")); assert_eq!( updated["fields"], - json!([ - { "title": "角色设定", "value": "红发骑士" }, - { "title": "抠图背景色", "value": "浅钢蓝 #B0C2E0" }, - ]), + json!([{ "title": "角色设定", "value": "红发骑士" }]), ); + // 无 generation_inputs 时创建骨架并写入内部元数据，fields 保持为空。 let created = apply_editor_screen_background_decision_to_generation_inputs(None, Some(&decision)) .expect("missing generation inputs should still record the decision"); assert_eq!( created, json!({ - "fields": [{ "title": "抠图背景色", "value": "浅钢蓝 #B0C2E0" }], + "fields": [], "references": [], "screenColorHex": "#B0C2E0", }), @@ -5714,9 +5702,10 @@ mod tests { } #[test] - fn screen_background_decision_generation_inputs_overwrite_existing_display_field() { + fn screen_background_decision_leaves_existing_visible_fields_untouched() { let decision = manual_screen_background_decision("#CFEFFF"); + // 即使旧数据的 fields 里残留了抠图背景色，后端也不修改用户可见字段。 let updated = apply_editor_screen_background_decision_to_generation_inputs( Some(json!({ "fields": [{ "title": "抠图背景色", "value": "旧值" }], @@ -5727,7 +5716,7 @@ mod tests { .expect("generation inputs should be preserved"); assert_eq!( updated["fields"], - json!([{ "title": "抠图背景色", "value": "浅雾蓝 #CFEFFF" }]), + json!([{ "title": "抠图背景色", "value": "旧值" }]), ); assert_eq!(updated["screenColorHex"], json!("#CFEFFF")); } diff --git a/server-rs/crates/api-server/src/editor_screen_background_decision.rs b/server-rs/crates/api-server/src/editor_screen_background_decision.rs index 48c47c27f..12f2bb8a1 100644 --- a/server-rs/crates/api-server/src/editor_screen_background_decision.rs +++ b/server-rs/crates/api-server/src/editor_screen_background_decision.rs @@ -278,22 +278,6 @@ pub(crate) fn is_auto_screen_background_color(value: Option<&str>) -> bool { .unwrap_or(true) } -pub(crate) fn format_editor_screen_background_decision_input( - decision: &EditorScreenBackgroundDecision, -) -> String { - let resolved = format!("{} {}", decision.color.label, decision.color.hex); - match decision.mode { - EditorScreenBackgroundDecisionMode::Manual => resolved, - EditorScreenBackgroundDecisionMode::Auto => { - if decision.fallback { - format!("自动 -> {resolved}（兜底）") - } else { - format!("自动 -> {resolved}") - } - } - } -} - fn editor_screen_background_decision_system_prompt() -> &'static str { "你是游戏素材生图的抠图背景色决策器。根据任务和用户输入，从候选列表中选择一个最不容易与主体混淆、最适合后续扣除的纯色背景。如果附带了主体参考图，必须以图中主体（含服饰、道具、光效）的实际配色为准，选择色相距离最远的候选色，忽略图中已有的背景或透明区域。只能返回 JSON：{\"hex\":\"#RRGGBB\"}。不要解释，不要返回候选列表外的颜色。" } -- 2.52.0 From a2b64290a1b0ede955231889770ccb97bf7dd65a Mon Sep 17 00:00:00 2001 From: Linghong Date: Thu, 9 Jul 2026 04:32:11 +0000 Subject: [PATCH 13/41] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=8A=A0=E5=9B=BE?= =?UTF-8?q?=E4=B8=8A=E6=B5=B7=E8=87=AA=E6=9C=89=20OSS=20=E6=AD=BB=E5=88=86?= =?UTF-8?q?=E6=94=AF=E5=B9=B6=E6=B8=85=E7=90=86=E5=AE=A1=E6=9F=A5=E5=8F=91?= =?UTF-8?q?=E7=8E=B0=E7=9A=84=E6=AD=BB=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 代码审查发现 4 项问题，本次一并处理： - 抠图输入"上海地域自有 OSS 上传"分支整体删除：我们没有上海地域 OSS，region marker 判定永远为假，该分支任何环境都不会触发，且其 matting-input/ 中间对象无清理会造成存储泄漏（若将来启用）。抠图 输入统一走 VIAPI 官方临时桶（1 天自动过期，天然无泄漏）。 - platform-matting：segment_image_to_transparent_png 去掉恒为 None 的 accessible_url 参数，简化提交分支；修正"生产建议用上海自有 OSS"的误导注释。 - editor_green_screen：删除不可达的 #00FF00 绿幕特判分支（legacy 绿幕色产生点已删、调色板无此色）。 - segment_smoke example：默认参数里的个人路径改为用法提示 + 退出。 Co-Authored-By: Claude Opus 4.8 --- .../crates/api-server/src/aliyun_matting.rs | 80 +------------------ .../api-server/src/editor_green_screen.rs | 3 - .../examples/segment_smoke.rs | 3 +- server-rs/crates/platform-matting/src/lib.rs | 31 +++---- 4 files changed, 15 insertions(+), 102 deletions(-) diff --git a/server-rs/crates/api-server/src/aliyun_matting.rs b/server-rs/crates/api-server/src/aliyun_matting.rs index 79fc2e986..3746f2f1c 100644 --- a/server-rs/crates/api-server/src/aliyun_matting.rs +++ b/server-rs/crates/api-server/src/aliyun_matting.rs @@ -1,24 +1,15 @@ //! 阿里云通用抠图在 api-server 侧的适配层。 //! -//! 输入 URL 策略：自有 OSS 在上海地域时直接上传草稿区并签名读 URL（抠图服务只认 -//! 上海地域 OSS）；其他地域交给 platform-matting 走 VIAPI 官方临时桶。 - -use std::collections::BTreeMap; +//! 输入 URL 策略：抠图服务只认上海地域 OSS URL，而我们没有上海地域自有 OSS， +//! 统一由 platform-matting 上传 VIAPI 官方临时桶（1 天自动过期，无需清理）。 use axum::http::StatusCode; -use platform_oss::{ - LegacyAssetPrefix, OssObjectAccess, OssPutObjectRequest, OssSignedGetObjectUrlRequest, -}; use serde_json::json; use crate::{ http_error::AppError, openai_image_generation::DownloadedOpenAiImage, state::AppState, }; -const MATTING_INPUT_PATH_SEGMENT: &str = "matting-input"; -const MATTING_INPUT_READ_EXPIRE_SECONDS: u64 = 10 * 60; -const ALIYUN_MATTING_OSS_REGION_MARKER: &str = "oss-cn-shanghai"; - /// 图片字节 → 阿里云通用抠图 → 原尺寸透明 PNG。 /// 未配置抠图客户端时返回错误，由调用方决定是否降级本地算法。 pub(crate) async fn segment_image_with_aliyun_matting( @@ -33,11 +24,10 @@ pub(crate) async fn segment_image_with_aliyun_matting( })) })?; - let accessible_url = prepare_shanghai_oss_input_url(state, image, log_label).await; let file_name = format!("{log_label}.png"); let started_at = std::time::Instant::now(); let output_bytes = matting_client - .segment_image_to_transparent_png(image.bytes.as_slice(), &file_name, accessible_url) + .segment_image_to_transparent_png(image.bytes.as_slice(), &file_name) .await .map_err(|error| { AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ @@ -58,67 +48,3 @@ pub(crate) async fn segment_image_with_aliyun_matting( extension: "png".to_string(), }) } - -/// 自有 OSS 在上海地域时，把待抠图图片传到草稿区并返回签名读 URL； -/// 其他情况返回 None（platform-matting 会转走官方临时桶）。 -/// 该步骤失败不阻断抠图，只是退化为临时桶通道。 -async fn prepare_shanghai_oss_input_url( - state: &AppState, - image: &DownloadedOpenAiImage, - log_label: &str, -) -> Option { - let oss_client = state.oss_client()?; - let endpoint = state.config.oss_endpoint.as_deref().unwrap_or(""); - if !endpoint.contains(ALIYUN_MATTING_OSS_REGION_MARKER) { - return None; - } - - let http_client = reqwest::Client::new(); - let file_name = format!( - "{log_label}-{}.{}", - shared_kernel::new_uuid_simple_string(), - image.extension - ); - let put_result = oss_client - .put_object( - &http_client, - OssPutObjectRequest { - prefix: LegacyAssetPrefix::CharacterDrafts, - path_segments: vec![MATTING_INPUT_PATH_SEGMENT.to_string()], - file_name, - content_type: Some(image.mime_type.clone()), - access: OssObjectAccess::Private, - metadata: BTreeMap::new(), - body: image.bytes.clone(), - }, - ) - .await; - let put_result = match put_result { - Ok(result) => result, - Err(error) => { - tracing::warn!( - provider = "aliyun-matting", - log_label, - error = %error, - "抠图输入图上传自有 OSS 失败，回退官方临时桶" - ); - return None; - } - }; - - match oss_client.sign_get_object_url(OssSignedGetObjectUrlRequest { - object_key: put_result.object_key, - expire_seconds: Some(MATTING_INPUT_READ_EXPIRE_SECONDS), - }) { - Ok(signed) => Some(signed.signed_url), - Err(error) => { - tracing::warn!( - provider = "aliyun-matting", - log_label, - error = %error, - "抠图输入图签名读 URL 失败，回退官方临时桶" - ); - None - } - } -} diff --git a/server-rs/crates/api-server/src/editor_green_screen.rs b/server-rs/crates/api-server/src/editor_green_screen.rs index 34fa243f0..5840533de 100644 --- a/server-rs/crates/api-server/src/editor_green_screen.rs +++ b/server-rs/crates/api-server/src/editor_green_screen.rs @@ -141,9 +141,6 @@ pub(crate) const EDITOR_GREEN_SCREEN_CHARACTER_GUARDRAILS: &str = "角色主体及其服饰、道具的颜色必须与背景色明显区分，不得带与背景色相同或相近的描边、投影或反光"; fn editor_screen_background_color_prompt(color: EditorScreenBackgroundColor) -> String { - if color.hex == "#00FF00" { - return "单一纯绿色 #00FF00 / RGB(0,255,0) 绿幕".to_string(); - } format!( "单一纯色背景 {} {} / RGB({},{},{})", color.label, color.hex, color.red, color.green, color.blue diff --git a/server-rs/crates/platform-matting/examples/segment_smoke.rs b/server-rs/crates/platform-matting/examples/segment_smoke.rs index 82d08639d..c778b5f02 100644 --- a/server-rs/crates/platform-matting/examples/segment_smoke.rs +++ b/server-rs/crates/platform-matting/examples/segment_smoke.rs @@ -33,7 +33,8 @@ async fn main() { load_env_files(); let input_path = std::env::args().nth(1).unwrap_or_else(|| { - r"C:\Users\lingh\Downloads\gpt-image-2-elf-archer-2048x2048.png".to_string() + eprintln!("用法：cargo run -p platform-matting --example segment_smoke -- <图片路径>"); + std::process::exit(1); }); let input_bytes = std::fs::read(&input_path) .unwrap_or_else(|error| panic!("读取测试图片失败（{input_path}）：{error}")); diff --git a/server-rs/crates/platform-matting/src/lib.rs b/server-rs/crates/platform-matting/src/lib.rs index 0a61e314d..42f38ee7b 100644 --- a/server-rs/crates/platform-matting/src/lib.rs +++ b/server-rs/crates/platform-matting/src/lib.rs @@ -18,8 +18,8 @@ pub const DEFAULT_IMAGESEG_ENDPOINT: &str = "imageseg.cn-shanghai.aliyuncs.com"; const IMAGESEG_API_VERSION: &str = "2019-12-30"; const SEGMENT_COMMON_IMAGE_ACTION: &str = "SegmentCommonImage"; -// VIAPI 官方临时上传通道：非上海地域 OSS / 本地文件先传到官方临时桶（1 天过期， -// 全用户共享 QPS，生产建议直接用上海地域自有 OSS）。 +// VIAPI 官方临时上传通道：抠图输入统一先传官方临时桶（1 天自动过期，无需清理； +// 全用户共享 QPS）。抠图服务只认上海地域 OSS URL，而我们没有上海地域自有 OSS。 // 文档：https://help.aliyun.com/document_detail/155645.html const VIAPI_UTILS_ENDPOINT: &str = "viapiutils.cn-shanghai.aliyuncs.com"; const VIAPI_UTILS_VERSION: &str = "2020-04-01"; @@ -282,15 +282,13 @@ impl MattingClient { /// 图片字节 → 通用抠图 → 原尺寸透明 PNG 字节。 /// - /// - `accessible_url`：该字节对应的、抠图服务可访问的 URL（上海地域 OSS 签名 URL）。 - /// 为 None 或图片超分辨率上限需要缩图时，自动改走官方临时桶上传。 + /// - 输入统一上传 VIAPI 官方临时桶（1 天自动过期，无需清理）后作为 ImageURL 送抠。 /// - 分辨率守卫：任一边 >= 2000 时先等比缩小送抠，抠完只取结果 alpha 上采样回贴 /// 原图 RGB，保证输出与输入同尺寸、画质无损。 pub async fn segment_image_to_transparent_png( &self, bytes: &[u8], file_name: &str, - accessible_url: Option, ) -> Result, MattingError> { let source = image::load_from_memory(bytes).map_err(|error| { MattingError::InvalidRequest(format!("解析待抠图图片失败：{error}")) @@ -299,28 +297,19 @@ impl MattingClient { let (source_width, source_height) = source_rgba.dimensions(); let needs_resize = source_width > MAX_INPUT_EDGE || source_height > MAX_INPUT_EDGE; - let (submit_bytes, submit_url) = if needs_resize { + let upload_bytes = if needs_resize { let resized = source.resize( MAX_INPUT_EDGE, MAX_INPUT_EDGE, image::imageops::FilterType::CatmullRom, ); - let resized_bytes = encode_rgba_png(&resized.to_rgba8())?; - (Some(resized_bytes), None) + encode_rgba_png(&resized.to_rgba8())? } else { - (None, accessible_url) - }; - - let image_url = match submit_url { - Some(url) => url, - None => { - let upload_bytes = submit_bytes - .clone() - .unwrap_or_else(|| bytes.to_vec()); - self.upload_temp_image(upload_bytes, file_name, "image/png") - .await? - } + bytes.to_vec() }; + let image_url = self + .upload_temp_image(upload_bytes, file_name, "image/png") + .await?; let result = self .segment_common_image(SegmentCommonImageRequest { @@ -385,7 +374,7 @@ impl MattingClient { } /// 把本地图片字节上传到 VIAPI 官方临时桶，返回可直接作为 ImageURL 的公网地址。 - /// 适用于本地文件或非上海地域 OSS 的开发/调试场景；生产建议用上海地域自有 OSS。 + /// 抠图输入的统一上传通道（我们没有上海地域自有 OSS，临时桶 1 天自动过期无需清理）。 pub async fn upload_temp_image( &self, bytes: Vec, -- 2.52.0 From 14a5cde9dd7110228a1e9d69c4282e89e7317af5 Mon Sep 17 00:00:00 2001 From: Linghong Date: Thu, 9 Jul 2026 06:30:45 +0000 Subject: [PATCH 14/41] Increase BGfilter TIMEOUT to fit 2048x2048 Graphs --- .../【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md | 2 +- ...【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md | 2 +- server-rs/crates/api-server/src/config.rs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md index bb126820a..d6a068814 100644 --- a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md +++ b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md @@ -21,7 +21,7 @@ - 生成资源右上角显示元数据按钮，点击打开独立元数据窗口。图片信息页不展示后端组装后的生图 Prompt，也不提供复制 Prompt；只展示该图片生成时用户在面板里提交的输入快照，包括普通生成提示词、规范表单字段、角色设定、图标素材描述、快速编辑提示词、重绘提示词，以及角色规范 / 常规参考图 / 图标规范 / 编辑参考图等参考图卡片，并提供“复制信息”复制当前可见字段。参考图输入快照只保存 `refType/refId` 行引用，其中 `refType="project-resource"` 指向 `editor_project_resource.resourceId`，`refType="asset"` 指向 `editor_asset.assetId`；不得把图片 Data URL、普通 URL 或 `objectKey` 写入 `generationInputs.references`。旧数据或上传图片没有输入快照时显示 `-`，禁止回退展示内部 Prompt。 - 对生成资源执行重绘时，在右侧创建新的生成结果图层，并自动调整视图显示原图和新图；重绘面板不因提交成功自动关闭，便于连续改提示词。重绘 / 改造输入框只允许从 `generationInputs.fields` 中恢复用户可见输入快照，例如普通生成提示词、视频描述、音效 `prompt`、背景音乐 `gpt_description_prompt`、角色设定、UI 用户输入、图标素材描述、规范表单和宣发素材字段；禁止回退展示资源 `prompt` / `actualPrompt` 中的后端拼接 Prompt、固定生成模板或模型默认提示词。没有用户输入快照的旧图层打开改造时保持空输入，等待用户重新填写。 - 图片生成 / 修改统一经 api-server BFF 接入 VectorEngine。普通生成、生成规范和重绘保留既有 `gpt-image-2` 路径；图片快速编辑统一打开框选区域 + 单提示词 + 模型选择面板，默认沿用原图模型，不展示参考图或比例 / 尺寸控件；其中生成规范类图片固定 `16:9`、`2K`、`gpt-image-2`，面板底部用与可编辑面板一致的比例 / 尺寸 / 模型胶囊按钮展示固定参数，但按钮为禁用态，不允许在该面板改比例、尺寸或模型。`生成角色形象` 与 `生成图标素材` 支持 `nanobanana2`（`gemini-3.1-flash-image-preview`）和 `gpt-image-2`，默认 `nanobanana2`，并在两类面板之间沿用用户上次选择的模型；两类面板不展示抠图背景色或抠图模型选择；前端用户路径固定提交 `screenColor=auto` 和 `segModel=birefnet`，由后端自动决策具体抠图背景色，`anime-seg` 作为内部保留能力不在用户界面暴露。`nanobanana2` 走 `/v1beta/models/{model}:generateContent`，请求体写入 `generationConfig.imageConfig.aspectRatio/imageSize`；`gpt-image-2` 走 `/v1/images/generations` 或 `/v1/images/edits`，请求体按 VectorEngine 文档映射 `size`。宣发素材三个工作流（游戏首图、详情五图、运营海报）固定使用 `gpt-image-2`，面板模型胶囊为禁用态，不提供 `nanobanana2` 入口；前端按 workflow 同时提交 `outputSize`、`aspectRatio` 和 `imageSize`，其中游戏首图为 `720x540 / 4:3`、详情单图为 `720x1280 / 9:16`、运营海报为 `1280x720 / 16:9`；后端收到 `kind: "publication-material"` 时也强制归一为 `gpt-image-2` 生成和计费，生成回填图层优先使用生成占位的 `originalWidth/originalHeight`，即使上游回包尺寸漂移也不得把宣发素材卡片变成随机 `1:1` 或 `4:3`。纯文本生成走 `/api/editor/images/generations`，重绘在前端读入当前图层图片 Data URL 后走同一图片生成 BFF，并在原图右侧生成一张新图；普通图层重绘作为 `quick-edit` 参考图提交，角色图层重绘必须按 `kind: "character"` 提交，继续套用角色生成器提示词限定、透明 PNG 后处理和角色资产持久化。`生成视频` 走 `/api/editor/videos/generations`，前端模型入口仅展示 Seedance 2.0 Fast / Seedance 2.0 / Kling 3.0 / Kling 3.0 Omni，不展示 Veo 入口，默认 Seedance 2.0 Fast；视频参数按当前正式面板支持的比例、时长、清晰度和声音开关提交，且 Seedance Fast 与 Seedance 标准版必须按各自真实模型 ID 独立映射，不得混用。生成结果以视频图层加入画布。纯文本生成入口采用 Lovart 式画布内占位图 + 锚定生成输入框：点击生成图片后以当前视口世界中心为目标，经统一 placement 避让后创建选中的灰色占位框，输入框跟随占位框显示；待生成、生成中和失败后保留的占位图都必须继续支持拖动，生成完成时真实生成图或视频落在最新占位框位置，输入框继续跟随新生成图层；占位图失焦时隐藏高亮边框、左上角生成器名称和右上角原始尺寸，重新聚焦时再显示，且名称 / 尺寸在画布缩小时按 viewport 反向缩放保持屏幕尺寸稳定；点击所有图片 / 视频生成入口并确认请求开始后，必须隐藏对应设置面板，只保留画布内占位图或原图预览，并在预览上显示 Lovart 式生成中遮罩，避免“面板仍占屏”或“预览一起消失”。图片快速编辑和重绘在调用图片 BFF 前必须把当前图层图片源读取为图片 Data URL；视频素材快速编辑走视频生成 BFF，不允许走图片模型；角色动作的 `生成动画` 仍固定使用 `seedance2.0-fast` 动作 / 视频模型，角色动作素材的 `快速编辑` 按当前帧图片走图片编辑。前端不持有 provider 密钥；上游失败或配置缺失时恢复当前生成设置面板展示失败，不创建 mock 成功图。 -- 图片画布抠图分两类：手动去除背景面向用户任意图片，走登录态同源 BFF `POST /api/editor/images/background-removals` 并转发远端 BiRefNet；编辑器自己生成的标准纯色背景抠图资产在保存源图后统一调用独立 BgFilter 服务 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL/remove-background`，默认 `http://58.87.105.82/bgfilter/remove-background`，默认请求超时 `120000ms`（BgFilter CPU 推理）。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的前端用户路径都固定把 `screenColor=auto` 注入请求体，但用户可见 `generationInputs.fields` 不再记录 `抠图背景色` 或 `抠图模型`；api-server 在组装 prompt 前调用背景决策模块，从 11 个候选色中选择具体 hex，最多重试 3 次，失败后兜底 `#CFEFFF`。后端仍保留手动 hex 解析能力供内部兼容。最终生图 prompt 和 BgFilter `screen_color` multipart 字段只接收解析后的具体 hex，不透传 `auto`。三条 BgFilter 路径还必须固定把默认 `segModel=birefnet` 传为 `seg_model`；后端仍保留识别 `anime-seg` 的内部兼容能力，但前端用户入口不展示也不提交该值。这里的 `birefnet` 只是 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。后端在调用 BgFilter 前必须先把带纯色背景 / 绿幕源图写入 OSS；若 BgFilter 请求失败、返回非成功状态、空图片或非法图片，api-server 对这些标准纯色背景生成图使用本地 `editor_green_screen` 按同一 `screenColor` 兜底去背；连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD=3` 后，冷却 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS=300` 秒内直接使用本地兜底。角色动作生成的序列帧背景色已与生图统一：前端固定提交 `screenColor=auto`，后端视觉决策出具体 hex 并把源角色图合成到该背景色后再图生视频；抽帧后逐帧优先阿里云通用抠图，失败降级本地 `editor_green_screen`（按选定背景色，而非固定 `#00FF00`）。BiRefNet 手动去背景服务地址为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL/remove-background`，默认 `http://58.87.105.82/remove-background`；BgFilter 可选访问令牌来自 `GENARRATIVE_EDITOR_BGFILTER_TOKEN`，未配置时复用 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN`，所有令牌都只在服务端注入，前端不持有令牌。api-server 对上游结果做响应字节和图片尺寸上限保护，并先落 OSS / asset object，再返回 `imageSrc/objectKey/assetObjectId/taskId`；queue 模式下手动去背景进入 SpacetimeDB 外部生成队列，画布任务侧栏只展示服务器任务阶段，生成中才显示耗时，不显示百分比；有项目上下文时前端同时创建去背景生成占位并把 `canvasCompletion` 交给后端，完成后由后端写入结果图层和最新项目快照。 +- 图片画布抠图分两类：手动去除背景面向用户任意图片，走登录态同源 BFF `POST /api/editor/images/background-removals` 并转发远端 BiRefNet；编辑器自己生成的标准纯色背景抠图资产在保存源图后统一调用独立 BgFilter 服务 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL/remove-background`，默认 `http://58.87.105.82/bgfilter/remove-background`，默认请求超时 `180000ms`（BgFilter CPU 推理）。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的前端用户路径都固定把 `screenColor=auto` 注入请求体，但用户可见 `generationInputs.fields` 不再记录 `抠图背景色` 或 `抠图模型`；api-server 在组装 prompt 前调用背景决策模块，从 11 个候选色中选择具体 hex，最多重试 3 次，失败后兜底 `#CFEFFF`。后端仍保留手动 hex 解析能力供内部兼容。最终生图 prompt 和 BgFilter `screen_color` multipart 字段只接收解析后的具体 hex，不透传 `auto`。三条 BgFilter 路径还必须固定把默认 `segModel=birefnet` 传为 `seg_model`；后端仍保留识别 `anime-seg` 的内部兼容能力，但前端用户入口不展示也不提交该值。这里的 `birefnet` 只是 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。后端在调用 BgFilter 前必须先把带纯色背景 / 绿幕源图写入 OSS；若 BgFilter 请求失败、返回非成功状态、空图片或非法图片，api-server 对这些标准纯色背景生成图使用本地 `editor_green_screen` 按同一 `screenColor` 兜底去背；连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD=3` 后，冷却 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS=300` 秒内直接使用本地兜底。角色动作生成的序列帧背景色已与生图统一：前端固定提交 `screenColor=auto`，后端视觉决策出具体 hex 并把源角色图合成到该背景色后再图生视频；抽帧后逐帧优先阿里云通用抠图，失败降级本地 `editor_green_screen`（按选定背景色，而非固定 `#00FF00`）。BiRefNet 手动去背景服务地址为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL/remove-background`，默认 `http://58.87.105.82/remove-background`；BgFilter 可选访问令牌来自 `GENARRATIVE_EDITOR_BGFILTER_TOKEN`，未配置时复用 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN`，所有令牌都只在服务端注入，前端不持有令牌。api-server 对上游结果做响应字节和图片尺寸上限保护，并先落 OSS / asset object，再返回 `imageSrc/objectKey/assetObjectId/taskId`；queue 模式下手动去背景进入 SpacetimeDB 外部生成队列，画布任务侧栏只展示服务器任务阶段，生成中才显示耗时，不显示百分比；有项目上下文时前端同时创建去背景生成占位并把 `canvasCompletion` 交给后端，完成后由后端写入结果图层和最新项目快照。 - 图片快速编辑面板只保留一个提示词输入框和模型选择，不展示额外参考图或比例 / 尺寸控件；原图 / 原素材作为 `/api/editor/images/edits` 的 `sourceImageSrc` 直接提交，不作为 `referenceImageSrcs`。打开快速编辑时画布必须自动平移缩放，让原素材完整落在可视区上半部分，底部面板固定出现在素材下方且不遮挡内容，竖屏 UI 素材也必须完整展示。快速编辑右侧显示矩形、椭圆、画笔框选工具，但进入时不默认启用；点击工具后显示选中态，再点同一工具取消启用。完成框选后，画布红色细框显示连续序号，提示词可按这些编号填写每个区域怎么改。点击 `修改` 后仍停留在当前快速编辑面板显示修改中，不创建独立 `Quick Edit Generator` 画布占位；生成成功后直接用结果覆盖原图图层，失败时保留当前面板并显示错误。 - 底部生成类按钮每次点击都必须创建独立的画布生成对象；新建规范、角色形象或图标素材时，只切换当前编辑面板，不得销毁此前尚未生成或已生成后的其它生成对象状态。归档为非当前编辑对象的生成占位仍可拖动、删除和等待异步完成，完成 / 失败回写必须按生成对象 ID 读取最新占位状态，不能使用提交瞬间的旧快照。 - 画布右上角提供自动隐藏任务侧栏。列表为空且侧栏关闭时只保留图标开关；生成或去背景任务进入时默认打开；用户可手动切换开关状态。 diff --git a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md index c323c1e7a..faf67d410 100644 --- a/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md +++ b/docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md @@ -220,7 +220,7 @@ npm run check:server-rs-ddd - LLM：通用 LLM 门面继续使用 `GENARRATIVE_LLM_*`；创意 Agent `gpt-5.4-mini` Chat Completions 文本链路已于 2026-06 从 APIMart 迁移到 VectorEngine，使用 `VECTOR_ENGINE_BASE_URL` / `VECTOR_ENGINE_API_KEY` 构造 OpenAI-compatible client，`api-server` 会把未带 `/v1` 的 VectorEngine base URL 规范化到 `/v1` 后请求 `/chat/completions`。通用 `/api/llm/chat/completions` 代理使用 `GENARRATIVE_LLM_PROVIDER=openai-compatible`、`GENARRATIVE_LLM_BASE_URL=https://api.vectorengine.cn/v1`、`GENARRATIVE_LLM_MODEL=gpt-5.4-mini`；未单独配置 `GENARRATIVE_LLM_API_KEY` 时可复用 `VECTOR_ENGINE_API_KEY`。`APIMART_BASE_URL` / `APIMART_API_KEY` 只作为历史残留，不再作为创意 Agent gpt-5.4-mini 客户端来源；后续排障时优先确认 VectorEngine `/v1/models`、`/v1/chat/completions` 和 `/v1/responses` 可用性。 - 图片生成：VectorEngine `gpt-image-2` 图片 provider 归属 `platform-image`，密钥只在后端环境变量中；`api-server` 内的 `openai_image_generation.rs` 只是兼容调用面和外部失败审计桥接，不再承载 provider 协议实现。实际外部生成运行记录统一落 `tracking_event`，`event_key = external_generation_run`，metadata 记录开始 / 结束时间、耗时、状态、成功标记、失败原因、provider task id 和结果摘要，不再写回过时的 `ai_task`。DashScope 只按仍在使用的历史能力单独处理，不作为 GPT-image-2 兜底。VectorEngine `/v1/images/generations` 和 `/v1/images/edits` 上游 POST 使用 `libcurl` 发送；`reqwest` 只保留给参考图 URL 下载和响应中图片 URL 下载。`/v1/images/edits` 的 multipart 参考图必须作为 libcurl 文件上传 part 发送，字段名为 `image`，实现上使用 `Form::buffer(file_name, bytes)` 并设置 `Content-Type`；不能只用 `contents(...).filename(...)`，否则上游会把请求转码为缺少图片并返回 `image is required`。`request_send` 阶段的 curl timeout / connect error 按可重试传输错误处理，最多尝试 5 次，并使用指数退避加短抖动；排障时优先看 `attempt`、`max_attempts`、`retry_delay_ms`、`reference_image_bytes_total` 和 `request_params`，不要把 `SendRequest` 当成上游业务错误。 -- 编辑器抠图服务：手动 `POST /api/editor/images/background-removals` 继续代理独立 BiRefNet 服务，配置为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL`、`GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN` 和 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_REQUEST_TIMEOUT_MS`。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的生成后纯色背景透明化改走独立 BgFilter 服务，配置为 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL`、`GENARRATIVE_EDITOR_BGFILTER_TOKEN` 和 `GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS`，默认 base URL 为 `http://58.87.105.82/bgfilter`，默认请求超时为 `120000ms`（BgFilter 当前为 CPU 推理，单次抠图较慢，必须留足超时），token 未配置时复用 BiRefNet token。BgFilter 请求必须显式传 `screen_color=` 和 `seg_model=`；前端用户路径不展示抠图模型选择并固定提交默认 `birefnet`，后端仍识别内部保留的 `anime-seg`，其中 `birefnet` 只表示 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。BgFilter 连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD`（默认 `3`）后，会在 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS`（默认 `300`）内直接使用本地 `editor_green_screen` 兜底，避免上游故障占住 worker。角色动作视频生成的背景色已与生图链路统一：`screenColor=auto` 时由视觉 LLM（`gpt-5-mini`，Responses 协议、low 推理档）读源角色图自动决策，并经硬过滤器剔除与前景 / 皮肤撞色的候选，手动 hex 则尊重用户选择；透明源角色图在提交 Ark 图生视频前先合成到选定背景色实色，使视频背景等于抠图键色。抽帧后逐帧优先走阿里云通用抠图，失败时降级本地 `editor_green_screen` 键色兜底（按生成时选定的背景色，而非固定 `#00FF00`）；BgFilter 与阿里云抠图失败都写入 `external_api_call_failure` 审计。 +- 编辑器抠图服务：手动 `POST /api/editor/images/background-removals` 继续代理独立 BiRefNet 服务，配置为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL`、`GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN` 和 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_REQUEST_TIMEOUT_MS`。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的生成后纯色背景透明化改走独立 BgFilter 服务，配置为 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL`、`GENARRATIVE_EDITOR_BGFILTER_TOKEN` 和 `GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS`，默认 base URL 为 `http://58.87.105.82/bgfilter`，默认请求超时为 `180000ms`（BgFilter 当前为 CPU 推理，单次抠图较慢，必须留足超时），token 未配置时复用 BiRefNet token。BgFilter 请求必须显式传 `screen_color=` 和 `seg_model=`；前端用户路径不展示抠图模型选择并固定提交默认 `birefnet`，后端仍识别内部保留的 `anime-seg`，其中 `birefnet` 只表示 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。BgFilter 连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD`（默认 `3`）后，会在 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS`（默认 `300`）内直接使用本地 `editor_green_screen` 兜底，避免上游故障占住 worker。角色动作视频生成的背景色已与生图链路统一：`screenColor=auto` 时由视觉 LLM（`gpt-5-mini`，Responses 协议、low 推理档）读源角色图自动决策，并经硬过滤器剔除与前景 / 皮肤撞色的候选，手动 hex 则尊重用户选择；透明源角色图在提交 Ark 图生视频前先合成到选定背景色实色，使视频背景等于抠图键色。抽帧后逐帧优先走阿里云通用抠图，失败时降级本地 `editor_green_screen` 键色兜底（按生成时选定的背景色，而非固定 `#00FF00`）；BgFilter 与阿里云抠图失败都写入 `external_api_call_failure` 审计。 - Match3D 物品 sheet：关卡整图完成后走 VectorEngine `/v1/images/edits` multipart `image`，模型为 `gpt-image-2`，`2K 1:1` 输出 `10*10` spritesheet；物品 sheet prompt 固定要求单一纯绿色 `#00FF00 / RGB(0,255,0)` 绿幕背景，后端上传 OSS 前必须把绿幕扣成透明 PNG，并把透明整图写入 `itemSpritesheetImageSrc/itemSpritesheetImageObjectKey`。后端优先按透明 alpha 连通域从该 sheet 识别真实素材矩形并持久化 20 个物品、每个 5 个形态；识别数量不足时才回退 `10*10` 固定网格。通用系列素材图集的行列索引按每行 2 个物品计算，必须落在 `1..=10`，难度只决定运行态加载 3 / 9 / 15 / 20 种。 - Match3D UI spritesheet 和背景派生图：关卡整图作为参考图并发生成 `1K 1:1` UI spritesheet 与 `1K 9:16` 背景图，模型均为 `gpt-image-2`。UI spritesheet prompt 固定要求单一纯绿色 `#00FF00 / RGB(0,255,0)` 绿幕背景，后端上传 OSS 前必须把绿幕扣成透明 PNG；背景图必须合成为全画幅不透明 PNG。 - Match3D 1:1 容器 UI：VectorEngine `/v1/images/edits` multipart 参考图。该容器参考图是后端生图协议输入，必须通过 `include_bytes!` 随 `api-server` 编译进二进制，避免 API 单独发布或运行目录缺少 `public/` 时生成失败。 diff --git a/server-rs/crates/api-server/src/config.rs b/server-rs/crates/api-server/src/config.rs index 6c6d55526..10e8b1d63 100644 --- a/server-rs/crates/api-server/src/config.rs +++ b/server-rs/crates/api-server/src/config.rs @@ -20,7 +20,7 @@ pub(crate) const DEFAULT_VECTOR_ENGINE_IMAGE_REQUEST_TIMEOUT_MS: u64 = 1_000_000 const DEFAULT_EDITOR_BACKGROUND_REMOVAL_BASE_URL: &str = "http://58.87.105.82"; const DEFAULT_EDITOR_BACKGROUND_REMOVAL_REQUEST_TIMEOUT_MS: u64 = 120_000; const DEFAULT_EDITOR_BGFILTER_BASE_URL: &str = "http://58.87.105.82/bgfilter"; -const DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS: u64 = 120_000; +const DEFAULT_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS: u64 = 180_000; const DEFAULT_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD: u32 = 3; const DEFAULT_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS: u64 = 300; const DEFAULT_ALIYUN_MATTING_ENDPOINT: &str = "imageseg.cn-shanghai.aliyuncs.com"; @@ -2334,7 +2334,7 @@ mod tests { "GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN", "shared-token", ); - std::env::set_var("GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS", "180000"); + std::env::set_var("GENARRATIVE_EDITOR_BGFILTER_REQUEST_TIMEOUT_MS", "240000"); } let config = AppConfig::from_env(); @@ -2343,7 +2343,7 @@ mod tests { config.editor_bgfilter_token.as_deref(), Some("shared-token") ); - assert_eq!(config.editor_bgfilter_request_timeout_ms, 180_000); + assert_eq!(config.editor_bgfilter_request_timeout_ms, 240_000); unsafe { std::env::set_var("GENARRATIVE_EDITOR_BGFILTER_TOKEN", "bgfilter-token"); -- 2.52.0 From 1847fde8b757c9387b149e936072636e484a1be2 Mon Sep 17 00:00:00 2001 From: Linghong Date: Thu, 9 Jul 2026 07:26:10 +0000 Subject: [PATCH 15/41] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=8A=A0=E5=9B=BE?= =?UTF-8?q?=E5=A4=96=E9=83=A8=E5=A4=B1=E8=B4=A5=E5=AE=A1=E8=AE=A1=E5=88=86?= =?UTF-8?q?=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留 BgFilter 上游真实 HTTP 状态与超时标记 保留阿里云抠图上游状态、超时与错误摘要 补充审计分类单测覆盖 429 与 timeout 场景 --- .../crates/api-server/src/aliyun_matting.rs | 62 +++++++- .../src/character_animation_assets.rs | 6 +- .../crates/api-server/src/editor_project.rs | 10 +- .../api-server/src/external_api_audit.rs | 141 +++++++++++++++--- 4 files changed, 193 insertions(+), 26 deletions(-) diff --git a/server-rs/crates/api-server/src/aliyun_matting.rs b/server-rs/crates/api-server/src/aliyun_matting.rs index 3746f2f1c..16487ece7 100644 --- a/server-rs/crates/api-server/src/aliyun_matting.rs +++ b/server-rs/crates/api-server/src/aliyun_matting.rs @@ -30,9 +30,19 @@ pub(crate) async fn segment_image_with_aliyun_matting( .segment_image_to_transparent_png(image.bytes.as_slice(), &file_name) .await .map_err(|error| { - AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + let message = error.message(); + let timeout = aliyun_matting_error_is_timeout(message); + let status = if timeout { + StatusCode::GATEWAY_TIMEOUT + } else { + StatusCode::BAD_GATEWAY + }; + AppError::from_status(status).with_details(json!({ "provider": "aliyun-matting", - "message": error.message(), + "message": message, + "timeout": timeout, + "upstreamStatus": aliyun_matting_error_http_status(message), + "rawExcerpt": message.chars().take(500).collect::(), })) })?; tracing::info!( @@ -48,3 +58,51 @@ pub(crate) async fn segment_image_with_aliyun_matting( extension: "png".to_string(), }) } + +fn aliyun_matting_error_is_timeout(message: &str) -> bool { + let normalized = message.to_ascii_lowercase(); + normalized.contains("timeout") || normalized.contains("timed out") +} + +fn aliyun_matting_error_http_status(message: &str) -> Option { + let marker = "HTTP "; + let start = message.find(marker)? + marker.len(); + let digits = message[start..] + .chars() + .take_while(|value| value.is_ascii_digit()) + .collect::(); + digits.parse().ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn aliyun_matting_error_http_status_extracts_provider_status() { + assert_eq!( + aliyun_matting_error_http_status( + "通用抠图接口返回失败（HTTP 429，Code=Throttled）：QPS exceeded" + ), + Some(429) + ); + assert_eq!( + aliyun_matting_error_http_status("下载抠图结果失败（HTTP 403）"), + Some(403) + ); + assert_eq!(aliyun_matting_error_http_status("网络失败"), None); + } + + #[test] + fn aliyun_matting_error_timeout_detects_transport_timeout() { + assert!(aliyun_matting_error_is_timeout( + "通用抠图请求失败：operation timed out" + )); + assert!(aliyun_matting_error_is_timeout( + "GetOssStsToken 请求失败：request timeout" + )); + assert!(!aliyun_matting_error_is_timeout( + "通用抠图接口返回失败（HTTP 400，Code=InvalidImage）：bad image" + )); + } +} diff --git a/server-rs/crates/api-server/src/character_animation_assets.rs b/server-rs/crates/api-server/src/character_animation_assets.rs index 1cf3bdb5d..072273b23 100644 --- a/server-rs/crates/api-server/src/character_animation_assets.rs +++ b/server-rs/crates/api-server/src/character_animation_assets.rs @@ -2245,9 +2245,11 @@ async fn remove_editor_character_animation_frame_backgrounds( state.config.aliyun_matting_endpoint.clone(), "editor-character-animation-frame-matting", "aliyun_segment", - error.status_code().as_u16(), + crate::external_api_audit::matting_failure_audit_status_code(&error), + crate::external_api_audit::matting_failure_audit_timeout(&error), error.message().to_string(), - Some(format!("frame_index={frame_index}")), + crate::external_api_audit::matting_failure_audit_raw_excerpt(&error) + .or_else(|| Some(format!("frame_index={frame_index}"))), ) .await; remove_editor_generated_green_screen_background(&image, screen_color)? diff --git a/server-rs/crates/api-server/src/editor_project.rs b/server-rs/crates/api-server/src/editor_project.rs index 029e9ff85..6efba3544 100644 --- a/server-rs/crates/api-server/src/editor_project.rs +++ b/server-rs/crates/api-server/src/editor_project.rs @@ -2266,9 +2266,10 @@ async fn remove_editor_generated_screen_background_with_bgfilter( state.config.editor_bgfilter_base_url.clone(), "editor-screen-background-removal", "bgfilter_segment", - error.status_code().as_u16(), + crate::external_api_audit::matting_failure_audit_status_code(&error), + crate::external_api_audit::matting_failure_audit_timeout(&error), error.message().to_string(), - None, + crate::external_api_audit::matting_failure_audit_raw_excerpt(&error), ) .await; match crate::aliyun_matting::segment_image_with_aliyun_matting( @@ -2294,9 +2295,10 @@ async fn remove_editor_generated_screen_background_with_bgfilter( state.config.aliyun_matting_endpoint.clone(), "editor-screen-background-removal", "aliyun_segment", - error.status_code().as_u16(), + crate::external_api_audit::matting_failure_audit_status_code(&error), + crate::external_api_audit::matting_failure_audit_timeout(&error), error.message().to_string(), - None, + crate::external_api_audit::matting_failure_audit_raw_excerpt(&error), ) .await; remove_editor_generated_green_screen_background(image, screen_color) diff --git a/server-rs/crates/api-server/src/external_api_audit.rs b/server-rs/crates/api-server/src/external_api_audit.rs index cc0dc824b..3d129e021 100644 --- a/server-rs/crates/api-server/src/external_api_audit.rs +++ b/server-rs/crates/api-server/src/external_api_audit.rs @@ -6,7 +6,7 @@ use serde_json::{Value, json}; use time::OffsetDateTime; use uuid::Uuid; -use crate::{state::AppState, tracking::TrackingEventDraft}; +use crate::{http_error::AppError, state::AppState, tracking::TrackingEventDraft}; pub(crate) const EXTERNAL_API_FAILURE_EVENT_KEY: &str = "external_api_call_failure"; pub(crate) const EXTERNAL_API_AUDIT_MODULE_KEY: &str = "external-api"; @@ -156,29 +156,63 @@ pub(crate) async fn record_matting_external_api_failure( endpoint: String, operation: &'static str, failure_stage: &'static str, - status_code: u16, + status_code: Option, + timeout: bool, error_message: String, raw_excerpt: Option, ) { - let draft = ExternalApiFailureDraft::new( - provider, - endpoint, - operation, - failure_stage, - error_message, - ) - .with_status_code(Some(status_code)) - .with_optional_status_class(Some(status_class(Some(status_code)))) - .with_retryable(is_retryable_external_api_failure( - Some(status_code), - false, - false, - )) - .with_raw_excerpt(raw_excerpt) - .with_audit_context(context); + let draft = + ExternalApiFailureDraft::new(provider, endpoint, operation, failure_stage, error_message) + .with_status_code(status_code) + .with_optional_status_class(Some(status_class(status_code))) + .with_timeout(timeout) + .with_retryable(is_retryable_external_api_failure( + status_code, + timeout, + false, + )) + .with_raw_excerpt(raw_excerpt) + .with_audit_context(context); record_external_api_failure(state, draft).await; } +pub(crate) fn matting_failure_audit_status_code(error: &AppError) -> Option { + error + .details() + .and_then(|details| details.get("upstreamStatus")) + .and_then(Value::as_u64) + .and_then(|value| u16::try_from(value).ok()) + .or_else(|| { + if matting_failure_audit_timeout(error) { + None + } else { + Some(error.status_code().as_u16()) + } + }) +} + +pub(crate) fn matting_failure_audit_timeout(error: &AppError) -> bool { + error + .details() + .and_then(|details| details.get("timeout")) + .and_then(Value::as_bool) + .unwrap_or(false) +} + +pub(crate) fn matting_failure_audit_raw_excerpt(error: &AppError) -> Option { + error + .details() + .and_then(|details| { + details + .get("upstreamMessage") + .or_else(|| details.get("rawExcerpt")) + }) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(|value| value.chars().take(800).collect()) +} + pub(crate) fn build_external_api_failure_draft_from_platform_image_audit( audit: &PlatformImageFailureAudit, ) -> ExternalApiFailureDraft { @@ -499,4 +533,75 @@ mod tests { assert_eq!(draft.metadata["statusCode"], 200); assert_eq!(draft.metadata["statusClass"], "5xx"); } + + #[test] + fn matting_failure_audit_uses_upstream_status_instead_of_wrapped_status() { + let error = AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + "provider": "bgfilter", + "message": "BgFilter 服务返回非成功状态", + "upstreamStatus": 429, + "upstreamMessage": "too many requests", + })); + + let status_code = matting_failure_audit_status_code(&error); + let timeout = matting_failure_audit_timeout(&error); + let tracking = build_external_api_failure_tracking_draft( + &ExternalApiFailureDraft::new( + "bgfilter", + "https://bgfilter.example/remove-background", + "editor-screen-background-removal", + "bgfilter_segment", + error.message(), + ) + .with_status_code(status_code) + .with_optional_status_class(Some(status_class(status_code))) + .with_timeout(timeout) + .with_retryable(is_retryable_external_api_failure( + status_code, + timeout, + false, + )) + .with_raw_excerpt(matting_failure_audit_raw_excerpt(&error)), + ); + + assert_eq!(tracking.metadata["statusCode"], 429); + assert_eq!(tracking.metadata["statusClass"], "4xx"); + assert_eq!(tracking.metadata["timeout"], false); + assert_eq!(tracking.metadata["retryable"], true); + assert_eq!(tracking.metadata["rawExcerpt"], "too many requests"); + } + + #[test] + fn matting_failure_audit_keeps_transport_timeout_classification() { + let error = AppError::from_status(StatusCode::GATEWAY_TIMEOUT).with_details(json!({ + "provider": "bgfilter", + "message": "请求 BgFilter 服务失败：operation timed out", + "timeout": true, + })); + + let status_code = matting_failure_audit_status_code(&error); + let timeout = matting_failure_audit_timeout(&error); + let tracking = build_external_api_failure_tracking_draft( + &ExternalApiFailureDraft::new( + "bgfilter", + "https://bgfilter.example/remove-background", + "editor-screen-background-removal", + "bgfilter_segment", + error.message(), + ) + .with_status_code(status_code) + .with_optional_status_class(Some(status_class(status_code))) + .with_timeout(timeout) + .with_retryable(is_retryable_external_api_failure( + status_code, + timeout, + false, + )), + ); + + assert_eq!(tracking.metadata["statusCode"], Value::Null); + assert_eq!(tracking.metadata["statusClass"], "transport"); + assert_eq!(tracking.metadata["timeout"], true); + assert_eq!(tracking.metadata["retryable"], true); + } } -- 2.52.0 From 1aeb905e80da12b688cf764c9afbfb41ce61aaf5 Mon Sep 17 00:00:00 2001 From: Linghong Date: Thu, 9 Jul 2026 07:39:20 +0000 Subject: [PATCH 16/41] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E7=94=BB=E6=9D=BF?= =?UTF-8?q?=E6=8A=A0=E5=9B=BE=E8=83=8C=E6=99=AF=E8=89=B2=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E5=8F=A3=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更新候选色数量为 12 个 同步角色动作多色背景与阿里云优先抠帧说明 --- docs/project-memory/shared-memory/decision-log.md | 4 ++-- .../【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md | 2 +- docs/【编辑器】画板UI设计图生成入口设计-2026-06-17.md | 2 +- docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 9062221b7..4d36fe0b8 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -27,7 +27,7 @@ ## 2026-07-02 图片画布生成抠图背景色使用 screenColor 传递 - 背景：画布角色、图标和 UI 素材生成过去固定要求 `#00FF00` 绿幕，后续 BGfilter 服务需要按生成时背景色做去背景，不能继续把背景色写死在 prompt 或后处理里。 -- 决策：角色形象、图标 spritesheet 和 UI 设计图素材提取不再向用户提供手动抠图背景色选择；前端用户路径统一提交 `screenColor=auto`，但用户可见生成输入快照不再写入 `抠图背景色` 或 `抠图模型`。api-server 在 11 个候选色中自动决策具体 hex，失败后兜底 `#CFEFFF`；最终 prompt 和 BgFilter 去背景只接收解析后的具体 hex 作为 `screen_color`。后端仍保留手动 hex 解析能力供内部兼容，角色动作抽帧暂不接入该选择，继续使用 legacy `#00FF00` 绿幕。 +- 决策：角色形象、图标 spritesheet 和 UI 设计图素材提取不再向用户提供手动抠图背景色选择；前端用户路径统一提交 `screenColor=auto`，但用户可见生成输入快照不再写入 `抠图背景色` 或 `抠图模型`。api-server 在 12 个候选色中自动决策具体 hex，失败后兜底 `#CFEFFF`；最终 prompt 和 BgFilter 去背景只接收解析后的具体 hex 作为 `screen_color`。后端仍保留手动 hex 解析能力供内部兼容，角色动作抽帧暂不接入该选择，继续使用 legacy `#00FF00` 绿幕。 - 影响范围：`/editor/canvas` 角色形象生成、图标素材生成、UI 设计图素材提取、BgFilter 服务入参、图片画布 MVP 和角色形象生成设计文档。 - 验证方式：运行画布生成模型 / workflow / API client 定向前端测试、`cargo test -p api-server editor_green_screen --manifest-path server-rs/Cargo.toml`、`cargo test -p platform-image generated_asset_sheet_light_blue_key_color_removes_selected_background --manifest-path server-rs/Cargo.toml`、`cargo check -p api-server --manifest-path server-rs/Cargo.toml`、`npm run check:encoding` 和 `git diff --check`。 - 关联文档：`docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md`、`docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md`。 @@ -35,7 +35,7 @@ ## 2026-07-05 图片画布抠图背景色自动决策 - 背景：手动背景色选择对用户负担较高，且不同角色、图标和 UI 素材主题需要避开不同主体色；但 BgFilter 和生成 prompt 仍必须拿到明确的纯色 hex。 -- 决策：前端用户路径直接固定通过 `screenColor=auto` 提交，不再展示背景色选项；api-server 新增 `editor_screen_background_decision` 模块，在角色形象、图标 spritesheet 和 UI 设计图素材提取组装 prompt 前解析 `screenColor`。手动 hex 直接校验并使用；`auto` 通过服务端 LLM 在 11 个候选色中选择具体 hex，最多重试 3 次，LLM 未配置、请求失败或返回非法颜色时 fallback 到 `浅雾蓝 #CFEFFF`。自动解析结果不写入用户可见生成输入快照；最终生图 prompt 和 BgFilter `screen_color` 永远只接收具体 hex，不透传 `auto`。 +- 决策：前端用户路径直接固定通过 `screenColor=auto` 提交，不再展示背景色选项；api-server 新增 `editor_screen_background_decision` 模块，在角色形象、图标 spritesheet 和 UI 设计图素材提取组装 prompt 前解析 `screenColor`。手动 hex 直接校验并使用；`auto` 通过服务端 LLM 在 12 个候选色中选择具体 hex，最多重试 3 次，LLM 未配置、请求失败或返回非法颜色时 fallback 到 `浅雾蓝 #CFEFFF`。自动解析结果不写入用户可见生成输入快照；最终生图 prompt 和 BgFilter `screen_color` 永远只接收具体 hex，不透传 `auto`。 - 影响范围：`/editor/canvas` 角色形象生成、图标素材生成、UI 设计图素材提取、api-server LLM 调用、BgFilter 参数、图片画布文档。 - 验证方式：运行背景决策模块单测、画布生成模型 / workflow / API client 定向前端测试、`cargo test -p api-server editor_screen_background_decision editor_green_screen --manifest-path server-rs/Cargo.toml`、`npm run check:encoding` 和 `git diff --check`。 - 关联文档：`docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md`、`docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md`、`docs/【编辑器】画板UI设计图生成入口设计-2026-06-17.md`、`docs/【编辑器】画板图标素材生成入口设计-2026-06-15.md`。 diff --git a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md index d6a068814..984a1f15a 100644 --- a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md +++ b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md @@ -21,7 +21,7 @@ - 生成资源右上角显示元数据按钮，点击打开独立元数据窗口。图片信息页不展示后端组装后的生图 Prompt，也不提供复制 Prompt；只展示该图片生成时用户在面板里提交的输入快照，包括普通生成提示词、规范表单字段、角色设定、图标素材描述、快速编辑提示词、重绘提示词，以及角色规范 / 常规参考图 / 图标规范 / 编辑参考图等参考图卡片，并提供“复制信息”复制当前可见字段。参考图输入快照只保存 `refType/refId` 行引用，其中 `refType="project-resource"` 指向 `editor_project_resource.resourceId`，`refType="asset"` 指向 `editor_asset.assetId`；不得把图片 Data URL、普通 URL 或 `objectKey` 写入 `generationInputs.references`。旧数据或上传图片没有输入快照时显示 `-`，禁止回退展示内部 Prompt。 - 对生成资源执行重绘时，在右侧创建新的生成结果图层，并自动调整视图显示原图和新图；重绘面板不因提交成功自动关闭，便于连续改提示词。重绘 / 改造输入框只允许从 `generationInputs.fields` 中恢复用户可见输入快照，例如普通生成提示词、视频描述、音效 `prompt`、背景音乐 `gpt_description_prompt`、角色设定、UI 用户输入、图标素材描述、规范表单和宣发素材字段；禁止回退展示资源 `prompt` / `actualPrompt` 中的后端拼接 Prompt、固定生成模板或模型默认提示词。没有用户输入快照的旧图层打开改造时保持空输入，等待用户重新填写。 - 图片生成 / 修改统一经 api-server BFF 接入 VectorEngine。普通生成、生成规范和重绘保留既有 `gpt-image-2` 路径；图片快速编辑统一打开框选区域 + 单提示词 + 模型选择面板，默认沿用原图模型，不展示参考图或比例 / 尺寸控件；其中生成规范类图片固定 `16:9`、`2K`、`gpt-image-2`，面板底部用与可编辑面板一致的比例 / 尺寸 / 模型胶囊按钮展示固定参数，但按钮为禁用态，不允许在该面板改比例、尺寸或模型。`生成角色形象` 与 `生成图标素材` 支持 `nanobanana2`（`gemini-3.1-flash-image-preview`）和 `gpt-image-2`，默认 `nanobanana2`，并在两类面板之间沿用用户上次选择的模型；两类面板不展示抠图背景色或抠图模型选择；前端用户路径固定提交 `screenColor=auto` 和 `segModel=birefnet`，由后端自动决策具体抠图背景色，`anime-seg` 作为内部保留能力不在用户界面暴露。`nanobanana2` 走 `/v1beta/models/{model}:generateContent`，请求体写入 `generationConfig.imageConfig.aspectRatio/imageSize`；`gpt-image-2` 走 `/v1/images/generations` 或 `/v1/images/edits`，请求体按 VectorEngine 文档映射 `size`。宣发素材三个工作流（游戏首图、详情五图、运营海报）固定使用 `gpt-image-2`，面板模型胶囊为禁用态，不提供 `nanobanana2` 入口；前端按 workflow 同时提交 `outputSize`、`aspectRatio` 和 `imageSize`，其中游戏首图为 `720x540 / 4:3`、详情单图为 `720x1280 / 9:16`、运营海报为 `1280x720 / 16:9`；后端收到 `kind: "publication-material"` 时也强制归一为 `gpt-image-2` 生成和计费，生成回填图层优先使用生成占位的 `originalWidth/originalHeight`，即使上游回包尺寸漂移也不得把宣发素材卡片变成随机 `1:1` 或 `4:3`。纯文本生成走 `/api/editor/images/generations`，重绘在前端读入当前图层图片 Data URL 后走同一图片生成 BFF，并在原图右侧生成一张新图；普通图层重绘作为 `quick-edit` 参考图提交，角色图层重绘必须按 `kind: "character"` 提交，继续套用角色生成器提示词限定、透明 PNG 后处理和角色资产持久化。`生成视频` 走 `/api/editor/videos/generations`，前端模型入口仅展示 Seedance 2.0 Fast / Seedance 2.0 / Kling 3.0 / Kling 3.0 Omni，不展示 Veo 入口，默认 Seedance 2.0 Fast；视频参数按当前正式面板支持的比例、时长、清晰度和声音开关提交，且 Seedance Fast 与 Seedance 标准版必须按各自真实模型 ID 独立映射，不得混用。生成结果以视频图层加入画布。纯文本生成入口采用 Lovart 式画布内占位图 + 锚定生成输入框：点击生成图片后以当前视口世界中心为目标，经统一 placement 避让后创建选中的灰色占位框，输入框跟随占位框显示；待生成、生成中和失败后保留的占位图都必须继续支持拖动，生成完成时真实生成图或视频落在最新占位框位置，输入框继续跟随新生成图层；占位图失焦时隐藏高亮边框、左上角生成器名称和右上角原始尺寸，重新聚焦时再显示，且名称 / 尺寸在画布缩小时按 viewport 反向缩放保持屏幕尺寸稳定；点击所有图片 / 视频生成入口并确认请求开始后，必须隐藏对应设置面板，只保留画布内占位图或原图预览，并在预览上显示 Lovart 式生成中遮罩，避免“面板仍占屏”或“预览一起消失”。图片快速编辑和重绘在调用图片 BFF 前必须把当前图层图片源读取为图片 Data URL；视频素材快速编辑走视频生成 BFF，不允许走图片模型；角色动作的 `生成动画` 仍固定使用 `seedance2.0-fast` 动作 / 视频模型，角色动作素材的 `快速编辑` 按当前帧图片走图片编辑。前端不持有 provider 密钥；上游失败或配置缺失时恢复当前生成设置面板展示失败，不创建 mock 成功图。 -- 图片画布抠图分两类：手动去除背景面向用户任意图片，走登录态同源 BFF `POST /api/editor/images/background-removals` 并转发远端 BiRefNet；编辑器自己生成的标准纯色背景抠图资产在保存源图后统一调用独立 BgFilter 服务 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL/remove-background`，默认 `http://58.87.105.82/bgfilter/remove-background`，默认请求超时 `180000ms`（BgFilter CPU 推理）。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的前端用户路径都固定把 `screenColor=auto` 注入请求体，但用户可见 `generationInputs.fields` 不再记录 `抠图背景色` 或 `抠图模型`；api-server 在组装 prompt 前调用背景决策模块，从 11 个候选色中选择具体 hex，最多重试 3 次，失败后兜底 `#CFEFFF`。后端仍保留手动 hex 解析能力供内部兼容。最终生图 prompt 和 BgFilter `screen_color` multipart 字段只接收解析后的具体 hex，不透传 `auto`。三条 BgFilter 路径还必须固定把默认 `segModel=birefnet` 传为 `seg_model`；后端仍保留识别 `anime-seg` 的内部兼容能力，但前端用户入口不展示也不提交该值。这里的 `birefnet` 只是 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。后端在调用 BgFilter 前必须先把带纯色背景 / 绿幕源图写入 OSS；若 BgFilter 请求失败、返回非成功状态、空图片或非法图片，api-server 对这些标准纯色背景生成图使用本地 `editor_green_screen` 按同一 `screenColor` 兜底去背；连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD=3` 后，冷却 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS=300` 秒内直接使用本地兜底。角色动作生成的序列帧背景色已与生图统一：前端固定提交 `screenColor=auto`，后端视觉决策出具体 hex 并把源角色图合成到该背景色后再图生视频；抽帧后逐帧优先阿里云通用抠图，失败降级本地 `editor_green_screen`（按选定背景色，而非固定 `#00FF00`）。BiRefNet 手动去背景服务地址为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL/remove-background`，默认 `http://58.87.105.82/remove-background`；BgFilter 可选访问令牌来自 `GENARRATIVE_EDITOR_BGFILTER_TOKEN`，未配置时复用 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN`，所有令牌都只在服务端注入，前端不持有令牌。api-server 对上游结果做响应字节和图片尺寸上限保护，并先落 OSS / asset object，再返回 `imageSrc/objectKey/assetObjectId/taskId`；queue 模式下手动去背景进入 SpacetimeDB 外部生成队列，画布任务侧栏只展示服务器任务阶段，生成中才显示耗时，不显示百分比；有项目上下文时前端同时创建去背景生成占位并把 `canvasCompletion` 交给后端，完成后由后端写入结果图层和最新项目快照。 +- 图片画布抠图分两类：手动去除背景面向用户任意图片，走登录态同源 BFF `POST /api/editor/images/background-removals` 并转发远端 BiRefNet；编辑器自己生成的标准纯色背景抠图资产在保存源图后统一调用独立 BgFilter 服务 `GENARRATIVE_EDITOR_BGFILTER_BASE_URL/remove-background`，默认 `http://58.87.105.82/bgfilter/remove-background`，默认请求超时 `180000ms`（BgFilter CPU 推理）。角色形象生成、图标 spritesheet 生成和 UI 设计图素材提取的前端用户路径都固定把 `screenColor=auto` 注入请求体，但用户可见 `generationInputs.fields` 不再记录 `抠图背景色` 或 `抠图模型`；api-server 在组装 prompt 前调用背景决策模块，从 12 个候选色中选择具体 hex，最多重试 3 次，失败后兜底 `#CFEFFF`。后端仍保留手动 hex 解析能力供内部兼容。最终生图 prompt 和 BgFilter `screen_color` multipart 字段只接收解析后的具体 hex，不透传 `auto`。三条 BgFilter 路径还必须固定把默认 `segModel=birefnet` 传为 `seg_model`；后端仍保留识别 `anime-seg` 的内部兼容能力，但前端用户入口不展示也不提交该值。这里的 `birefnet` 只是 BgFilter 管线内部后端，不等同于手动去背景的独立 BiRefNet 服务。后端在调用 BgFilter 前必须先把带纯色背景 / 绿幕源图写入 OSS；若 BgFilter 请求失败、返回非成功状态、空图片或非法图片，api-server 对这些标准纯色背景生成图使用本地 `editor_green_screen` 按同一 `screenColor` 兜底去背；连续失败达到 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD=3` 后，冷却 `GENARRATIVE_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS=300` 秒内直接使用本地兜底。角色动作生成的序列帧背景色已与生图统一：前端固定提交 `screenColor=auto`，后端视觉决策出具体 hex 并把源角色图合成到该背景色后再图生视频；抽帧后逐帧优先阿里云通用抠图，失败降级本地 `editor_green_screen`（按选定背景色，而非固定 `#00FF00`）。BiRefNet 手动去背景服务地址为 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_BASE_URL/remove-background`，默认 `http://58.87.105.82/remove-background`；BgFilter 可选访问令牌来自 `GENARRATIVE_EDITOR_BGFILTER_TOKEN`，未配置时复用 `GENARRATIVE_EDITOR_BACKGROUND_REMOVAL_TOKEN`，所有令牌都只在服务端注入，前端不持有令牌。api-server 对上游结果做响应字节和图片尺寸上限保护，并先落 OSS / asset object，再返回 `imageSrc/objectKey/assetObjectId/taskId`；queue 模式下手动去背景进入 SpacetimeDB 外部生成队列，画布任务侧栏只展示服务器任务阶段，生成中才显示耗时，不显示百分比；有项目上下文时前端同时创建去背景生成占位并把 `canvasCompletion` 交给后端，完成后由后端写入结果图层和最新项目快照。 - 图片快速编辑面板只保留一个提示词输入框和模型选择，不展示额外参考图或比例 / 尺寸控件；原图 / 原素材作为 `/api/editor/images/edits` 的 `sourceImageSrc` 直接提交，不作为 `referenceImageSrcs`。打开快速编辑时画布必须自动平移缩放，让原素材完整落在可视区上半部分，底部面板固定出现在素材下方且不遮挡内容，竖屏 UI 素材也必须完整展示。快速编辑右侧显示矩形、椭圆、画笔框选工具，但进入时不默认启用；点击工具后显示选中态，再点同一工具取消启用。完成框选后，画布红色细框显示连续序号，提示词可按这些编号填写每个区域怎么改。点击 `修改` 后仍停留在当前快速编辑面板显示修改中，不创建独立 `Quick Edit Generator` 画布占位；生成成功后直接用结果覆盖原图图层，失败时保留当前面板并显示错误。 - 底部生成类按钮每次点击都必须创建独立的画布生成对象；新建规范、角色形象或图标素材时，只切换当前编辑面板，不得销毁此前尚未生成或已生成后的其它生成对象状态。归档为非当前编辑对象的生成占位仍可拖动、删除和等待异步完成，完成 / 失败回写必须按生成对象 ID 读取最新占位状态，不能使用提交瞬间的旧快照。 - 画布右上角提供自动隐藏任务侧栏。列表为空且侧栏关闭时只保留图标开关；生成或去背景任务进入时默认打开；用户可手动切换开关状态。 diff --git a/docs/【编辑器】画板UI设计图生成入口设计-2026-06-17.md b/docs/【编辑器】画板UI设计图生成入口设计-2026-06-17.md index 84a4f9de7..1829eb851 100644 --- a/docs/【编辑器】画板UI设计图生成入口设计-2026-06-17.md +++ b/docs/【编辑器】画板UI设计图生成入口设计-2026-06-17.md @@ -28,7 +28,7 @@ - 支持自定义画面比例和大小尺寸。 - 模型固定为 `gpt-image-2`，模型展示对齐角色规范面板底部固定模型样式，不响应点击、不弹出模型切换菜单；历史草稿如果残留其他模型，提交时也必须强制改为 `gpt-image-2`。 - 默认画面比例为 `16:9`，默认大小为 `1K`。 -- UI 素材提取面板不展示抠图背景色或抠图模型选择；前端用户路径固定提交 `screenColor=auto` 和 `segModel=birefnet`。后端先在 11 个候选色中自动决策具体 hex，最多重试 3 次，失败兜底 `#CFEFFF`；后端调用 BgFilter 时只把解析后的具体 hex 作为 `screen_color` 传入。后端仍识别内部保留的 `anime-seg`，但该选项不对用户可见。 +- UI 素材提取面板不展示抠图背景色或抠图模型选择；前端用户路径固定提交 `screenColor=auto` 和 `segModel=birefnet`。后端先在 12 个候选色中自动决策具体 hex，最多重试 3 次，失败兜底 `#CFEFFF`；后端调用 BgFilter 时只把解析后的具体 hex 作为 `screen_color` 传入。后端仍识别内部保留的 `anime-seg`，但该选项不对用户可见。 ## 提示词契约 diff --git a/docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md b/docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md index d55d5a2c2..fe9c1bec7 100644 --- a/docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md +++ b/docs/【编辑器】画板角色形象生成入口设计-2026-06-15.md @@ -153,7 +153,7 @@ - 后端 prompt 使用以下固定骨架，并把面板输入追加到 `动作描述：` 后： ```text -生成游戏角色动画，参考图作为首帧和尾帧，画面中心构图，角色主体完整置于画面中央，禁止镜头透视，禁止特写。背景固定为单一纯绿色 #00FF00 / RGB(0,255,0) 绿幕，只作为抠像底色；绿幕背景必须平整无纹理、无渐变、无阴影、无地面、无环境、无道具；角色主体不得带绿色描边、绿色投影或绿色反光；禁止出现建筑、室内布景、风景、地面道具、漂浮物、烟雾叙事元素、文字或其他角色以外的场景内容。 +生成游戏角色动画，参考图作为首帧和尾帧，画面中心构图，角色主体完整置于画面中央，禁止镜头透视，禁止特写。背景使用后端自动决策出的单一纯色抠像底色；纯色背景必须平整无纹理、无渐变、无阴影、无地面、无环境、无道具；角色主体不得带与抠像底色相近的描边、投影或反光；禁止出现建筑、室内布景、风景、地面道具、漂浮物、烟雾叙事元素、文字或其他角色以外的场景内容。 动作描述： <用户输入的动画描述> ``` @@ -162,9 +162,9 @@ - 视频生成完成后，后端按面板选择抽取对应帧数：`32`、`40` 或 `48`。 - 抽帧采样必须按目标帧数预留视频尾部安全步长，例如 `32帧·4秒` 最后一帧采 `3.875s`，避免 FFmpeg 在尾点附近返回成功但输出 `0` 帧。 -- 每帧必须先把带 legacy `#00FF00` 绿幕源图写入 OSS，再执行 `editor_green_screen` 统一绿幕去背，输出透明背景 PNG。角色动作暂不接入角色 / 图标生图的 `screenColor` 选择。 +- 每帧必须先把带自动决策纯色背景的源图写入 OSS，再优先调用阿里云通用抠图输出透明背景 PNG；阿里云失败时降级执行本地 `editor_green_screen`，并按同一次生成已选定的 `screenColor` 去背。 - 抽帧结果写入 OSS，并返回帧路径、帧尺寸、帧数、fps、预览视频路径、模型、价格和实际 prompt。 - 画板前端回填角色动作结果时，必须以 `frames[0].imageSrc` 创建 `mediaType: "image-sequence"`、`assetKind: "character-animation"` 图层，并把完整 `frames` 保存为图层 `imageSequenceFrames`；`previewVideoPath` 只保留为上游预览视频来源，不作为画布主媒体。 - 角色动作图层在画布中使用序列帧播放器循环展示透明 PNG 帧；刷新恢复时必须继续读取 `imageSequenceFrames`，不能回退到 `