From 923993ea6cfa2a26c086fa92353890f961c7d53c Mon Sep 17 00:00:00 2001 From: Linghong Date: Wed, 8 Jul 2026 13:37:44 +0000 Subject: [PATCH] =?UTF-8?q?=E8=83=8C=E6=99=AF=E8=89=B2=E5=86=B3=E7=AD=96?= =?UTF-8?q?=E5=8D=87=E7=BA=A7=EF=BC=9A=E8=A7=86=E8=A7=89=E9=80=89=E8=89=B2?= =?UTF-8?q?=20+=20=E7=9A=AE=E8=82=A4=E7=A1=AC=E8=BF=87=E6=BB=A4=20+=20?= =?UTF-8?q?=E6=BA=90=E5=9B=BE=E5=90=88=E6=88=90=20+=20=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E5=8C=96=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复动作视频背景色的三类真实事故并加确定性防线： 1. 视觉背景色决策：让 LLM 看参考图选背景色（gpt-4o-mini 视觉模型， 路由到 VectorEngine 视觉客户端），修复原纯文本盲选导致的"蓝撞蓝"。 2. 背景色候选硬过滤器（新增 editor_screen_background_filter）： 按参考图前景配色算 Lab 危险质量 + 皮肤专属三判据（ΔE 距离、 色调投影、RGB 分离）剔除撞色候选，LLM 只在安全集合里审美选。 露肤角色自动收敛到冷区。新增"灰竹绿"候选补齐安全弧绿色段。 3. 源图合成到背景色：透明源图先填成选定实色再发给 Ark，视频背景 确定性等于抠图键色，不再赌模型服从提示词（修复背景变白）。 4. screenColorHex 结构化记录：四条链路统一把实际背景色写进 generation_inputs，并修复生图链路"抠图背景色"字段从未生效的旧逻辑。 Co-Authored-By: Claude Opus 4.8 --- .../src/character_animation_assets.rs | 157 ++++- .../api-server/src/editor_green_screen.rs | 15 +- .../crates/api-server/src/editor_project.rs | 101 ++- .../src/editor_screen_background_decision.rs | 233 ++++++- .../src/editor_screen_background_filter.rs | 611 ++++++++++++++++++ .../api-server/src/llm_model_routing.rs | 2 + server-rs/crates/api-server/src/main.rs | 1 + 7 files changed, 1075 insertions(+), 45 deletions(-) create mode 100644 server-rs/crates/api-server/src/editor_screen_background_filter.rs diff --git a/server-rs/crates/api-server/src/character_animation_assets.rs b/server-rs/crates/api-server/src/character_animation_assets.rs index 49eb850fb..dd7899953 100644 --- a/server-rs/crates/api-server/src/character_animation_assets.rs +++ b/server-rs/crates/api-server/src/character_animation_assets.rs @@ -70,6 +70,7 @@ use crate::{ }, editor_project::{ EditorCanvasGeneratedLayerInput, PersistEditorGeneratedAssetRequest, + apply_editor_screen_background_decision_to_generation_inputs, build_editor_canvas_generated_layer_item, complete_editor_canvas_generation_with_items, persist_editor_generated_media_asset, }, @@ -108,6 +109,7 @@ const FIXED_ARK_CHARACTER_VIDEO_RESOLUTION: &str = "480p"; const FIXED_ARK_CHARACTER_VIDEO_RATIO: &str = "1:1"; const FIXED_ARK_CHARACTER_VIDEO_DURATION_SECONDS: u32 = 4; const ARK_VIDEO_TASK_POLL_INTERVAL_MS: u64 = 5_000; +const SOURCE_IMAGE_RESOLVE_TIMEOUT_MS: u64 = 30_000; const EDITOR_CHARACTER_ANIMATION_MODEL: &str = "seedance2.0-fast"; const EDITOR_CHARACTER_ANIMATION_ASSET_KIND: &str = "editor_character_animation"; const EDITOR_GREEN_SCREEN_SOURCE_ASSET_KIND: &str = "editor_green_screen_source"; @@ -635,19 +637,37 @@ pub(crate) async fn generate_editor_character_animation_for_owner( })), ) })?; - // 背景色决策：显式传入 → 继承源角色图（前端从图层 generationInputs 带入）→ LLM 自动决策 → 默认色。 + // 先解析源角色图：图生视频的主体颜色完全由源图决定，背景色决策必须看到这张图。 + let source_resolve_client = build_upstream_http_client(SOURCE_IMAGE_RESOLVE_TIMEOUT_MS) + .map_err(|error| character_animation_error_response(&request_context, error))?; + let source_data_url = resolve_media_source_as_data_url( + &state, + &source_resolve_client, + payload.source_image_src.trim(), + "sourceImageSrc", + ) + .await + .map_err(|error| character_animation_error_response(&request_context, error))?; + // 背景色决策：显式传入 → 视觉 LLM 按源图主体配色自动决策 → 默认色。 let screen_background_decision = resolve_editor_screen_background_color( state.llm_client(), + state.creative_agent_gpt5_client(), EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::CharacterAnimation, screen_color: payload.screen_color.clone(), prompt: payload.prompt_text.clone(), icon_descriptions: Vec::new(), reference_count: 1, + source_image_data_url: Some(source_data_url.clone()), }, ) .await .map_err(|error| character_animation_error_response(&request_context, error))?; + // 把实际选定的背景色落进资产记录，供后续以该资产为参考图时反查旧背景色。 + let generation_inputs = apply_editor_screen_background_decision_to_generation_inputs( + generation_inputs, + Some(&screen_background_decision), + ); let normalized = normalize_editor_character_animation_request_with_pricing( payload, &pricing, @@ -656,18 +676,16 @@ pub(crate) async fn generate_editor_character_animation_for_owner( .map_err(|error| character_animation_error_response(&request_context, error))?; let settings = require_editor_character_animation_settings(&state, &normalized) .map_err(|error| character_animation_error_response(&request_context, error))?; + // 透明源图先合成到选定背景色：让视频背景确定性地等于抠图键色，而不是赌模型服从提示词。 + let source_data_url = composite_source_image_onto_screen_color( + source_data_url.as_str(), + screen_background_decision.color, + ) + .unwrap_or(source_data_url); let extraction_settings = resolve_backend_frame_extraction_settings(&state); let http_client = build_upstream_http_client(settings.ark.request_timeout_ms) .map_err(|error| character_animation_error_response(&request_context, error))?; let task_id = generate_ai_task_id(current_utc_micros()); - let source_data_url = resolve_media_source_as_data_url( - &state, - &http_client, - normalized.source_image_src.as_str(), - "sourceImageSrc", - ) - .await - .map_err(|error| character_animation_error_response(&request_context, error))?; let result = execute_billable_asset_operation_with_cost( &state, @@ -2907,7 +2925,8 @@ fn normalize_editor_character_animation_request_with_pricing( "sourceLayerId 不能为空。", )); } - let source_image_src = trim_optional_text(Some(payload.source_image_src.as_str())) + // 源图在 handler 里已提前解析（决策要看图）；这里保留空值校验，队列路径入队前也走这份校验。 + trim_optional_text(Some(payload.source_image_src.as_str())) .ok_or_else(|| editor_character_animation_bad_request("sourceImageSrc 不能为空。"))?; let prompt_text = payload .prompt_text @@ -2943,7 +2962,6 @@ fn normalize_editor_character_animation_request_with_pricing( Ok(NormalizedEditorCharacterAnimationRequest { source_layer_id, - source_image_src, prompt, screen_color, resolution: resolution.to_string(), @@ -4067,6 +4085,68 @@ fn parse_media_data_url(value: &str) -> Option { }) } +/// 把带透明通道的源图合成到选定背景色上，返回新的 PNG data URL。 +/// 视频模型不支持 alpha：透明源图会被按白底理解，且图生视频里源图背景的视觉证据 +/// 权重高于文字提示，导致提示词指定的背景色被无视。先填成实色让源图和提示词一致， +/// 视频背景就是抠图键色。源图本身不透明或无法解码时返回 None（沿用原图）。 +fn composite_source_image_onto_screen_color( + source_data_url: &str, + screen_color: EditorScreenBackgroundColor, +) -> Option { + let payload = parse_media_data_url(source_data_url)?; + if !payload.mime_type.starts_with("image/") { + return None; + } + let Ok(image) = image::load_from_memory(payload.bytes.as_slice()) else { + tracing::warn!("character_animation_source_composite_skip_undecodable_image"); + return None; + }; + let rgba = image.to_rgba8(); + if rgba.pixels().all(|pixel| pixel.0[3] == u8::MAX) { + return None; + } + let background = [ + f32::from(screen_color.red), + f32::from(screen_color.green), + f32::from(screen_color.blue), + ]; + let mut composited = RgbaImage::new(rgba.width(), rgba.height()); + for (x, y, pixel) in rgba.enumerate_pixels() { + let [red, green, blue, alpha] = pixel.0; + let opacity = f32::from(alpha) / 255.0; + let blend = |source: u8, background: f32| -> u8 { + (f32::from(source) * opacity + background * (1.0 - opacity)).round() as u8 + }; + composited.put_pixel( + x, + y, + Rgba([ + blend(red, background[0]), + blend(green, background[1]), + blend(blue, background[2]), + u8::MAX, + ]), + ); + } + let mut bytes = Vec::new(); + if PngEncoder::new(&mut bytes) + .write_image( + composited.as_raw(), + composited.width(), + composited.height(), + ColorType::Rgba8.into(), + ) + .is_err() + { + tracing::warn!("character_animation_source_composite_skip_encode_failed"); + return None; + } + Some(format!( + "data:image/png;base64,{}", + encode_base64(bytes.as_slice()) + )) +} + fn resolve_object_key_from_legacy_path(value: &str, field: &str) -> Result { let trimmed = value.trim(); if trimmed.is_empty() { @@ -5144,7 +5224,6 @@ struct EditorVideoSettings { #[derive(Debug)] struct NormalizedEditorCharacterAnimationRequest { source_layer_id: String, - source_image_src: String, prompt: String, resolution: String, ratio: String, @@ -5247,6 +5326,56 @@ mod tests { assert!(parse_video_data_url("data:image/png;base64,aGVsbG8=").is_none()); } + fn encode_rgba_png_data_url(image: &RgbaImage) -> String { + let mut bytes = Vec::new(); + PngEncoder::new(&mut bytes) + .write_image( + image.as_raw(), + image.width(), + image.height(), + ColorType::Rgba8.into(), + ) + .expect("test image should encode"); + format!("data:image/png;base64,{}", encode_base64(bytes.as_slice())) + } + + #[test] + fn composites_transparent_source_onto_screen_color() { + let mut image = RgbaImage::from_pixel(8, 8, Rgba([0, 0, 0, 0])); + image.put_pixel(4, 4, Rgba([10, 20, 30, 255])); + let screen_color = crate::editor_green_screen::EDITOR_SCREEN_BACKGROUND_COLORS[0]; + + let composited = + composite_source_image_onto_screen_color(&encode_rgba_png_data_url(&image), screen_color) + .expect("透明源图应被合成"); + let payload = parse_media_data_url(&composited).expect("合成结果应是图片 data URL"); + let output = image::load_from_memory(payload.bytes.as_slice()) + .expect("合成结果应可解码") + .to_rgba8(); + + assert_eq!( + output.get_pixel(0, 0).0, + [screen_color.red, screen_color.green, screen_color.blue, 255], + "透明像素应填成背景色" + ); + assert_eq!(output.get_pixel(4, 4).0, [10, 20, 30, 255], "不透明像素应保持原色"); + } + + #[test] + fn opaque_source_image_skips_compositing() { + let image = RgbaImage::from_pixel(4, 4, Rgba([1, 2, 3, 255])); + let screen_color = crate::editor_green_screen::EDITOR_SCREEN_BACKGROUND_COLORS[0]; + + assert!( + composite_source_image_onto_screen_color( + &encode_rgba_png_data_url(&image), + screen_color + ) + .is_none(), + "不透明源图应沿用原图" + ); + } + #[test] fn sanitize_storage_segment_falls_back_for_chinese_label() { assert_eq!( @@ -5475,7 +5604,9 @@ mod tests { assert!(prompt.contains("参考图作为首帧和尾帧")); assert!(prompt.contains(color.hex)); assert!(prompt.contains("纯色背景必须平整无纹理、无渐变、无阴影")); - assert!(prompt.contains("角色主体不得带与背景色相同或相近的描边、投影或反光")); + assert!( + prompt.contains("角色主体及其服饰、道具的颜色必须与背景色明显区分，不得带与背景色相同或相近的描边、投影或反光") + ); assert!(prompt.contains("动作描述：\n行走两步后回到站姿。")); } diff --git a/server-rs/crates/api-server/src/editor_green_screen.rs b/server-rs/crates/api-server/src/editor_green_screen.rs index 6ea4be842..34fa243f0 100644 --- a/server-rs/crates/api-server/src/editor_green_screen.rs +++ b/server-rs/crates/api-server/src/editor_green_screen.rs @@ -16,7 +16,7 @@ pub(crate) struct EditorScreenBackgroundColor { pub(crate) blue: u8, } -pub(crate) const EDITOR_SCREEN_BACKGROUND_COLORS: [EditorScreenBackgroundColor; 11] = [ +pub(crate) const EDITOR_SCREEN_BACKGROUND_COLORS: [EditorScreenBackgroundColor; 12] = [ EditorScreenBackgroundColor { label: "浅雾蓝", hex: "#CFEFFF", @@ -94,6 +94,15 @@ pub(crate) const EDITOR_SCREEN_BACKGROUND_COLORS: [EditorScreenBackgroundColor; green: 247, blue: 240, }, + // 中明度低饱和柔雾绿：填补安全弧的绿色段。浅粉彩绿（淡薄荷绿）会被露肤角色的皮肤规则剔除， + // 故绿色改用中明度以过 RGB 分离（Rule 3）；色相远离皮肤橙调，过色调投影（Rule 2）。 + EditorScreenBackgroundColor { + label: "灰竹绿", + hex: "#A0BBA0", + red: 160, + green: 187, + blue: 160, + }, ]; pub(crate) fn default_editor_screen_background_color() -> EditorScreenBackgroundColor { @@ -127,9 +136,9 @@ pub(crate) fn parse_editor_screen_background_color( pub(crate) const EDITOR_GREEN_SCREEN_BACKGROUND_GUARDRAILS: &str = "纯色背景必须平整无纹理、无渐变、无阴影、无地面、无环境、无道具"; pub(crate) const EDITOR_GREEN_SCREEN_ASSET_GUARDRAILS: &str = - "素材自身不要出现与背景色相同或相近的描边、底板、投影或反光"; + "素材主体及其配色必须与背景色明显区分，不要出现与背景色相同或相近的描边、底板、投影或反光"; pub(crate) const EDITOR_GREEN_SCREEN_CHARACTER_GUARDRAILS: &str = - "角色主体不得带与背景色相同或相近的描边、投影或反光"; + "角色主体及其服饰、道具的颜色必须与背景色明显区分，不得带与背景色相同或相近的描边、投影或反光"; fn editor_screen_background_color_prompt(color: EditorScreenBackgroundColor) -> String { if color.hex == "#00FF00" { diff --git a/server-rs/crates/api-server/src/editor_project.rs b/server-rs/crates/api-server/src/editor_project.rs index 1e7d1dd66..9558c3f6d 100644 --- a/server-rs/crates/api-server/src/editor_project.rs +++ b/server-rs/crates/api-server/src/editor_project.rs @@ -1348,12 +1348,14 @@ pub(crate) async fn generate_editor_image_for_owner( Some( resolve_editor_screen_background_color( state.llm_client(), + state.creative_agent_gpt5_client(), EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::Character, screen_color: payload.screen_color.clone(), prompt: role_setting.clone(), icon_descriptions: Vec::new(), reference_count: payload.reference_image_srcs.as_ref().map_or(0, Vec::len), + source_image_data_url: None, }, ) .await?, @@ -1636,25 +1638,31 @@ fn editor_image_generation_billing_asset_kind(normalized_kind: Option<&str>) -> } } -fn apply_editor_screen_background_decision_to_generation_inputs( +pub(crate) fn apply_editor_screen_background_decision_to_generation_inputs( generation_inputs: Option, decision: Option<&EditorScreenBackgroundDecision>, ) -> Option { let Some(decision) = decision else { return generation_inputs; }; - let mut value = generation_inputs?; - let Some(fields) = value.get_mut("fields").and_then(Value::as_array_mut) else { - return Some(value); + let mut value = match generation_inputs { + Some(value) if value.is_object() => value, + Some(other) => return Some(other), + None => json!({ "fields": [], "references": [] }), }; let resolved_value = format_editor_screen_background_decision_input(decision); - for field in fields { - let title = field.get("title").and_then(Value::as_str); - if title == Some("抠图背景色") { + if let Some(fields) = value.get_mut("fields").and_then(Value::as_array_mut) { + if let Some(field) = fields + .iter_mut() + .find(|field| field.get("title").and_then(Value::as_str) == Some("抠图背景色")) + { field["value"] = Value::String(resolved_value); - break; + } else { + fields.push(json!({ "title": "抠图背景色", "value": resolved_value })); } } + // 结构化背景色，供旧背景色先验等程序化消费；展示字段的中文格式不可作解析依据。 + value["screenColorHex"] = Value::String(decision.color.hex.to_string()); Some(value) } @@ -2727,12 +2735,14 @@ pub(crate) async fn generate_editor_icon_spritesheet_for_owner( let icon_descriptions = normalize_icon_descriptions(payload.icon_descriptions)?; let screen_background_decision = resolve_editor_screen_background_color( state.llm_client(), + state.creative_agent_gpt5_client(), EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::IconSpritesheet, screen_color: payload.screen_color.clone(), prompt: icon_descriptions.join("\n"), icon_descriptions: icon_descriptions.clone(), reference_count: 1 + payload.reference_image_srcs.as_ref().map_or(0, Vec::len), + source_image_data_url: None, }, ) .await?; @@ -2994,6 +3004,7 @@ pub(crate) async fn extract_editor_ui_design_assets_for_owner( ) -> Result, AppError> { let screen_background_decision = resolve_editor_screen_background_color( state.llm_client(), + state.creative_agent_gpt5_client(), EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::UiDesignAssetExtraction, screen_color: payload.screen_color.clone(), @@ -3003,6 +3014,7 @@ pub(crate) async fn extract_editor_ui_design_assets_for_owner( ), icon_descriptions: Vec::new(), reference_count: 1 + payload.reference_image_srcs.as_ref().map_or(0, Vec::len), + source_image_data_url: None, }, ) .await?; @@ -5525,6 +5537,69 @@ mod tests { ); } + fn manual_screen_background_decision(hex: &str) -> EditorScreenBackgroundDecision { + EditorScreenBackgroundDecision { + color: parse_editor_screen_background_color(Some(hex)) + .expect("hex should be a known background color candidate"), + mode: crate::editor_screen_background_decision::EditorScreenBackgroundDecisionMode::Manual, + attempts: 0, + fallback: false, + } + } + + #[test] + fn screen_background_decision_generation_inputs_record_structured_hex() { + let decision = manual_screen_background_decision("#B0C2E0"); + + let updated = apply_editor_screen_background_decision_to_generation_inputs( + Some(json!({ + "fields": [{ "title": "角色设定", "value": "红发骑士" }], + "references": [], + })), + Some(&decision), + ) + .expect("generation inputs should be preserved"); + assert_eq!(updated["screenColorHex"], json!("#B0C2E0")); + assert_eq!( + updated["fields"], + json!([ + { "title": "角色设定", "value": "红发骑士" }, + { "title": "抠图背景色", "value": "浅钢蓝 #B0C2E0" }, + ]), + ); + + let created = + apply_editor_screen_background_decision_to_generation_inputs(None, Some(&decision)) + .expect("missing generation inputs should still record the decision"); + assert_eq!( + created, + json!({ + "fields": [{ "title": "抠图背景色", "value": "浅钢蓝 #B0C2E0" }], + "references": [], + "screenColorHex": "#B0C2E0", + }), + ); + } + + #[test] + fn screen_background_decision_generation_inputs_overwrite_existing_display_field() { + let decision = manual_screen_background_decision("#CFEFFF"); + + let updated = apply_editor_screen_background_decision_to_generation_inputs( + Some(json!({ + "fields": [{ "title": "抠图背景色", "value": "旧值" }], + "references": [], + })), + Some(&decision), + ) + .expect("generation inputs should be preserved"); + assert_eq!( + updated["fields"], + json!([{ "title": "抠图背景色", "value": "浅雾蓝 #CFEFFF" }]), + ); + assert_eq!(updated["screenColorHex"], json!("#CFEFFF")); + } + fn apply_editor_canvas_generation_completion( layers: Value, completion: &EditorCanvasGenerationCompletionRequest, @@ -6195,7 +6270,9 @@ mod tests { assert!(prompt.contains("生成游戏角色立绘")); assert!(prompt.contains("背景固定为单一纯色背景 暖浅桃色 #FFD6C2 / RGB(255,214,194)")); assert!(prompt.contains("纯色背景必须平整无纹理、无渐变、无阴影")); - assert!(prompt.contains("角色主体不得带与背景色相同或相近的描边、投影或反光")); + assert!( + prompt.contains("角色主体及其服饰、道具的颜色必须与背景色明显区分，不得带与背景色相同或相近的描边、投影或反光") + ); assert!(prompt.contains("禁止镜头透视")); assert!(prompt.contains("角色设定：菜市场卖菜大妈")); } @@ -6737,7 +6814,9 @@ mod tests { assert!(prompt.contains("参考图1的图标素材规范")); assert!(prompt.contains("背景必须是单一纯色背景 中度天蓝 #7FB3FF / RGB(127,179,255)")); assert!(prompt.contains("平整无纹理、无渐变、无阴影")); - assert!(prompt.contains("素材自身不要出现与背景色相同或相近的描边、底板、投影或反光")); + assert!( + prompt.contains("素材主体及其配色必须与背景色明显区分，不要出现与背景色相同或相近的描边、底板、投影或反光") + ); assert!(prompt.contains("返回按钮、设置按钮、下一关按钮")); } @@ -6759,7 +6838,7 @@ mod tests { parse_editor_screen_background_color(None) .expect("default screen color should parse") ), - "仅提取被红色框框选的素材并整理成spritesheet，图集背景必须使用单一纯色背景 浅雾蓝 #CFEFFF / RGB(207,239,255)。纯色背景必须平整无纹理、无渐变、无阴影、无地面、无环境、无道具，方便后续扣除背景；素材自身不要出现与背景色相同或相近的描边、底板、投影或反光。" + "仅提取被红色框框选的素材并整理成spritesheet，图集背景必须使用单一纯色背景 浅雾蓝 #CFEFFF / RGB(207,239,255)。纯色背景必须平整无纹理、无渐变、无阴影、无地面、无环境、无道具，方便后续扣除背景；素材主体及其配色必须与背景色明显区分，不要出现与背景色相同或相近的描边、底板、投影或反光。" ); } diff --git a/server-rs/crates/api-server/src/editor_screen_background_decision.rs b/server-rs/crates/api-server/src/editor_screen_background_decision.rs index 848520451..d2f59a7bd 100644 --- a/server-rs/crates/api-server/src/editor_screen_background_decision.rs +++ b/server-rs/crates/api-server/src/editor_screen_background_decision.rs @@ -1,4 +1,4 @@ -use platform_llm::{LlmClient, LlmTextRequest}; +use platform_llm::{LlmClient, LlmMessage, LlmTextRequest}; use serde_json::json; use tracing::{info, warn}; @@ -7,6 +7,10 @@ use crate::{ EDITOR_SCREEN_BACKGROUND_COLORS, EditorScreenBackgroundColor, default_editor_screen_background_color, parse_editor_screen_background_color, }, + editor_screen_background_filter::{ + ScreenBackgroundColorSafetyReport, decode_image_data_url, + filter_editor_screen_background_colors, safety_report_log_json, + }, http_error::AppError, }; @@ -54,11 +58,15 @@ pub(crate) struct EditorScreenBackgroundDecisionInput { pub(crate) prompt: String, pub(crate) icon_descriptions: Vec, pub(crate) reference_count: usize, + /// 主体参考图（data URL）。图生视频等主体颜色由参考图决定的场景必须传， + /// 让视觉 LLM 按主体实际配色避开撞色背景；纯文生图场景可为 None。 + pub(crate) source_image_data_url: Option, } pub(crate) async fn resolve_editor_screen_background_color( llm_client: Option<&LlmClient>, - input: EditorScreenBackgroundDecisionInput, + vision_llm_client: Option<&LlmClient>, + mut input: EditorScreenBackgroundDecisionInput, ) -> Result { if !is_auto_screen_background_color(input.screen_color.as_deref()) { return Ok(EditorScreenBackgroundDecision { @@ -69,7 +77,50 @@ pub(crate) async fn resolve_editor_screen_background_color( }); } - let fallback_color = default_editor_screen_background_color(); + let has_source_image = input + .source_image_data_url + .as_deref() + .map(str::trim) + .is_some_and(|value| !value.is_empty()); + // 硬过滤先于 LLM：按参考图前景配色剔除撞色候选，LLM 只在安全集合里做审美选择。 + // 分析失败（图片不可解码、前景不足）只降级为不过滤，不阻断生成。 + let safety_report = if has_source_image { + resolve_screen_background_safety_report(&input) + } else { + None + }; + let candidate_colors: Vec = safety_report + .as_ref() + .map(|report| report.allowed.clone()) + .unwrap_or_else(|| EDITOR_SCREEN_BACKGROUND_COLORS.to_vec()); + + // 默认 llm_client 是纯文本模型（Ark），收到图片分片会被上游 400 拒绝； + // 带图决策必须走 VectorEngine 视觉客户端，没有视觉客户端时降级为纯文本决策。 + let (llm_client, vision_model) = if has_source_image { + match vision_llm_client { + Some(client) => ( + Some(client), + Some(crate::llm_model_routing::EDITOR_SCREEN_BACKGROUND_VISION_LLM_MODEL), + ), + None => { + warn!( + kind = input.kind.label(), + "editor_screen_background_vision_client_missing_downgrade_to_text" + ); + input.source_image_data_url = None; + (llm_client, None) + } + } + } else { + (llm_client, None) + }; + + // 默认兜底色被过滤掉时，改用危险度最小的候选兜底。 + let default_color = default_editor_screen_background_color(); + let fallback_color = match safety_report.as_ref() { + Some(report) if !report.contains(default_color.hex) => report.safest, + _ => default_color, + }; let Some(llm_client) = llm_client else { warn!( kind = input.kind.label(), @@ -85,15 +136,33 @@ pub(crate) async fn resolve_editor_screen_background_color( }; let system_prompt = editor_screen_background_decision_system_prompt(); - let user_prompt = editor_screen_background_decision_user_prompt(&input); + let user_prompt = + editor_screen_background_decision_user_prompt(&input, &candidate_colors, fallback_color); + let source_image_data_url = input + .source_image_data_url + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()); let mut last_error: Option = None; for attempt in 1..=EDITOR_SCREEN_BACKGROUND_DECISION_MAX_ATTEMPTS { - let request = LlmTextRequest::single_turn(system_prompt, user_prompt.as_str()) + let user_message = match source_image_data_url { + Some(image_url) => { + LlmMessage::user(user_prompt.as_str()).with_image_url(image_url) + } + None => LlmMessage::user(user_prompt.as_str()), + }; + let mut request = LlmTextRequest::new(vec![LlmMessage::system(system_prompt), user_message]) .with_max_tokens(96) .with_request_timeout_ms(EDITOR_SCREEN_BACKGROUND_DECISION_TIMEOUT_MS); + if let Some(vision_model) = vision_model { + request = request.with_model(vision_model); + } match llm_client.request_text(request).await { Ok(response) => { - match parse_editor_screen_background_decision_response(response.content.as_str()) { + match parse_editor_screen_background_decision_response( + response.content.as_str(), + &candidate_colors, + ) { Some(color) => { info!( kind = input.kind.label(), @@ -148,6 +217,51 @@ pub(crate) async fn resolve_editor_screen_background_color( }) } +/// 解出参考图并做候选色硬过滤；任何一步失败都返回 None（降级为不过滤）。 +fn resolve_screen_background_safety_report( + input: &EditorScreenBackgroundDecisionInput, +) -> Option { + let data_url = input.source_image_data_url.as_deref()?.trim(); + let Some(image_bytes) = decode_image_data_url(data_url) else { + warn!( + kind = input.kind.label(), + "editor_screen_background_filter_skip_undecodable_data_url" + ); + return None; + }; + // 角色/动作类启用皮肤专属硬否决（皮肤恒暖恒在，是约束最紧的前景）；图标/UI 类只跑统计过滤。 + let apply_skin_veto = matches!( + input.kind, + EditorScreenBackgroundDecisionKind::Character + | EditorScreenBackgroundDecisionKind::CharacterAnimation + ); + match filter_editor_screen_background_colors(&image_bytes, apply_skin_veto) { + Ok(report) => { + if report.excluded.is_empty() { + info!( + kind = input.kind.label(), + "editor_screen_background_filter_no_exclusion" + ); + } else { + info!( + kind = input.kind.label(), + result = %safety_report_log_json(&report), + "editor_screen_background_filter_excluded_colliding_colors" + ); + } + Some(report) + } + Err(reason) => { + warn!( + kind = input.kind.label(), + reason = reason.as_str(), + "editor_screen_background_filter_skip_analysis_failed" + ); + None + } + } +} + pub(crate) fn is_auto_screen_background_color(value: Option<&str>) -> bool { value .map(str::trim) @@ -174,13 +288,15 @@ pub(crate) fn format_editor_screen_background_decision_input( } fn editor_screen_background_decision_system_prompt() -> &'static str { - "你是游戏素材生图的抠图背景色决策器。根据任务和用户输入，从候选列表中选择一个最不容易与主体混淆、最适合后续扣除的纯色背景。只能返回 JSON：{\"hex\":\"#RRGGBB\"}。不要解释，不要返回候选列表外的颜色。" + "你是游戏素材生图的抠图背景色决策器。根据任务和用户输入，从候选列表中选择一个最不容易与主体混淆、最适合后续扣除的纯色背景。如果附带了主体参考图，必须以图中主体（含服饰、道具、光效）的实际配色为准，选择色相距离最远的候选色，忽略图中已有的背景或透明区域。只能返回 JSON：{\"hex\":\"#RRGGBB\"}。不要解释，不要返回候选列表外的颜色。" } fn editor_screen_background_decision_user_prompt( input: &EditorScreenBackgroundDecisionInput, + candidate_colors: &[EditorScreenBackgroundColor], + fallback_color: EditorScreenBackgroundColor, ) -> String { - let colors = EDITOR_SCREEN_BACKGROUND_COLORS + let colors = candidate_colors .iter() .map(|color| { format!( @@ -190,16 +306,23 @@ fn editor_screen_background_decision_user_prompt( }) .collect::>() .join("\n"); + let has_source_image = input + .source_image_data_url + .as_deref() + .map(str::trim) + .is_some_and(|value| !value.is_empty()); let context = json!({ "task": input.kind.label(), "userPrompt": input.prompt, "iconDescriptions": input.icon_descriptions, "referenceCount": input.reference_count, + "hasSourceImage": has_source_image, "selectionCriteria": [ "优先选择与主体、主题色、UI 面板底色、发光效果和描边明显区分的背景色", "避免选择可能出现在主体、服饰、道具、按钮、图标或 UI 面板里的颜色", + "有主体参考图时以图中主体实际配色为准，文本描述只作辅助", "角色类优先低干扰浅色，图标和 UI 提取类优先更高对比但仍保持纯净", - "信息不足时选择浅雾蓝 #CFEFFF" + format!("信息不足时选择{} {}", fallback_color.label, fallback_color.hex) ], }); format!("候选背景色：\n{colors}\n\n任务上下文 JSON：\n{context}") @@ -207,27 +330,32 @@ fn editor_screen_background_decision_user_prompt( fn parse_editor_screen_background_decision_response( content: &str, + candidate_colors: &[EditorScreenBackgroundColor], ) -> Option { let trimmed = content.trim(); let json_source = extract_json_object(trimmed).unwrap_or(trimmed); if let Ok(value) = serde_json::from_str::(json_source) { if let Some(hex) = value.get("hex").and_then(serde_json::Value::as_str) { - return find_editor_screen_background_color(hex); + return find_editor_screen_background_color(candidate_colors, hex); } if let Some(hex) = value .get("screenColor") .or_else(|| value.get("screen_color")) .and_then(serde_json::Value::as_str) { - return find_editor_screen_background_color(hex); + return find_editor_screen_background_color(candidate_colors, hex); } } - find_hex_in_text(trimmed).and_then(find_editor_screen_background_color) + find_hex_in_text(trimmed) + .and_then(|hex| find_editor_screen_background_color(candidate_colors, hex)) } -fn find_editor_screen_background_color(value: &str) -> Option { +fn find_editor_screen_background_color( + candidate_colors: &[EditorScreenBackgroundColor], + value: &str, +) -> Option { let normalized = value.trim().to_ascii_uppercase(); - EDITOR_SCREEN_BACKGROUND_COLORS + candidate_colors .iter() .copied() .find(|color| color.hex == normalized) @@ -266,6 +394,7 @@ mod tests { #[tokio::test] async fn manual_screen_background_decision_keeps_selected_color() { let decision = resolve_editor_screen_background_color( + None, None, EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::Character, @@ -273,6 +402,7 @@ mod tests { prompt: "绿色史莱姆".to_string(), icon_descriptions: Vec::new(), reference_count: 1, + source_image_data_url: None, }, ) .await @@ -286,6 +416,7 @@ mod tests { #[tokio::test] async fn auto_screen_background_decision_falls_back_without_llm() { let decision = resolve_editor_screen_background_color( + None, None, EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::IconSpritesheet, @@ -293,6 +424,7 @@ mod tests { prompt: String::new(), icon_descriptions: vec!["金币按钮".to_string()], reference_count: 2, + source_image_data_url: None, }, ) .await @@ -306,6 +438,7 @@ mod tests { #[tokio::test] async fn missing_screen_background_decision_uses_auto_default() { let decision = resolve_editor_screen_background_color( + None, None, EditorScreenBackgroundDecisionInput { kind: EditorScreenBackgroundDecisionKind::UiDesignAssetExtraction, @@ -313,6 +446,7 @@ mod tests { prompt: "提取 UI 按钮".to_string(), icon_descriptions: Vec::new(), reference_count: 1, + source_image_data_url: None, }, ) .await @@ -325,9 +459,11 @@ mod tests { #[test] fn parses_llm_json_background_decision() { - let color = - parse_editor_screen_background_decision_response("```json\n{\"hex\":\"#a8f7f0\"}\n```") - .expect("known color should parse"); + let color = parse_editor_screen_background_decision_response( + "```json\n{\"hex\":\"#a8f7f0\"}\n```", + &EDITOR_SCREEN_BACKGROUND_COLORS, + ) + .expect("known color should parse"); assert_eq!(color.label, "高对比浅青"); } @@ -335,7 +471,68 @@ mod tests { #[test] fn rejects_unknown_llm_background_decision() { assert!( - parse_editor_screen_background_decision_response("{\"hex\":\"#123456\"}").is_none() + parse_editor_screen_background_decision_response( + "{\"hex\":\"#123456\"}", + &EDITOR_SCREEN_BACKGROUND_COLORS, + ) + .is_none() + ); + } + + #[test] + fn rejects_filtered_out_llm_background_decision() { + // LLM 返回了被硬过滤剔除的颜色（不在候选集里）时按无效响应处理。 + let allowed = [EDITOR_SCREEN_BACKGROUND_COLORS[0]]; + assert!( + parse_editor_screen_background_decision_response("{\"hex\":\"#7FB3FF\"}", &allowed) + .is_none() + ); + } + + #[tokio::test] + async fn auto_decision_fallback_avoids_colliding_default_color() { + use base64::Engine as _; + use image::{Rgba, RgbaImage}; + + // 前景是与默认兜底色（浅雾蓝 #CFEFFF）同色的实心块： + // 无 LLM 时兜底不能再落在被硬过滤剔除的默认色上。 + let mut image = RgbaImage::from_pixel(64, 64, Rgba([0, 0, 0, 0])); + for y in 16..48 { + for x in 16..48 { + image.put_pixel(x, y, Rgba([207, 239, 255, 255])); + } + } + let mut bytes = Vec::new(); + image::DynamicImage::ImageRgba8(image) + .write_to( + &mut std::io::Cursor::new(&mut bytes), + image::ImageFormat::Png, + ) + .expect("test image should encode"); + let data_url = format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(&bytes) + ); + + let decision = resolve_editor_screen_background_color( + None, + None, + EditorScreenBackgroundDecisionInput { + kind: EditorScreenBackgroundDecisionKind::CharacterAnimation, + screen_color: Some("auto".to_string()), + prompt: "起跳动作".to_string(), + icon_descriptions: Vec::new(), + reference_count: 1, + source_image_data_url: Some(data_url), + }, + ) + .await + .expect("decision should fall back"); + + assert!(decision.fallback); + assert_ne!( + decision.color.hex, "#CFEFFF", + "默认兜底色与前景撞色时应改用危险度最小的候选" ); } } diff --git a/server-rs/crates/api-server/src/editor_screen_background_filter.rs b/server-rs/crates/api-server/src/editor_screen_background_filter.rs new file mode 100644 index 000000000..f5e85b6d9 --- /dev/null +++ b/server-rs/crates/api-server/src/editor_screen_background_filter.rs @@ -0,0 +1,611 @@ +//! 背景色候选硬过滤：统计参考图前景配色，剔除会与前景（尤其皮肤）撞色的候选背景色。 +//! +//! 抠图管线对像素的判定由 ΔE（Lab 距离）和色调投影（像素色度在背景色调方向上的投影） +//! 两个量决定：ΔE ≤ 6.7 的前景近乎必被抹除；ΔE ≤ 28 且投影 ≥ 4（同色调壳层）时命运交给 +//! 后端 matting，同色调恰是其最不可靠的区间。经验上危险质量占前景 0.5% 以上就会出现 +//! 可见破损，故以此为剔除阈值。 +//! +//! 角色/动作类另加皮肤专属硬否决（三判据取并集）：动漫皮肤恒暖且恒在，是约束最紧的前景。 +//! Rule 1 由危险质量的 ΔE 门（28）承载；Rule 2 皮肤色度在候选色调方向投影 < 4（同色调否决， +//! 覆盖 trimap hue-split 与 cross-check 削薄）；Rule 3 候选与最亮皮肤的 RGB 合成距离 ≥ 52 +//! （两个都很浅的暖色在 RGB 天然接近，Lab ΔE 防不住，是第二条判死路径）。 +//! 三条的净效果是把角色候选压进冷区（蓝/青/紫），LLM 只在安全候选里做审美选择。 + +use std::collections::HashMap; + +use base64::Engine as _; +use image::GenericImageView; +use serde_json::json; + +use crate::editor_green_screen::{EDITOR_SCREEN_BACKGROUND_COLORS, EditorScreenBackgroundColor}; + +/// 危险格 ΔE 上限：超过它的前景像素被管线稳定判为前景。实测第 13 帧损伤发生在 ΔE 22–30 +/// （前景最近分位 p0.5≈28），故取 28 覆盖真实杀伤半径；proj 门保证异色调 bin 不被误伤。 +const DANGER_DELTA_E: f64 = 28.0; +/// 无条件危险 ΔE 上限：低于它的前景像素不看色调也近乎必被抹除。 +const SURE_LOSS_DELTA_E: f64 = 6.7; +/// 色调投影阈值：投影低于它（色度与背景色调近正交）的像素会被规则救回。 +const DANGER_HUE_PROJECTION: f64 = 4.0; +/// 危险质量占比达到该值即剔除候选色。 +const DANGER_MASS_EXCLUSION_RATIO: f64 = 0.005; +/// 单个颜色桶低于该质量占比视为噪声（残留背景、孤立杂色），不参与危险统计。 +const BIN_MASS_FLOOR_RATIO: f64 = 0.001; +/// 分析用缩略图最长边。 +const ANALYSIS_MAX_EDGE: u32 = 192; +/// 不透明图边框背景估计：主色覆盖率达到该值才认为边框是平坦背景。 +const BORDER_BACKGROUND_MIN_COVERAGE: f64 = 0.5; +/// 不透明图中与边框背景色 ΔE 不超过该值的像素按背景剔除。 +const BORDER_BACKGROUND_DELTA_E: f64 = 12.0; + +// 皮肤 bin 判定：高 L*、a*/b* 双正的暖色，且质量足够。 +const SKIN_MIN_LIGHTNESS: f64 = 55.0; +const SKIN_MIN_A: f64 = 3.0; +const SKIN_MIN_B: f64 = 8.0; +/// 皮肤 bin 质量下限：低到能保护小脸（远景角色脸可能只占前景零点几个百分点）。 +const SKIN_BIN_MASS_FLOOR_RATIO: f64 = 0.001; +/// Rule 2：皮肤色度在候选色调方向的投影达到该值即同色调撞色，否决。 +const SKIN_HUE_PROJECTION_LIMIT: f64 = 4.0; +/// Rule 3：候选与最亮皮肤的 RGB 合成距离（0–255）低于该值即过近，否决。取 52 是管线 RGB 通路 +/// bg_conf 冲到 0.5 的反解并留余量；不取完全沉默半径 68，否则会误杀已验证安全的 #CFEFFF（L2≈66）。 +const SKIN_MIN_RGB_DISTANCE: f64 = 52.0; + +#[derive(Clone, Debug)] +pub(crate) struct ScreenBackgroundColorSafetyReport { + /// 过滤后仍可用的候选色（保持调色板顺序），保证非空。 + pub(crate) allowed: Vec, + /// 被剔除的候选色及其危险质量占比。 + pub(crate) excluded: Vec<(EditorScreenBackgroundColor, f64)>, + /// 兜底色：皮肤安全候选里危险质量最小者（无皮肤安全候选时退回全局最小）。 + pub(crate) safest: EditorScreenBackgroundColor, +} + +impl ScreenBackgroundColorSafetyReport { + pub(crate) fn contains(&self, hex: &str) -> bool { + self.allowed.iter().any(|color| color.hex == hex) + } + + #[cfg(test)] + pub(crate) fn excluded_summary(&self) -> String { + self.excluded + .iter() + .map(|(color, ratio)| format!("{} {}({:.1}‰)", color.label, color.hex, ratio * 1000.0)) + .collect::>() + .join(", ") + } +} + +/// 从 image data URL 解出图片字节；非图片或无法解码时返回 None。 +pub(crate) fn decode_image_data_url(value: &str) -> Option> { + let body = value.trim().strip_prefix("data:")?; + let (mime_type, data) = body.split_once(";base64,")?; + if !mime_type.trim().starts_with("image/") { + return None; + } + base64::engine::general_purpose::STANDARD + .decode(data.trim()) + .ok() + .filter(|bytes| !bytes.is_empty()) +} + +/// 按参考图前景配色过滤候选背景色。`apply_skin_veto` 为 true（角色/动作类）时，检测到皮肤 bin +/// 还会启用 Rule 2/3 皮肤硬否决。失败（图片不可解码、前景不足）返回原因字符串， +/// 调用方应降级为不过滤，而不是阻断生成。 +pub(crate) fn filter_editor_screen_background_colors( + image_bytes: &[u8], + apply_skin_veto: bool, +) -> Result { + let image = image::load_from_memory(image_bytes) + .map_err(|error| format!("参考图无法解码：{error}"))?; + let (width, height) = image.dimensions(); + if width == 0 || height == 0 { + return Err("参考图尺寸为空".to_string()); + } + let image = if width.max(height) > ANALYSIS_MAX_EDGE { + // 最近邻缩放不混色：双线性会把前景像素和透明/背景像素调和出不存在的颜色。 + image.resize( + ANALYSIS_MAX_EDGE, + ANALYSIS_MAX_EDGE, + image::imageops::FilterType::Nearest, + ) + } else { + image + }; + let rgba = image.to_rgba8(); + + let histogram = build_foreground_histogram(&rgba)?; + let total_mass = histogram.total_mass; + // 皮肤约束基准：最亮的皮肤 bin（RGB 上最贴近浅色背景，是判据最紧点）。 + let skin = if apply_skin_veto { + histogram.brightest_skin_bin() + } else { + None + }; + + let mut allowed = Vec::new(); + let mut excluded = Vec::new(); + let mut safest = EDITOR_SCREEN_BACKGROUND_COLORS[0]; + let mut safest_ratio = f64::INFINITY; + for color in EDITOR_SCREEN_BACKGROUND_COLORS { + let danger_ratio = danger_mass(&histogram, color) / total_mass; + let skin_unsafe = skin + .as_ref() + .is_some_and(|reference| candidate_collides_with_skin(reference, color)); + // 兜底只在皮肤安全候选里挑危险质量最小者，避免把撞肤色（可能危险质量很低）兜回来。 + if !skin_unsafe && danger_ratio < safest_ratio { + safest_ratio = danger_ratio; + safest = color; + } + if danger_ratio >= DANGER_MASS_EXCLUSION_RATIO || skin_unsafe { + excluded.push((color, danger_ratio)); + } else { + allowed.push(color); + } + } + if safest_ratio.is_infinite() { + // 所有候选都皮肤不安全：退回全局危险质量最小者。 + for color in EDITOR_SCREEN_BACKGROUND_COLORS { + let danger_ratio = danger_mass(&histogram, color) / total_mass; + if danger_ratio < safest_ratio { + safest_ratio = danger_ratio; + safest = color; + } + } + } + if allowed.is_empty() { + allowed.push(safest); + excluded.retain(|(color, _)| color != &safest); + } + Ok(ScreenBackgroundColorSafetyReport { + allowed, + excluded, + safest, + }) +} + +/// 皮肤约束基准：最亮皮肤 bin 的 Lab 色度与 RGB 均值。 +struct SkinReference { + lab: [f64; 3], + rgb: [f64; 3], +} + +struct ForegroundHistogram { + bins: HashMap<(i32, i32, i32), HistogramBin>, + total_mass: f64, +} + +impl ForegroundHistogram { + /// 取最亮的皮肤 bin 作为约束基准；没有合格皮肤 bin（纯蓝机器人等）时返回 None。 + fn brightest_skin_bin(&self) -> Option { + let mass_floor = self.total_mass * SKIN_BIN_MASS_FLOOR_RATIO; + self.bins + .values() + .filter(|bin| bin.mass >= mass_floor) + .filter_map(|bin| { + let lab = bin.mean(); + (lab[0] > SKIN_MIN_LIGHTNESS && lab[1] > SKIN_MIN_A && lab[2] > SKIN_MIN_B) + .then(|| SkinReference { + lab, + rgb: bin.mean_rgb(), + }) + }) + .max_by(|left, right| { + left.lab[0] + .partial_cmp(&right.lab[0]) + .unwrap_or(std::cmp::Ordering::Equal) + }) + } +} + +#[derive(Default)] +struct HistogramBin { + mass: f64, + sum_l: f64, + sum_a: f64, + sum_b: f64, + sum_red: f64, + sum_green: f64, + sum_blue: f64, +} + +impl HistogramBin { + fn mean(&self) -> [f64; 3] { + [ + self.sum_l / self.mass, + self.sum_a / self.mass, + self.sum_b / self.mass, + ] + } + + fn mean_rgb(&self) -> [f64; 3] { + [ + self.sum_red / self.mass, + self.sum_green / self.mass, + self.sum_blue / self.mass, + ] + } +} + +const BIN_L_STEP: f64 = 8.0; +const BIN_AB_STEP: f64 = 6.0; + +fn bin_key(lab: [f64; 3]) -> (i32, i32, i32) { + ( + (lab[0] / BIN_L_STEP).floor() as i32, + (lab[1] / BIN_AB_STEP).floor() as i32, + (lab[2] / BIN_AB_STEP).floor() as i32, + ) +} + +fn build_foreground_histogram(rgba: &image::RgbaImage) -> Result { + let has_transparency = rgba.pixels().any(|pixel| pixel.0[3] < 250); + let mut bins: HashMap<(i32, i32, i32), HistogramBin> = HashMap::new(); + let mut total_mass = 0.0; + let mut add = |lab: [f64; 3], rgb: [f64; 3], weight: f64| { + let bin = bins.entry(bin_key(lab)).or_default(); + bin.mass += weight; + bin.sum_l += lab[0] * weight; + bin.sum_a += lab[1] * weight; + bin.sum_b += lab[2] * weight; + bin.sum_red += rgb[0] * weight; + bin.sum_green += rgb[1] * weight; + bin.sum_blue += rgb[2] * weight; + total_mass += weight; + }; + + if has_transparency { + // alpha² 加权：半透明羽化带（残留背景最集中的位置）权重自然趋零， + // 实心小区域（如角色的脸）质量不受损。 + for pixel in rgba.pixels() { + let [red, green, blue, alpha] = pixel.0; + if alpha == 0 { + continue; + } + let opacity = f64::from(alpha) / 255.0; + add( + srgb_to_lab(red, green, blue), + [f64::from(red), f64::from(green), f64::from(blue)], + opacity * opacity, + ); + } + } else { + // 不透明图：用边框主色估计原背景，剔除与其相近的像素；边框不平坦则全图算前景。 + let border_background = estimate_border_background(rgba); + for pixel in rgba.pixels() { + let [red, green, blue, _] = pixel.0; + let lab = srgb_to_lab(red, green, blue); + if let Some(background_lab) = border_background { + if delta_e(lab, background_lab) <= BORDER_BACKGROUND_DELTA_E { + continue; + } + } + add( + lab, + [f64::from(red), f64::from(green), f64::from(blue)], + 1.0, + ); + } + } + + let pixel_count = f64::from(rgba.width()) * f64::from(rgba.height()); + if total_mass < pixel_count * 0.01 { + return Err("参考图前景像素不足，无法做背景色安全分析".to_string()); + } + Ok(ForegroundHistogram { bins, total_mass }) +} + +/// 估计不透明图的边框背景色：边框环里主导颜色桶覆盖率足够高才认为存在平坦背景。 +fn estimate_border_background(rgba: &image::RgbaImage) -> Option<[f64; 3]> { + let (width, height) = rgba.dimensions(); + let ring = (width.min(height) / 32).max(1); + let mut bins: HashMap<(i32, i32, i32), HistogramBin> = HashMap::new(); + let mut total = 0.0; + for (x, y, pixel) in rgba.enumerate_pixels() { + let on_border = x < ring + || y < ring + || x >= width.saturating_sub(ring) + || y >= height.saturating_sub(ring); + if !on_border { + continue; + } + let [red, green, blue, _] = pixel.0; + let lab = srgb_to_lab(red, green, blue); + let bin = bins.entry(bin_key(lab)).or_default(); + bin.mass += 1.0; + bin.sum_l += lab[0]; + bin.sum_a += lab[1]; + bin.sum_b += lab[2]; + total += 1.0; + } + let dominant = bins.values().max_by(|left, right| { + left.mass + .partial_cmp(&right.mass) + .unwrap_or(std::cmp::Ordering::Equal) + })?; + (dominant.mass / total >= BORDER_BACKGROUND_MIN_COVERAGE).then(|| dominant.mean()) +} + +/// 候选背景色的危险质量：落入「ΔE ≤ 6.7 无条件」或「ΔE ≤ 28 且同色调（投影 ≥ 4）」的前景质量之和。 +fn danger_mass(histogram: &ForegroundHistogram, color: EditorScreenBackgroundColor) -> f64 { + let background_lab = srgb_to_lab(color.red, color.green, color.blue); + let chroma_norm = background_lab[1].hypot(background_lab[2]); + let mass_floor = histogram.total_mass * BIN_MASS_FLOOR_RATIO; + let mut danger = 0.0; + for bin in histogram.bins.values() { + if bin.mass < mass_floor { + continue; + } + let mean = bin.mean(); + let distance = delta_e(mean, background_lab); + if distance > DANGER_DELTA_E { + continue; + } + let projection = if chroma_norm > 1e-6 { + (mean[1] * background_lab[1] + mean[2] * background_lab[2]) / chroma_norm + } else { + mean[1].hypot(mean[2]) + }; + if distance <= SURE_LOSS_DELTA_E || projection >= DANGER_HUE_PROJECTION { + danger += bin.mass; + } + } + danger +} + +/// 皮肤专属硬否决（Rule 2 + Rule 3）：皮肤色度在候选色调方向投影 ≥ 4（同色调）， +/// 或候选与最亮皮肤的 RGB 合成距离 < 52（两个浅暖色天然接近）。任一命中即否决。 +fn candidate_collides_with_skin(skin: &SkinReference, color: EditorScreenBackgroundColor) -> bool { + let background_lab = srgb_to_lab(color.red, color.green, color.blue); + let chroma_norm = background_lab[1].hypot(background_lab[2]); + // Rule 2：近中性灰候选无色调可撞（chroma≈0），投影记 0，交给 Rule 3 的亮度分离判定。 + let projection = if chroma_norm > 1e-6 { + (skin.lab[1] * background_lab[1] + skin.lab[2] * background_lab[2]) / chroma_norm + } else { + 0.0 + }; + if projection >= SKIN_HUE_PROJECTION_LIMIT { + return true; + } + let delta_red = f64::from(color.red) - skin.rgb[0]; + let delta_green = f64::from(color.green) - skin.rgb[1]; + let delta_blue = f64::from(color.blue) - skin.rgb[2]; + let rgb_distance = + (delta_red * delta_red + delta_green * delta_green + delta_blue * delta_blue).sqrt(); + rgb_distance < SKIN_MIN_RGB_DISTANCE +} + +fn delta_e(left: [f64; 3], right: [f64; 3]) -> f64 { + let dl = left[0] - right[0]; + let da = left[1] - right[1]; + let db = left[2] - right[2]; + (dl * dl + da * da + db * db).sqrt() +} + +fn srgb_to_lab(red: u8, green: u8, blue: u8) -> [f64; 3] { + fn linearize(channel: u8) -> f64 { + let value = f64::from(channel) / 255.0; + if value <= 0.04045 { + value / 12.92 + } else { + ((value + 0.055) / 1.055).powf(2.4) + } + } + let (r, g, b) = (linearize(red), linearize(green), linearize(blue)); + let x = r * 0.4124564 + g * 0.3575761 + b * 0.1804375; + let y = r * 0.2126729 + g * 0.7151522 + b * 0.0721750; + let z = r * 0.0193339 + g * 0.1191920 + b * 0.9503041; + fn transfer(t: f64) -> f64 { + if t > 0.008856 { + t.cbrt() + } else { + 7.787 * t + 16.0 / 116.0 + } + } + let (fx, fy, fz) = ( + transfer(x / 0.95047), + transfer(y / 1.0), + transfer(z / 1.08883), + ); + [116.0 * fy - 16.0, 500.0 * (fx - fy), 200.0 * (fy - fz)] +} + +/// 供日志使用的过滤结果 JSON。 +pub(crate) fn safety_report_log_json( + report: &ScreenBackgroundColorSafetyReport, +) -> serde_json::Value { + json!({ + "allowed": report.allowed.iter().map(|color| color.hex).collect::>(), + "excluded": report + .excluded + .iter() + .map(|(color, ratio)| json!({ "hex": color.hex, "dangerPermille": (ratio * 1000.0 * 10.0).round() / 10.0 })) + .collect::>(), + "safest": report.safest.hex, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use image::{Rgba, RgbaImage}; + + fn encode_png(image: &RgbaImage) -> Vec { + let mut bytes = Vec::new(); + image::DynamicImage::ImageRgba8(image.clone()) + .write_to( + &mut std::io::Cursor::new(&mut bytes), + image::ImageFormat::Png, + ) + .expect("test image should encode"); + bytes + } + + fn transparent_image_with_center_block(color: [u8; 3]) -> RgbaImage { + let mut image = RgbaImage::from_pixel(64, 64, Rgba([0, 0, 0, 0])); + for y in 16..48 { + for x in 16..48 { + image.put_pixel(x, y, Rgba([color[0], color[1], color[2], 255])); + } + } + image + } + + fn report_for(image: &RgbaImage, apply_skin_veto: bool) -> ScreenBackgroundColorSafetyReport { + filter_editor_screen_background_colors(&encode_png(image), apply_skin_veto) + .expect("filter should analyze the test image") + } + + #[test] + fn skin_bearing_character_excludes_warm_keeps_cool() { + // 亮暖肤块：Rule 2/3 应同时剔除暖桃与淡黄（两次真实事故的撞色），保留冷色。 + let report = report_for(&transparent_image_with_center_block([250, 224, 200]), true); + + assert!( + report.excluded.iter().any(|(color, _)| color.hex == "#FFD6C2"), + "肤色前景应剔除暖浅桃色，excluded: {}", + report.excluded_summary() + ); + assert!( + report.excluded.iter().any(|(color, _)| color.hex == "#FFF2A8"), + "肤色前景应剔除淡黄（Rule 2/3 关键新覆盖），excluded: {}", + report.excluded_summary() + ); + assert!(report.contains("#CFEFFF"), "冷色浅雾蓝应保留"); + assert_ne!(report.safest.hex, "#FFD6C2"); + assert_ne!(report.safest.hex, "#FFF2A8"); + } + + #[test] + fn skin_veto_gating_only_matters_for_minority_skin() { + // 大面积暗中性前景 + 极小肤块（占前景 ~0.3%，低于危险质量阈值 0.5%）： + // 关闭皮肤否决时统计过滤抓不到淡黄；开启时皮肤规则应抓到。这正是「小脸」场景。 + let mut image = RgbaImage::from_pixel(64, 64, Rgba([0, 0, 0, 0])); + for y in 0..56 { + for x in 0..54 { + image.put_pixel(x, y, Rgba([40, 40, 40, 255])); // 3024 px 暗中性，不撞任何浅候选 + } + } + for y in 60..63 { + for x in 60..63 { + image.put_pixel(x, y, Rgba([250, 224, 200, 255])); // 9 px 肤块 ≈ 0.3% 前景 + } + } + + let disabled = report_for(&image, false); + assert!( + disabled.contains("#FFF2A8"), + "关闭皮肤否决时小肤块不足以统计剔除淡黄，excluded: {}", + disabled.excluded_summary() + ); + + let enabled = report_for(&image, true); + assert!( + enabled.excluded.iter().any(|(color, _)| color.hex == "#FFF2A8"), + "开启皮肤否决时淡黄应被剔除，excluded: {}", + enabled.excluded_summary() + ); + } + + #[test] + fn no_skin_detected_skips_skin_veto() { + // 无皮肤 bin（纯青前景）即使 apply_skin_veto=true 也不否决暖色。 + let report = report_for(&transparent_image_with_center_block([0, 180, 180]), true); + assert!( + report.contains("#FFD6C2"), + "无皮肤时暖桃不应被皮肤规则剔除，excluded: {}", + report.excluded_summary() + ); + } + + #[test] + fn blue_foreground_excludes_blue_candidate() { + let report = report_for(&transparent_image_with_center_block([127, 179, 255]), false); + + assert!( + report.excluded.iter().any(|(color, _)| color.hex == "#7FB3FF"), + "蓝色前景应剔除中度天蓝，excluded: {}", + report.excluded_summary() + ); + assert!(report.contains("#FFD6C2"), "暖浅桃色应保留"); + } + + #[test] + fn semi_transparent_fringe_does_not_exclude_candidates() { + // 前景是红色实心块，另有 100 个 alpha=32 的中度天蓝残留像素（模拟没滤净的旧背景羽化带）。 + let mut image = transparent_image_with_center_block([200, 30, 30]); + let mut placed = 0u32; + 'outer: for y in 0..16u32 { + for x in 0..64u32 { + if placed >= 100 { + break 'outer; + } + image.put_pixel(x, y, Rgba([127, 179, 255, 32])); + placed += 1; + } + } + + let report = report_for(&image, false); + assert!( + report.contains("#7FB3FF"), + "半透明残留不应剔除候选色，excluded: {}", + report.excluded_summary() + ); + } + + #[test] + fn opaque_flat_background_is_not_counted_as_foreground() { + // 不透明图：浅雾蓝底 + 深红主体。若边框背景估计失效，浅雾蓝会被误判撞色。 + let mut image = RgbaImage::from_pixel(64, 64, Rgba([207, 239, 255, 255])); + for y in 24..40 { + for x in 24..40 { + image.put_pixel(x, y, Rgba([139, 0, 0, 255])); + } + } + + let report = report_for(&image, false); + assert!( + report.contains("#CFEFFF"), + "平坦背景应被剔除出前景统计，excluded: {}", + report.excluded_summary() + ); + } + + #[test] + fn all_candidates_dangerous_keeps_safest_only() { + // 前景铺满全部候选色，任何候选都撞色 → 只保留危险度最小的一个。 + let stripe = 6u32; + let width = EDITOR_SCREEN_BACKGROUND_COLORS.len() as u32 * stripe; + let mut image = RgbaImage::from_pixel(width, 64, Rgba([0, 0, 0, 0])); + for (index, color) in EDITOR_SCREEN_BACKGROUND_COLORS.iter().enumerate() { + let x_start = index as u32 * stripe; + for y in 0..64 { + for x in x_start..x_start + stripe { + image.put_pixel(x, y, Rgba([color.red, color.green, color.blue, 255])); + } + } + } + + let report = report_for(&image, false); + assert_eq!(report.allowed.len(), 1); + assert_eq!(report.allowed[0].hex, report.safest.hex); + } + + #[test] + fn fully_transparent_image_fails_analysis() { + let image = RgbaImage::from_pixel(32, 32, Rgba([0, 0, 0, 0])); + assert!(filter_editor_screen_background_colors(&encode_png(&image), true).is_err()); + } + + #[test] + fn decodes_image_data_url() { + let image = transparent_image_with_center_block([10, 20, 30]); + let bytes = encode_png(&image); + let data_url = format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(&bytes) + ); + + assert_eq!(decode_image_data_url(&data_url), Some(bytes)); + assert_eq!(decode_image_data_url("data:video/mp4;base64,AAAA"), None); + assert_eq!(decode_image_data_url("not a data url"), None); + } +} diff --git a/server-rs/crates/api-server/src/llm_model_routing.rs b/server-rs/crates/api-server/src/llm_model_routing.rs index 4fd986fde..16cd06539 100644 --- a/server-rs/crates/api-server/src/llm_model_routing.rs +++ b/server-rs/crates/api-server/src/llm_model_routing.rs @@ -1,3 +1,5 @@ pub(crate) const RPG_STORY_LLM_MODEL: &str = "doubao-seed-character-251128"; pub(crate) const CREATION_TEMPLATE_LLM_MODEL: &str = "deepseek-v3-2-251201"; pub(crate) const PUZZLE_LEVEL_NAME_VISION_LLM_MODEL: &str = "gpt-4o-mini"; +// 抠图背景色决策的视觉模型：只需看图选色，用低成本视觉模型即可。 +pub(crate) const EDITOR_SCREEN_BACKGROUND_VISION_LLM_MODEL: &str = "gpt-4o-mini"; diff --git a/server-rs/crates/api-server/src/main.rs b/server-rs/crates/api-server/src/main.rs index 06b145b50..142c6bff3 100644 --- a/server-rs/crates/api-server/src/main.rs +++ b/server-rs/crates/api-server/src/main.rs @@ -43,6 +43,7 @@ mod editor_generation_queue; mod editor_green_screen; mod editor_project; mod editor_screen_background_decision; +mod editor_screen_background_filter; mod edutainment_baby_drawing; mod edutainment_baby_object; mod error_middleware;