From f3aad7b828fee193f940a1d8b81d00ea7a89b54b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Fri, 17 Jul 2026 15:00:34 +0800 Subject: [PATCH 01/25] enforce more tool param check --- server-rs/Cargo.lock | 2 + .../crates/api-server/src/editor_agent/api.rs | 246 ++++++++++++--- .../src/editor_agent/display_args.rs | 122 +++++--- .../crates/api-server/src/editor_agent/mod.rs | 1 + .../api-server/src/editor_agent/pricing.rs | 111 ++++--- .../api-server/src/editor_agent/reconcile.rs | 46 ++- .../api-server/src/editor_agent/tool_args.rs | 184 +++++++++++ .../crates/platform-editor-agent/Cargo.toml | 2 + .../src/agent/tools/edit_image.rs | 35 ++- .../agent/tools/generate_background_music.rs | 45 ++- .../src/agent/tools/generate_character.rs | 66 +++- .../agent/tools/generate_icon_spritesheet.rs | 148 ++++++++- .../src/agent/tools/generate_image.rs | 67 +++- .../src/agent/tools/generate_sound_effect.rs | 98 +++++- .../src/agent/tools/generate_ui_design.rs | 137 +++++++- .../src/agent/tools/generate_video.rs | 296 ++++++++++++++++-- .../agent/tools/image_generation_options.rs | 167 ++++++++++ .../src/agent/tools/mod.rs | 167 ++++++++++ server-rs/crates/platform-image/src/lib.rs | 4 +- .../src/vector_engine/constants.rs | 1 + .../platform-image/src/vector_engine/mod.rs | 3 +- 21 files changed, 1698 insertions(+), 250 deletions(-) create mode 100644 server-rs/crates/api-server/src/editor_agent/tool_args.rs create mode 100644 server-rs/crates/platform-editor-agent/src/agent/tools/image_generation_options.rs diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index 0dc45c3b1..e2b6682ad 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -4056,6 +4056,8 @@ name = "platform-editor-agent" version = "0.1.0" dependencies = [ "hmac", + "platform-audio", + "platform-image", "platform-llm", "serde", "serde_json", diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index e26bd6b69..274706e87 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -42,7 +42,7 @@ use crate::editor_agent::utils::{ ensure_editor_project_access, normalize_editor_agent_attachments, now_rfc3339, read_messages_document, require_editor_agent_sidebar_enabled, write_messages_document, }; -use crate::editor_agent::{context, display_args, reconcile}; +use crate::editor_agent::{context, display_args, reconcile, tool_args}; use crate::editor_generation_config::EditorGenerationPricingConfig; use crate::editor_generation_queue::{ EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND, @@ -63,7 +63,9 @@ use platform_editor_agent::agent::tools::edit_image::{EditImageTool, EditImageTo use platform_editor_agent::agent::tools::generate_background_music::{ GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, }; -use platform_editor_agent::agent::tools::generate_character::GenerateCharacterTool; +use platform_editor_agent::agent::tools::generate_character::{ + GenerateCharacterTool, GenerateCharacterToolArgs, +}; use platform_editor_agent::agent::tools::generate_icon_spritesheet::{ GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, }; @@ -73,7 +75,9 @@ use platform_editor_agent::agent::tools::generate_image::{ use platform_editor_agent::agent::tools::generate_sound_effect::{ GenerateSoundEffectTool, GenerateSoundEffectToolArgs, }; -use platform_editor_agent::agent::tools::generate_ui_design::GenerateUiDesignTool; +use platform_editor_agent::agent::tools::generate_ui_design::{ + GenerateUiDesignTool, GenerateUiDesignToolArgs, +}; use platform_editor_agent::agent::tools::generate_video::{ GenerateVideoTool, GenerateVideoToolArgs, }; @@ -567,6 +571,34 @@ mod tests { ); } + #[test] + fn legacy_confirm_args_use_current_defaults_and_ignore_retired_fields() { + let music: GenerateBackgroundMusicToolArgs = parse_confirm_tool_args(&json!({ + "prompt": "温暖舒缓的钢琴背景音乐", + "make_instrumental": false + })) + .expect("旧背景音乐字段不应阻断确认"); + assert_eq!(music.model, GenerateBackgroundMusicTool::DEFAULT_MODEL); + + let ui: GenerateUiDesignToolArgs = parse_confirm_tool_args(&json!({ + "prompt": "生成游戏主界面" + })) + .expect("旧 UI 消息缺少 model 时应使用当前固定模型"); + assert_eq!(ui.model, platform_image::GPT_IMAGE_2_MODEL); + + let video: GenerateVideoToolArgs = parse_confirm_tool_args(&json!({ + "prompt": "镜头向前推进" + })) + .expect("旧视频消息缺少可选字段时应使用当前默认值"); + assert_eq!(video.sound.as_deref(), Some("on")); + + let sound: GenerateSoundEffectToolArgs = parse_confirm_tool_args(&json!({ + "prompt": "按钮点击声" + })) + .expect("旧音效消息缺少时长时应使用当前默认值"); + assert_eq!(sound.duration, Some(5)); + } + #[tokio::test] async fn prompt_timeout_applies_to_the_whole_agent_run() { let error = run_editor_agent_prompt_with_timeout( @@ -624,23 +656,31 @@ fn build_delta_messages( }); } PromptOutput::Tool(tco) => { - let display_args = display_args::build_tool_call_display_args( - tco.tool_call.name.as_str(), + let tool_name = tco.tool_call.name; + let normalized_args = tool_args::normalize_tool_args( + tool_name.as_str(), &tco.tool_call.args, + tool_context, + )?; + let display_args = display_args::build_tool_call_display_args( + tool_name.as_str(), + &normalized_args, document, tool_context, pricing, )?; + let text = + format!("[tool_call:{tool_name}] args: {normalized_args} output: 等待用户确认"); messages.push(EditorAgentMessage { id: absolute_idx, client_message_id: None, role: EditorAgentMessageRole::System, - text: tco.message, + text, attachments: Vec::new(), tool_call: Some(EditorAgentToolCall { - tool_name: tco.tool_call.name, + tool_name, status: EditorAgentToolCallStatus::NotCompleted, - args: tco.tool_call.args, + args: normalized_args, display_args, external_job_id: None, images: Vec::new(), @@ -933,7 +973,7 @@ pub async fn confirm_editor_agent_tool_call( }; let (job_kind, request_label, price_mud_points, payload) = match tool_name.as_str() { - GenerateImageTool::NAME | GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => { + GenerateImageTool::NAME => { let args: GenerateImageToolArgs = parse_confirm_tool_args(&tool_args)?; let tool = GenerateImageTool { context: context.clone(), @@ -941,22 +981,7 @@ pub async fn confirm_editor_agent_tool_call( tool.validate_args(&args).map_err(|error| { editor_agent_bad_request(format!("invalid tool call args: {error}")) })?; - let kind = match tool_name.as_str() { - GenerateCharacterTool::NAME => Some("character"), - GenerateUiDesignTool::NAME => Some("ui-design"), - _ => None, - }; - let price = match kind { - Some("character") => GenerateCharacterTool { - context: context.clone(), - } - .pricing(&pricing, &args), - Some("ui-design") => GenerateUiDesignTool { - context: context.clone(), - } - .pricing(&pricing, &args), - _ => tool.pricing(&pricing, &args), - }; + let price = tool.pricing(&pricing, &args); let title = args.prompt.clone(); let reference_image_srcs = args .reference_image_ids @@ -973,15 +998,93 @@ pub async fn confirm_editor_agent_tool_call( let payload = EditorImageGenerationRequest { prompt: args.prompt, size: None, - kind: kind.map(ToOwned::to_owned), - model: None, - screen_color: (kind == Some("character")).then(|| "auto".to_string()), - seg_model: (kind == Some("character")).then(|| "birefnet".to_string()), + kind: None, + model: Some(args.model), + screen_color: None, + seg_model: None, aspect_ratio: args.aspect_ratio, image_size: args.image_size, reference_image_srcs: Some(reference_image_srcs), project_id: Some(conversation.project_id.clone()), - asset_kind: Some(kind.unwrap_or("editor_agent_generated_image").to_string()), + asset_kind: Some("editor_agent_generated_image".to_string()), + generation_inputs: generation_inputs("用户指令", &title), + asset_folder_id: Some("project".to_string()), + asset_label: Some(title.clone()), + source_resource_id: None, + canvas_completion: Some(build_editor_agent_canvas_completion( + &project, &tool_name, &title, + )), + }; + ( + EDITOR_IMAGE_GENERATION_JOB_KIND, + "画布 Agent 生成图片", + price, + serde_json::to_value(payload), + ) + } + GenerateCharacterTool::NAME => { + let args: GenerateCharacterToolArgs = parse_confirm_tool_args(&tool_args)?; + let tool = GenerateCharacterTool { + context: context.clone(), + }; + tool.validate_args(&args).map_err(|error| { + editor_agent_bad_request(format!("invalid tool call args: {error}")) + })?; + let price = tool.pricing(&pricing, &args); + let title = args.prompt.clone(); + let reference_image_srcs = + resolve_editor_agent_image_ids(args.reference_image_ids.as_slice(), &context)?; + let payload = EditorImageGenerationRequest { + prompt: args.prompt, + size: None, + kind: Some("character".to_string()), + model: Some(args.model), + screen_color: Some("auto".to_string()), + seg_model: Some("birefnet".to_string()), + aspect_ratio: args.aspect_ratio, + image_size: args.image_size, + reference_image_srcs: Some(reference_image_srcs), + project_id: Some(conversation.project_id.clone()), + asset_kind: Some("character".to_string()), + generation_inputs: generation_inputs("用户指令", &title), + asset_folder_id: Some("project".to_string()), + asset_label: Some(title.clone()), + source_resource_id: None, + canvas_completion: Some(build_editor_agent_canvas_completion( + &project, &tool_name, &title, + )), + }; + ( + EDITOR_IMAGE_GENERATION_JOB_KIND, + "画布 Agent 生成图片", + price, + serde_json::to_value(payload), + ) + } + GenerateUiDesignTool::NAME => { + let args: GenerateUiDesignToolArgs = parse_confirm_tool_args(&tool_args)?; + let tool = GenerateUiDesignTool { + context: context.clone(), + }; + tool.validate_args(&args).map_err(|error| { + editor_agent_bad_request(format!("invalid tool call args: {error}")) + })?; + let price = tool.pricing(&pricing, &args); + let title = args.prompt.clone(); + let reference_image_srcs = + resolve_editor_agent_image_ids(args.reference_image_ids.as_slice(), &context)?; + let payload = EditorImageGenerationRequest { + prompt: args.prompt, + size: None, + kind: Some("ui-design".to_string()), + model: Some(args.model), + screen_color: None, + seg_model: None, + aspect_ratio: args.aspect_ratio, + image_size: args.image_size, + reference_image_srcs: Some(reference_image_srcs), + project_id: Some(conversation.project_id.clone()), + asset_kind: Some("ui-design".to_string()), generation_inputs: generation_inputs("用户指令", &title), asset_folder_id: Some("project".to_string()), asset_label: Some(title.clone()), @@ -999,8 +1102,13 @@ pub async fn confirm_editor_agent_tool_call( } EditImageTool::NAME => { let args: EditImageToolArgs = parse_confirm_tool_args(&tool_args)?; - if args.prompt.trim().is_empty() { - return Err(editor_agent_bad_request("prompt not provided")); + let tool = EditImageTool { + context: context.clone(), + }; + if let Some(error) = tool.validate_args(&args) { + return Err(editor_agent_bad_request(format!( + "invalid tool call args: {error}" + ))); } let source_image_src = context .image_data_key(&args.object_image_id) @@ -1019,15 +1127,12 @@ pub async fn confirm_editor_agent_tool_call( }) .collect::, _>>()?; let title = args.prompt.clone(); - let price = EditImageTool { - context: context.clone(), - } - .pricing(&pricing, &args); + let price = tool.pricing(&pricing, &args); let payload = EditorImageEditRequest { prompt: args.prompt, source_image_src, size: None, - model: None, + model: Some(args.model), aspect_ratio: None, image_size: None, reference_image_srcs: Some(reference_image_srcs), @@ -1056,6 +1161,9 @@ pub async fn confirm_editor_agent_tool_call( let tool = GenerateIconSpritesheetTool { context: context.clone(), }; + tool.validate_args(&args).map_err(|error| { + editor_agent_bad_request(format!("invalid tool call args: {error}")) + })?; let price = tool.pricing(&pricing, &args); let reference_image_src = context .image_data_key(&args.reference_image_id) @@ -1077,7 +1185,7 @@ pub async fn confirm_editor_agent_tool_call( reference_image_src, reference_image_srcs: Some(reference_image_srcs), icon_descriptions: args.icon_descriptions.clone(), - model: None, + model: Some(args.model), screen_color: Some("auto".to_string()), seg_model: Some("birefnet".to_string()), aspect_ratio: args.aspect_ratio, @@ -1103,6 +1211,12 @@ pub async fn confirm_editor_agent_tool_call( } GenerateVideoTool::NAME => { let args: GenerateVideoToolArgs = parse_confirm_tool_args(&tool_args)?; + let tool = GenerateVideoTool { + context: context.clone(), + }; + tool.validate_args(&args).map_err(|error| { + editor_agent_bad_request(format!("invalid tool call args: {error}")) + })?; let reference_image_srcs = args .reference_image_ids .iter() @@ -1116,18 +1230,23 @@ pub async fn confirm_editor_agent_tool_call( }) .collect::, _>>()?; let title = args.prompt.clone(); - let price = GenerateVideoTool { - context: context.clone(), - } - .pricing(&pricing, &args); + let price = tool.pricing(&pricing, &args); let payload = EditorVideoGenerateRequest { prompt: args.prompt, - model: args.model.unwrap_or_else(|| "seedance2.0-fast".to_string()), - aspect_ratio: args.aspect_ratio.unwrap_or_else(|| "16:9".to_string()), - duration_seconds: args.duration_seconds.unwrap_or(4), - resolution: args.resolution.unwrap_or_else(|| "720p".to_string()), + model: args.model, + aspect_ratio: args + .aspect_ratio + .unwrap_or_else(|| GenerateVideoTool::DEFAULT_VIDEO_ASPECT_RATIO.to_string()), + duration_seconds: args + .duration_seconds + .unwrap_or(GenerateVideoTool::DEFAULT_VIDEO_DURATION_SECONDS), + resolution: args + .resolution + .unwrap_or_else(|| GenerateVideoTool::DEFAULT_VIDEO_RESOLUTION.to_string()), mode: "std".to_string(), - sound: args.sound.unwrap_or_else(|| "off".to_string()), + sound: args + .sound + .unwrap_or_else(|| GenerateVideoTool::DEFAULT_VIDEO_SOUND.to_string()), web_search_enabled: false, reference_image_srcs, reference_video_srcs: Vec::new(), @@ -1153,12 +1272,19 @@ pub async fn confirm_editor_agent_tool_call( } GenerateSoundEffectTool::NAME => { let args: GenerateSoundEffectToolArgs = parse_confirm_tool_args(&tool_args)?; + GenerateSoundEffectTool + .validate_args(&args) + .map_err(|error| { + editor_agent_bad_request(format!("invalid tool call args: {error}")) + })?; let price = GenerateSoundEffectTool.pricing(&pricing, &args); let title = args.prompt.clone(); let payload = EditorSoundEffectGenerateRequest { prompt: args.prompt, - model: None, - duration: args.duration.unwrap_or(3), + model: Some(args.model), + duration: args + .duration + .unwrap_or(GenerateSoundEffectTool::DEFAULT_DURATION), project_id: Some(conversation.project_id.clone()), canvas_completion: Some(build_editor_agent_canvas_completion( &project, @@ -1178,11 +1304,16 @@ pub async fn confirm_editor_agent_tool_call( } GenerateBackgroundMusicTool::NAME => { let args: GenerateBackgroundMusicToolArgs = parse_confirm_tool_args(&tool_args)?; + GenerateBackgroundMusicTool + .validate_args(&args) + .map_err(|error| { + editor_agent_bad_request(format!("invalid tool call args: {error}")) + })?; let price = GenerateBackgroundMusicTool.pricing(&pricing, &args); let title = args.prompt.clone(); let payload = EditorBackgroundMusicGenerateRequest { gpt_description_prompt: args.prompt, - make_instrumental: args.make_instrumental, + make_instrumental: true, project_id: Some(conversation.project_id.clone()), canvas_completion: Some(build_editor_agent_canvas_completion( &project, @@ -1246,6 +1377,21 @@ fn parse_confirm_tool_args(value: &Value) -> Res .map_err(|error| editor_agent_bad_request(format!("invalid tool call args: {error}"))) } +fn resolve_editor_agent_image_ids( + image_ids: &[platform_editor_agent::agent::asset::ImageId], + context: &EditorToolContext, +) -> Result, AppError> { + image_ids + .iter() + .map(|image_id| { + context + .image_data_key(image_id) + .map(ToOwned::to_owned) + .ok_or_else(|| editor_agent_bad_request(format!("image {image_id} not found"))) + }) + .collect() +} + fn editor_agent_tool_job_identity( conversation_id: &str, message_id: usize, diff --git a/server-rs/crates/api-server/src/editor_agent/display_args.rs b/server-rs/crates/api-server/src/editor_agent/display_args.rs index df0ca205d..cbf2de4de 100644 --- a/server-rs/crates/api-server/src/editor_agent/display_args.rs +++ b/server-rs/crates/api-server/src/editor_agent/display_args.rs @@ -7,7 +7,9 @@ use platform_editor_agent::agent::tools::edit_image::{EditImageTool, EditImageTo use platform_editor_agent::agent::tools::generate_background_music::{ GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, }; -use platform_editor_agent::agent::tools::generate_character::GenerateCharacterTool; +use platform_editor_agent::agent::tools::generate_character::{ + GenerateCharacterTool, GenerateCharacterToolArgs, +}; use platform_editor_agent::agent::tools::generate_icon_spritesheet::{ GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, }; @@ -17,7 +19,9 @@ use platform_editor_agent::agent::tools::generate_image::{ use platform_editor_agent::agent::tools::generate_sound_effect::{ GenerateSoundEffectTool, GenerateSoundEffectToolArgs, }; -use platform_editor_agent::agent::tools::generate_ui_design::GenerateUiDesignTool; +use platform_editor_agent::agent::tools::generate_ui_design::{ + GenerateUiDesignTool, GenerateUiDesignToolArgs, +}; use platform_editor_agent::agent::tools::generate_video::{ GenerateVideoTool, GenerateVideoToolArgs, }; @@ -47,6 +51,7 @@ pub fn build_tool_call_display_args( } .pricing(pricing, &args); push_string_display_arg(&mut display_args, "prompt", "修改要求", args.prompt); + push_string_display_arg(&mut display_args, "model", "模型", args.model); push_image_display_arg( &mut display_args, document, @@ -63,42 +68,54 @@ pub fn build_tool_call_display_args( )?; price_mud_points } - GenerateImageTool::NAME | GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => { + GenerateImageTool::NAME => { let args: GenerateImageToolArgs = parse_display_tool_args(tool_name, args)?; - let price_mud_points = match tool_name { - GenerateImageTool::NAME => GenerateImageTool { - context: tool_context.clone(), - } - .pricing(pricing, &args), - GenerateCharacterTool::NAME => GenerateCharacterTool { - context: tool_context.clone(), - } - .pricing(pricing, &args), - GenerateUiDesignTool::NAME => GenerateUiDesignTool { - context: tool_context.clone(), - } - .pricing(pricing, &args), - _ => unreachable!("tool name was matched above"), - }; - push_string_display_arg(&mut display_args, "prompt", "提示词", args.prompt); - push_optional_string_display_arg( - &mut display_args, - "aspect_ratio", - "画面比例", - args.aspect_ratio, - ); - push_optional_string_display_arg( - &mut display_args, - "image_size", - "图片尺寸", - args.image_size, - ); - push_image_display_arg( + let price_mud_points = GenerateImageTool { + context: tool_context.clone(), + } + .pricing(pricing, &args); + push_image_generation_display_args( &mut display_args, document, - "reference_image_ids", - "参考图片", - args.reference_image_ids.as_slice(), + args.prompt, + args.model, + args.aspect_ratio, + args.image_size, + args.reference_image_ids, + )?; + price_mud_points + } + GenerateCharacterTool::NAME => { + let args: GenerateCharacterToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = GenerateCharacterTool { + context: tool_context.clone(), + } + .pricing(pricing, &args); + push_image_generation_display_args( + &mut display_args, + document, + args.prompt, + args.model, + args.aspect_ratio, + args.image_size, + args.reference_image_ids, + )?; + price_mud_points + } + GenerateUiDesignTool::NAME => { + let args: GenerateUiDesignToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = GenerateUiDesignTool { + context: tool_context.clone(), + } + .pricing(pricing, &args); + push_image_generation_display_args( + &mut display_args, + document, + args.prompt, + args.model, + args.aspect_ratio, + args.image_size, + args.reference_image_ids, )?; price_mud_points } @@ -114,10 +131,11 @@ pub fn build_tool_call_display_args( "图标描述", args.icon_descriptions.join("\n"), ); + push_string_display_arg(&mut display_args, "model", "模型", args.model); push_optional_string_display_arg( &mut display_args, "aspect_ratio", - "图集比例", + "画面比例", args.aspect_ratio, ); push_optional_string_display_arg( @@ -163,7 +181,7 @@ pub fn build_tool_call_display_args( duration_seconds.to_string(), ); } - push_optional_string_display_arg(&mut display_args, "model", "模型", args.model); + push_string_display_arg(&mut display_args, "model", "模型", args.model); push_optional_string_display_arg( &mut display_args, "resolution", @@ -184,6 +202,7 @@ pub fn build_tool_call_display_args( let args: GenerateSoundEffectToolArgs = parse_display_tool_args(tool_name, args)?; let price_mud_points = GenerateSoundEffectTool.pricing(pricing, &args); push_string_display_arg(&mut display_args, "prompt", "音效描述", args.prompt); + push_string_display_arg(&mut display_args, "model", "模型", args.model); if let Some(duration) = args.duration { push_string_display_arg( &mut display_args, @@ -198,12 +217,7 @@ pub fn build_tool_call_display_args( let args: GenerateBackgroundMusicToolArgs = parse_display_tool_args(tool_name, args)?; let price_mud_points = GenerateBackgroundMusicTool.pricing(pricing, &args); push_string_display_arg(&mut display_args, "prompt", "音乐描述", args.prompt); - push_string_display_arg( - &mut display_args, - "make_instrumental", - "纯音乐", - if args.make_instrumental { "是" } else { "否" }, - ); + push_string_display_arg(&mut display_args, "model", "模型", args.model); price_mud_points } _ => { @@ -229,6 +243,28 @@ where }) } +fn push_image_generation_display_args( + display_args: &mut EditorAgentToolCallDisplayArgs, + document: &EditorAgentConversationMessagesDocument, + prompt: String, + model: String, + aspect_ratio: Option, + image_size: Option, + reference_image_ids: Vec, +) -> Result<(), PromptError> { + push_string_display_arg(display_args, "prompt", "提示词", prompt); + push_string_display_arg(display_args, "model", "模型", model); + push_optional_string_display_arg(display_args, "aspect_ratio", "画面比例", aspect_ratio); + push_optional_string_display_arg(display_args, "image_size", "图片尺寸", image_size); + push_image_display_arg( + display_args, + document, + "reference_image_ids", + "参考图片", + reference_image_ids.as_slice(), + ) +} + fn push_string_display_arg( display_args: &mut EditorAgentToolCallDisplayArgs, name: &str, diff --git a/server-rs/crates/api-server/src/editor_agent/mod.rs b/server-rs/crates/api-server/src/editor_agent/mod.rs index eaa31b0fe..c73b28bb9 100644 --- a/server-rs/crates/api-server/src/editor_agent/mod.rs +++ b/server-rs/crates/api-server/src/editor_agent/mod.rs @@ -4,6 +4,7 @@ mod display_args; pub mod pricing; mod reconcile; mod resp_to_asset; +mod tool_args; mod utils; pub use api::{ diff --git a/server-rs/crates/api-server/src/editor_agent/pricing.rs b/server-rs/crates/api-server/src/editor_agent/pricing.rs index 42bd43e47..75438ebeb 100644 --- a/server-rs/crates/api-server/src/editor_agent/pricing.rs +++ b/server-rs/crates/api-server/src/editor_agent/pricing.rs @@ -9,7 +9,9 @@ use platform_editor_agent::agent::tools::edit_image::{EditImageTool, EditImageTo use platform_editor_agent::agent::tools::generate_background_music::{ GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, }; -use platform_editor_agent::agent::tools::generate_character::GenerateCharacterTool; +use platform_editor_agent::agent::tools::generate_character::{ + GenerateCharacterTool, GenerateCharacterToolArgs, +}; use platform_editor_agent::agent::tools::generate_icon_spritesheet::{ GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, }; @@ -19,7 +21,9 @@ use platform_editor_agent::agent::tools::generate_image::{ use platform_editor_agent::agent::tools::generate_sound_effect::{ GenerateSoundEffectTool, GenerateSoundEffectToolArgs, }; -use platform_editor_agent::agent::tools::generate_ui_design::GenerateUiDesignTool; +use platform_editor_agent::agent::tools::generate_ui_design::{ + GenerateUiDesignTool, GenerateUiDesignToolArgs, +}; use platform_editor_agent::agent::tools::generate_video::{ GenerateVideoTool, GenerateVideoToolArgs, }; @@ -39,6 +43,18 @@ fn context() -> EditorToolContext { fn image_args(image_size: Option<&str>) -> GenerateImageToolArgs { GenerateImageToolArgs { prompt: "生成图片".to_string(), + model: platform_image::NANOBANANA_2_MODEL.to_string(), + reference_image_ids: Vec::new(), + aspect_ratio: Some("1:1".to_string()), + image_size: image_size.map(ToOwned::to_owned), + } +} + +#[cfg(test)] +fn character_args(image_size: Option<&str>) -> GenerateCharacterToolArgs { + GenerateCharacterToolArgs { + prompt: "生成角色".to_string(), + model: platform_image::NANOBANANA_2_MODEL.to_string(), reference_image_ids: Vec::new(), aspect_ratio: Some("1:1".to_string()), image_size: image_size.map(ToOwned::to_owned), @@ -63,6 +79,7 @@ fn every_editor_agent_tool_exposes_argument_based_pricing() { }, reference_image_ids: Vec::new(), prompt: "改成蓝色".to_string(), + model: GPT_IMAGE_2_MODEL.to_string(), }, ), 3 @@ -72,20 +89,29 @@ fn every_editor_agent_tool_exposes_argument_based_pricing() { context: context.clone(), } .pricing(&pricing, &image_args(Some("2K"))), - 5 + 24 ); assert_eq!( GenerateCharacterTool { context: context.clone(), } - .pricing(&pricing, &image_args(None)), - 3 + .pricing(&pricing, &character_args(None)), + 12 ); assert_eq!( GenerateUiDesignTool { context: context.clone(), } - .pricing(&pricing, &image_args(Some("2K"))), + .pricing( + &pricing, + &GenerateUiDesignToolArgs { + prompt: "生成游戏主界面".to_string(), + model: GPT_IMAGE_2_MODEL.to_string(), + reference_image_ids: Vec::new(), + aspect_ratio: Some("1:1".to_string()), + image_size: Some("2K".to_string()), + } + ), 5 ); assert_eq!( @@ -98,13 +124,14 @@ fn every_editor_agent_tool_exposes_argument_based_pricing() { reference_image_id: ImageId { id: "image-1".to_string(), }, + model: platform_image::NANOBANANA_2_MODEL.to_string(), reference_image_ids: Vec::new(), icon_descriptions: vec!["背包".to_string(), "地图".to_string()], aspect_ratio: Some("1:1".to_string()), image_size: Some("2K".to_string()), }, ), - 5 + 24 ); assert_eq!( GenerateVideoTool { @@ -117,7 +144,7 @@ fn every_editor_agent_tool_exposes_argument_based_pricing() { reference_image_ids: Vec::new(), aspect_ratio: None, duration_seconds: Some(6), - model: Some("seedance2.0".to_string()), + model: "seedance2.0".to_string(), resolution: Some("720p".to_string()), sound: None, }, @@ -129,6 +156,7 @@ fn every_editor_agent_tool_exposes_argument_based_pricing() { &pricing, &GenerateSoundEffectToolArgs { prompt: "按钮点击声".to_string(), + model: platform_audio::VIDU_AUDIO_MODEL.to_string(), duration: None, }, ), @@ -139,7 +167,7 @@ fn every_editor_agent_tool_exposes_argument_based_pricing() { &pricing, &GenerateBackgroundMusicToolArgs { prompt: "轻松背景音乐".to_string(), - make_instrumental: true, + model: platform_audio::SUNO_DEFAULT_MODEL.to_string(), }, ), 12 @@ -168,8 +196,10 @@ fn pricing_uses_the_supplied_runtime_snapshot() { .expect("sound pricing should exist") .price = Some(19); + let mut image2_args = image_args(Some("2K")); + image2_args.model = GPT_IMAGE_2_MODEL.to_string(); assert_eq!( - GenerateImageTool { context: context() }.pricing(&pricing, &image_args(Some("2K"))), + GenerateImageTool { context: context() }.pricing(&pricing, &image2_args), 37 ); assert_eq!( @@ -180,7 +210,7 @@ fn pricing_uses_the_supplied_runtime_snapshot() { reference_image_ids: Vec::new(), aspect_ratio: None, duration_seconds: None, - model: None, + model: GenerateVideoTool::DEFAULT_VIDEO_MODEL.to_string(), resolution: None, sound: None, }, @@ -192,6 +222,7 @@ fn pricing_uses_the_supplied_runtime_snapshot() { &pricing, &GenerateSoundEffectToolArgs { prompt: "sound".to_string(), + model: platform_audio::VIDU_AUDIO_MODEL.to_string(), duration: None, }, ), @@ -205,12 +236,7 @@ impl EditorAgentPricedTool for GenerateVideoTool { pricing: &EditorGenerationPricingConfig, args: &GenerateVideoToolArgs, ) -> u32 { - let model = args - .model - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or(Self::DEFAULT_VIDEO_MODEL); + let model = args.model.as_str(); let resolution = args .resolution .as_deref() @@ -228,9 +254,14 @@ impl EditorAgentPricedTool for GenerateUiDesignTool { fn pricing( &self, pricing: &EditorGenerationPricingConfig, - args: &GenerateImageToolArgs, + args: &GenerateUiDesignToolArgs, ) -> u32 { - editor_agent_image_mud_points(pricing, Some("ui-design"), args.image_size.as_deref()) + editor_agent_image_mud_points( + pricing, + Some("ui-design"), + args.model.as_str(), + args.image_size.as_deref(), + ) } } @@ -238,9 +269,9 @@ impl EditorAgentPricedTool for GenerateSoundEffectTool { fn pricing( &self, pricing: &EditorGenerationPricingConfig, - _args: &GenerateSoundEffectToolArgs, + args: &GenerateSoundEffectToolArgs, ) -> u32 { - pricing.sound_effect_model_mud_points(None) + pricing.sound_effect_model_mud_points(Some(args.model.as_str())) } } @@ -250,7 +281,12 @@ impl EditorAgentPricedTool for GenerateImageTool { pricing: &EditorGenerationPricingConfig, args: &GenerateImageToolArgs, ) -> u32 { - editor_agent_image_mud_points(pricing, None, args.image_size.as_deref()) + editor_agent_image_mud_points( + pricing, + None, + args.model.as_str(), + args.image_size.as_deref(), + ) } } @@ -260,7 +296,12 @@ impl EditorAgentPricedTool for GenerateIconSpritesheetTool { pricing: &EditorGenerationPricingConfig, args: &GenerateIconSpritesheetToolArgs, ) -> u32 { - editor_agent_image_mud_points(pricing, Some("icon"), args.image_size.as_deref()) + editor_agent_image_mud_points( + pricing, + Some("icon"), + args.model.as_str(), + args.image_size.as_deref(), + ) } } @@ -268,9 +309,14 @@ impl EditorAgentPricedTool for GenerateCharacterTool { fn pricing( &self, pricing: &EditorGenerationPricingConfig, - args: &GenerateImageToolArgs, + args: &GenerateCharacterToolArgs, ) -> u32 { - editor_agent_image_mud_points(pricing, Some("character"), args.image_size.as_deref()) + editor_agent_image_mud_points( + pricing, + Some("character"), + args.model.as_str(), + args.image_size.as_deref(), + ) } } @@ -285,29 +331,24 @@ pub(crate) trait EditorAgentPricedTool: Tool { pub(crate) fn editor_agent_image_mud_points( pricing: &EditorGenerationPricingConfig, kind: Option<&str>, + model: &str, image_size: Option<&str>, ) -> u32 { - // 这些 Agent 工具当前向既有 BFF 传 model=None；BFF 会先归一为 gpt-image-2。 - // 尺寸同样只把精确的 2K 识别为 2K，其余值回落到 1K。 - let normalized_image_size = match image_size.map(str::trim) { - Some("2K") => "2K", - _ => "1K", - }; - pricing.image_generation_mud_points(kind, Some(GPT_IMAGE_2_MODEL), Some(normalized_image_size)) + pricing.image_generation_mud_points(kind, Some(model), image_size) } impl EditorAgentPricedTool for GenerateBackgroundMusicTool { fn pricing( &self, pricing: &EditorGenerationPricingConfig, - _args: &GenerateBackgroundMusicToolArgs, + args: &GenerateBackgroundMusicToolArgs, ) -> u32 { - pricing.background_music_model_mud_points(None) + pricing.background_music_model_mud_points(Some(args.model.as_str())) } } impl EditorAgentPricedTool for EditImageTool { fn pricing(&self, pricing: &EditorGenerationPricingConfig, _args: &EditImageToolArgs) -> u32 { - editor_agent_image_mud_points(pricing, Some("quick-edit"), Some("1K")) + editor_agent_image_mud_points(pricing, Some("quick-edit"), GPT_IMAGE_2_MODEL, Some("1K")) } } diff --git a/server-rs/crates/api-server/src/editor_agent/reconcile.rs b/server-rs/crates/api-server/src/editor_agent/reconcile.rs index 20d21a02f..4b48bd1c0 100644 --- a/server-rs/crates/api-server/src/editor_agent/reconcile.rs +++ b/server-rs/crates/api-server/src/editor_agent/reconcile.rs @@ -8,7 +8,9 @@ use platform_editor_agent::agent::tools::edit_image::{ use platform_editor_agent::agent::tools::generate_background_music::{ GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, }; -use platform_editor_agent::agent::tools::generate_character::GenerateCharacterTool; +use platform_editor_agent::agent::tools::generate_character::{ + GenerateCharacterTool, GenerateCharacterToolArgs, +}; use platform_editor_agent::agent::tools::generate_icon_spritesheet::{ EditorIconSpritesheetResult, GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, }; @@ -18,7 +20,9 @@ use platform_editor_agent::agent::tools::generate_image::{ use platform_editor_agent::agent::tools::generate_sound_effect::{ GenerateSoundEffectTool, GenerateSoundEffectToolArgs, }; -use platform_editor_agent::agent::tools::generate_ui_design::GenerateUiDesignTool; +use platform_editor_agent::agent::tools::generate_ui_design::{ + GenerateUiDesignTool, GenerateUiDesignToolArgs, +}; use platform_editor_agent::agent::tools::generate_video::{ GenerateVideoTool, GenerateVideoToolArgs, }; @@ -202,23 +206,31 @@ fn reconcile_completed_editor_agent_tool_call( let tool_name = tool_call.tool_name.clone(); match tool_name.as_str() { - GenerateImageTool::NAME | GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => { + GenerateImageTool::NAME => { let args: GenerateImageToolArgs = parse_reconciled_value(&tool_call.args)?; let result: EditorImageGenerationResult = parse_reconciled_value(&response)?; - message.text = match tool_name.as_str() { - GenerateCharacterTool::NAME => GenerateCharacterTool { - context: EditorToolContext::default(), - } - .format_execute_message(&args, &result), - GenerateUiDesignTool::NAME => GenerateUiDesignTool { - context: EditorToolContext::default(), - } - .format_execute_message(&args, &result), - _ => GenerateImageTool { - context: EditorToolContext::default(), - } - .format_execute_message(&args, &result), - }; + message.text = GenerateImageTool { + context: EditorToolContext::default(), + } + .format_execute_message(&args, &result); + tool_call.images = vec![resp_to_asset::editor_agent_generated_image(&result)]; + } + GenerateCharacterTool::NAME => { + let args: GenerateCharacterToolArgs = parse_reconciled_value(&tool_call.args)?; + let result: EditorImageGenerationResult = parse_reconciled_value(&response)?; + message.text = GenerateCharacterTool { + context: EditorToolContext::default(), + } + .format_execute_message(&args, &result); + tool_call.images = vec![resp_to_asset::editor_agent_generated_image(&result)]; + } + GenerateUiDesignTool::NAME => { + let args: GenerateUiDesignToolArgs = parse_reconciled_value(&tool_call.args)?; + let result: EditorImageGenerationResult = parse_reconciled_value(&response)?; + message.text = GenerateUiDesignTool { + context: EditorToolContext::default(), + } + .format_execute_message(&args, &result); tool_call.images = vec![resp_to_asset::editor_agent_generated_image(&result)]; } EditImageTool::NAME => { diff --git a/server-rs/crates/api-server/src/editor_agent/tool_args.rs b/server-rs/crates/api-server/src/editor_agent/tool_args.rs new file mode 100644 index 000000000..7a2784dc9 --- /dev/null +++ b/server-rs/crates/api-server/src/editor_agent/tool_args.rs @@ -0,0 +1,184 @@ +use platform_editor_agent::agent::tools::context::EditorToolContext; +use platform_editor_agent::agent::tools::edit_image::{EditImageTool, EditImageToolArgs}; +use platform_editor_agent::agent::tools::generate_background_music::{ + GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, +}; +use platform_editor_agent::agent::tools::generate_character::{ + GenerateCharacterTool, GenerateCharacterToolArgs, +}; +use platform_editor_agent::agent::tools::generate_icon_spritesheet::{ + GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, +}; +use platform_editor_agent::agent::tools::generate_image::{ + GenerateImageTool, GenerateImageToolArgs, +}; +use platform_editor_agent::agent::tools::generate_sound_effect::{ + GenerateSoundEffectTool, GenerateSoundEffectToolArgs, +}; +use platform_editor_agent::agent::tools::generate_ui_design::{ + GenerateUiDesignTool, GenerateUiDesignToolArgs, +}; +use platform_editor_agent::agent::tools::generate_video::{ + GenerateVideoTool, GenerateVideoToolArgs, +}; +use platform_editor_agent::framework::error::PromptError; +use platform_editor_agent::framework::tool::Tool; +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_json::Value; + +pub fn normalize_tool_args( + tool_name: &str, + args: &Value, + context: &EditorToolContext, +) -> Result { + match tool_name { + GenerateImageTool::NAME => { + let args: GenerateImageToolArgs = parse_args(tool_name, args)?; + let tool = GenerateImageTool { + context: context.clone(), + }; + tool.validate_args(&args) + .map_err(|error| invalid_args(tool_name, error))?; + serialize_args(tool_name, args) + } + GenerateCharacterTool::NAME => { + let args: GenerateCharacterToolArgs = parse_args(tool_name, args)?; + let tool = GenerateCharacterTool { + context: context.clone(), + }; + tool.validate_args(&args) + .map_err(|error| invalid_args(tool_name, error))?; + serialize_args(tool_name, args) + } + GenerateUiDesignTool::NAME => { + let args: GenerateUiDesignToolArgs = parse_args(tool_name, args)?; + let tool = GenerateUiDesignTool { + context: context.clone(), + }; + tool.validate_args(&args) + .map_err(|error| invalid_args(tool_name, error))?; + serialize_args(tool_name, args) + } + EditImageTool::NAME => { + let args: EditImageToolArgs = parse_args(tool_name, args)?; + let tool = EditImageTool { + context: context.clone(), + }; + if let Some(error) = tool.validate_args(&args) { + return Err(invalid_args(tool_name, error)); + } + serialize_args(tool_name, args) + } + GenerateIconSpritesheetTool::NAME => { + let args: GenerateIconSpritesheetToolArgs = parse_args(tool_name, args)?; + let tool = GenerateIconSpritesheetTool { + context: context.clone(), + }; + tool.validate_args(&args) + .map_err(|error| invalid_args(tool_name, error))?; + serialize_args(tool_name, args) + } + GenerateVideoTool::NAME => { + let args: GenerateVideoToolArgs = parse_args(tool_name, args)?; + let tool = GenerateVideoTool { + context: context.clone(), + }; + tool.validate_args(&args) + .map_err(|error| invalid_args(tool_name, error))?; + serialize_args(tool_name, args) + } + GenerateSoundEffectTool::NAME => { + let args: GenerateSoundEffectToolArgs = parse_args(tool_name, args)?; + GenerateSoundEffectTool + .validate_args(&args) + .map_err(|error| invalid_args(tool_name, error))?; + serialize_args(tool_name, args) + } + GenerateBackgroundMusicTool::NAME => { + let args: GenerateBackgroundMusicToolArgs = parse_args(tool_name, args)?; + GenerateBackgroundMusicTool + .validate_args(&args) + .map_err(|error| invalid_args(tool_name, error))?; + serialize_args(tool_name, args) + } + _ => Err(PromptError::ToolError(format!( + "unsupported editor agent tool: {tool_name}" + ))), + } +} + +fn parse_args(tool_name: &str, args: &Value) -> Result { + serde_json::from_value(args.clone()).map_err(|error| { + PromptError::ToolError(format!("invalid args for tool {tool_name}: {error}")) + }) +} + +fn serialize_args(tool_name: &str, args: T) -> Result { + serde_json::to_value(args).map_err(|error| { + PromptError::InternalError(format!( + "failed to normalize args for tool {tool_name}: {error}" + )) + }) +} + +fn invalid_args(tool_name: &str, error: impl std::fmt::Display) -> PromptError { + PromptError::ToolError(format!("invalid args for tool {tool_name}: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use platform_audio::SUNO_DEFAULT_MODEL; + use platform_image::GPT_IMAGE_2_MODEL; + use serde_json::json; + + #[test] + fn normalizes_defaults_and_drops_unknown_fields_before_persistence() { + let normalized = normalize_tool_args( + GenerateUiDesignTool::NAME, + &json!({ + "prompt": "生成游戏主界面", + "unknown": "drop-me" + }), + &EditorToolContext::default(), + ) + .expect("UI 参数应使用工具默认模型并完成规范化"); + + assert_eq!(normalized["model"], GPT_IMAGE_2_MODEL); + assert_eq!(normalized["aspect_ratio"], "1:1"); + assert_eq!(normalized["image_size"], "1K"); + assert!(normalized.get("unknown").is_none()); + } + + #[test] + fn drops_legacy_background_music_make_instrumental_field() { + let normalized = normalize_tool_args( + GenerateBackgroundMusicTool::NAME, + &json!({ + "prompt": "温暖舒缓的钢琴背景音乐", + "model": SUNO_DEFAULT_MODEL, + "make_instrumental": false + }), + &EditorToolContext::default(), + ) + .expect("旧背景音乐字段应被安全忽略"); + + assert!(normalized.get("make_instrumental").is_none()); + } + + #[test] + fn rejects_invalid_sound_duration_before_persistence() { + let error = normalize_tool_args( + GenerateSoundEffectTool::NAME, + &json!({ + "prompt": "按钮点击声", + "duration": 11 + }), + &EditorToolContext::default(), + ) + .expect_err("非法音效时长必须在待确认消息持久化前失败"); + + assert!(error.to_string().contains("11")); + } +} diff --git a/server-rs/crates/platform-editor-agent/Cargo.toml b/server-rs/crates/platform-editor-agent/Cargo.toml index be86ce3af..b60f96fec 100644 --- a/server-rs/crates/platform-editor-agent/Cargo.toml +++ b/server-rs/crates/platform-editor-agent/Cargo.toml @@ -11,6 +11,8 @@ serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } shared-contracts = { workspace = true, features = ["oss-contracts"] } +platform-image = { workspace = true } +platform-audio = { workspace = true } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs index d54b83b89..d4fcc6df5 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs @@ -1,16 +1,19 @@ use crate::agent::asset::ImageId; use crate::agent::tools::context::EditorToolContext; use crate::framework::tool::{Tool, ToolFailure, ToolFailureKind}; +use platform_image::GPT_IMAGE_2_MODEL; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::error::Error; use std::fmt::Display; + pub struct EditImageTool { pub context: EditorToolContext, } #[derive(Debug, Clone)] pub enum EditImageError { + InvalidModel(String), ObjectImageNotProvided, PromptNotProvided, AssetNotFound(ImageId), @@ -26,6 +29,12 @@ impl Display for EditImageError { EditImageError::AssetNotFound(image_id) => { write!(f, "asset {image_id} not found in context") } + EditImageError::InvalidModel(model) => { + write!( + f, + "{model} is not a valid model name, only {GPT_IMAGE_2_MODEL} is supported for now." + ) + } } } } @@ -38,8 +47,11 @@ pub struct EditImageToolArgs { #[serde(default)] pub reference_image_ids: Vec, pub prompt: String, - // #[serde(default)] - // pub tag: Option, + #[serde(default = "default_model_name")] + pub model: String, +} +fn default_model_name() -> String { + GPT_IMAGE_2_MODEL.to_string() } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -75,10 +87,12 @@ impl Tool for EditImageTool { "type": "string", "description": "编辑提示词，描述希望如何修改图片。例如「把背景换成红色」、「把人物改成坐着」。" }, - // "tag": { - // "type": "string", - // "description": "为新生成的图片添加标签，用于后续在上下文中引用。" - // } + "model": { + "type": "string", + "enum": [GPT_IMAGE_2_MODEL], + "default": GPT_IMAGE_2_MODEL, + "description": format!("图片编辑固定使用{GPT_IMAGE_2_MODEL}") + } }, "required": ["object_image_id", "prompt"], "additionalProperties": false @@ -108,9 +122,7 @@ impl Tool for EditImageTool { EditImageError::AssetNotFound(_) => { ToolFailure::new(ToolFailureKind::NotFound, error.to_string()) } - EditImageError::ObjectImageNotProvided | EditImageError::PromptNotProvided => { - ToolFailure::invalid_args(error.to_string()) - } + _ => ToolFailure::invalid_args(error.to_string()), } } } @@ -140,7 +152,10 @@ pub struct EditorImageEditResult { impl EditImageTool { /// Validate the semantic correctness of the arguments. - fn validate_args(&self, args: &EditImageToolArgs) -> Option { + pub fn validate_args(&self, args: &EditImageToolArgs) -> Option { + if args.model != GPT_IMAGE_2_MODEL { + return Some(EditImageError::InvalidModel(args.model.clone())); + } if args.prompt.trim().is_empty() { return Some(EditImageError::PromptNotProvided); } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs index 9c19fe2ef..8cdfb529a 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs @@ -1,4 +1,5 @@ use crate::framework::tool::{Tool, ToolFailure}; +use platform_audio::SUNO_DEFAULT_MODEL; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use shared_contracts::assets::EditorAudioGenerateResponse; @@ -9,11 +10,18 @@ pub struct GenerateBackgroundMusicTool; #[derive(Debug, Clone)] pub enum GenerateBackgroundMusicError { + InvalidModel(String), PromptNotProvided, } impl Display for GenerateBackgroundMusicError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "background music prompt not provided") + match self { + Self::InvalidModel(model) => write!( + f, + "{model} is not a valid background music model; only {SUNO_DEFAULT_MODEL} is supported" + ), + Self::PromptNotProvided => write!(f, "background music prompt not provided"), + } } } impl Error for GenerateBackgroundMusicError {} @@ -21,13 +29,13 @@ impl Error for GenerateBackgroundMusicError {} #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GenerateBackgroundMusicToolArgs { pub prompt: String, - #[serde(default = "default_instrumental")] - pub make_instrumental: bool, -} -fn default_instrumental() -> bool { - true + #[serde(default = "default_background_music_model")] + pub model: String, } +fn default_background_music_model() -> String { + SUNO_DEFAULT_MODEL.to_string() +} #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GenerateBackgroundMusicToolOutput { pub message: String, @@ -39,13 +47,13 @@ impl Tool for GenerateBackgroundMusicTool { type Args = GenerateBackgroundMusicToolArgs; type Output = GenerateBackgroundMusicToolOutput; fn description(&self) -> String { - "根据文字描述生成背景音乐。默认生成纯音乐，除非明确要求歌词或人声。".to_string() + "根据文字描述生成纯音乐背景音乐；当前不支持歌词或人声。".to_string() } fn parameters(&self) -> Value { json!({ "type": "object", "properties": { "prompt": { "type": "string", "description": "音乐风格、情绪、乐器和节奏描述。" }, - "make_instrumental": { "type": "boolean", "description": "是否生成纯音乐，默认 true。" } + "model": { "type": "string", "enum": [SUNO_DEFAULT_MODEL], "default": SUNO_DEFAULT_MODEL, "description": "背景音乐固定使用 Suno。" } }, "required": ["prompt"], "additionalProperties": false }) } @@ -54,9 +62,7 @@ impl Tool for GenerateBackgroundMusicTool { args: Self::Args, ) -> impl Future> + Send { async move { - if args.prompt.trim().is_empty() { - return Err(GenerateBackgroundMusicError::PromptNotProvided); - } + self.validate_args(&args)?; Ok(GenerateBackgroundMusicToolOutput { message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string(), }) @@ -72,6 +78,23 @@ impl Tool for GenerateBackgroundMusicTool { } impl GenerateBackgroundMusicTool { + pub const DEFAULT_MODEL: &'static str = SUNO_DEFAULT_MODEL; + + pub fn validate_args( + &self, + args: &GenerateBackgroundMusicToolArgs, + ) -> Result<(), GenerateBackgroundMusicError> { + if args.prompt.trim().is_empty() { + return Err(GenerateBackgroundMusicError::PromptNotProvided); + } + if args.model != SUNO_DEFAULT_MODEL { + return Err(GenerateBackgroundMusicError::InvalidModel( + args.model.clone(), + )); + } + Ok(()) + } + pub fn format_execute_message( &self, args: &GenerateBackgroundMusicToolArgs, diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs index 0b9227420..70f41692a 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs @@ -1,19 +1,39 @@ +use crate::agent::asset::ImageId; use crate::agent::tools::context::EditorToolContext; use crate::agent::tools::generate_image::{ - EditorImageGenerationResult, GenerateImageError, GenerateImageTool, GenerateImageToolArgs, - GenerateImageToolOutput, + EditorImageGenerationResult, GenerateImageError, GenerateImageToolOutput, }; +use crate::agent::tools::image_generation_options::{ + default_image_aspect_ratio, default_image_model, default_image_size, + image_aspect_ratio_parameter_schema, image_model_parameter_schema, image_size_parameter_schema, + validate_image_generation_options, +}; +use crate::framework::tool::ToolFailureKind; use crate::framework::tool::{Tool, ToolFailure}; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; pub struct GenerateCharacterTool { pub context: EditorToolContext, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateCharacterToolArgs { + pub prompt: String, + #[serde(default = "default_image_model")] + pub model: String, + #[serde(default)] + pub reference_image_ids: Vec, + #[serde(default = "default_image_aspect_ratio")] + pub aspect_ratio: Option, + #[serde(default = "default_image_size")] + pub image_size: Option, +} + impl Tool for GenerateCharacterTool { const NAME: &'static str = "generate-character"; type Error = GenerateImageError; - type Args = GenerateImageToolArgs; + type Args = GenerateCharacterToolArgs; type Output = GenerateImageToolOutput; fn description(&self) -> String { @@ -26,9 +46,10 @@ impl Tool for GenerateCharacterTool { "type": "object", "properties": { "prompt": { "type": "string", "description": "角色外貌、服装、姿势、画风和构图的完整描述。" }, + "model": image_model_parameter_schema(), "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选角色或画风参考图 ID。" }, - "aspect_ratio": { "type": "string", "description": "可选比例，例如 2:3 或 9:16。" }, - "image_size": { "type": "string", "description": "可选清晰度，例如 1K 或 2K。" } + "aspect_ratio": image_aspect_ratio_parameter_schema(), + "image_size": image_size_parameter_schema(), }, "required": ["prompt"], "additionalProperties": false @@ -40,10 +61,7 @@ impl Tool for GenerateCharacterTool { args: Self::Args, ) -> impl Future> + Send { async move { - GenerateImageTool { - context: self.context.clone(), - } - .validate_args(&args)?; + self.validate_args(&args)?; Ok(GenerateImageToolOutput { message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string() }) } } @@ -53,17 +71,39 @@ impl Tool for GenerateCharacterTool { } fn classify_error(&self, error: &Self::Error) -> ToolFailure { - GenerateImageTool { - context: self.context.clone(), + match error { + GenerateImageError::AssetNotFound(_) => { + ToolFailure::new(ToolFailureKind::NotFound, error.to_string()) + } + _ => ToolFailure::invalid_args(error.to_string()), } - .classify_error(error) } } impl GenerateCharacterTool { + pub fn validate_args( + &self, + args: &GenerateCharacterToolArgs, + ) -> Result<(), GenerateImageError> { + validate_image_generation_options( + args.model.as_str(), + args.aspect_ratio.as_deref(), + args.image_size.as_deref(), + )?; + if args.prompt.trim().is_empty() { + return Err(GenerateImageError::PromptNotProvided); + } + for image_id in &args.reference_image_ids { + if !self.context.contains_image(image_id) { + return Err(GenerateImageError::AssetNotFound(image_id.clone())); + } + } + Ok(()) + } + pub fn format_execute_message( &self, - args: &GenerateImageToolArgs, + args: &GenerateCharacterToolArgs, result: &EditorImageGenerationResult, ) -> String { let tool_name = Self::NAME; diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs index dfd2a008c..9e7cff650 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs @@ -1,6 +1,12 @@ use crate::agent::asset::ImageId; use crate::agent::tools::context::EditorToolContext; +use crate::agent::tools::image_generation_options::{ + ImageGenerationOptionsError, default_image_aspect_ratio, default_image_model, + default_image_size, image_aspect_ratio_parameter_schema, image_model_parameter_schema, + image_size_parameter_schema, validate_image_generation_options, +}; use crate::framework::tool::{Tool, ToolFailure, ToolFailureKind}; +use platform_image::{GPT_IMAGE_2_MODEL, NANOBANANA_2_MODEL}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::error::Error; @@ -12,16 +18,35 @@ pub struct GenerateIconSpritesheetTool { #[derive(Debug, Clone)] pub enum GenerateIconSpritesheetError { + InvalidModel(String), + InvalidAspectRatio(String), + InvalidImageSize { model: String, image_size: String }, ReferenceNotProvided, DescriptionsNotProvided, + TooManyDescriptions(usize), AssetNotFound(ImageId), } impl Display for GenerateIconSpritesheetError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Self::InvalidModel(model) => write!( + f, + "{model} is not a valid image model; supported models: {NANOBANANA_2_MODEL}, {GPT_IMAGE_2_MODEL}" + ), + Self::InvalidAspectRatio(aspect_ratio) => { + write!(f, "invalid aspect ratio: {aspect_ratio}") + } + Self::InvalidImageSize { model, image_size } => { + write!(f, "invalid image size {image_size} for model {model}") + } Self::ReferenceNotProvided => write!(f, "reference image not provided"), Self::DescriptionsNotProvided => write!(f, "icon descriptions not provided"), + Self::TooManyDescriptions(count) => write!( + f, + "icon description count must be between 1 and {}, got {count}", + GenerateIconSpritesheetTool::MAX_ICON_DESCRIPTIONS + ), Self::AssetNotFound(image_id) => write!(f, "asset {image_id} not found in context"), } } @@ -29,15 +54,31 @@ impl Display for GenerateIconSpritesheetError { impl Error for GenerateIconSpritesheetError {} +impl From for GenerateIconSpritesheetError { + fn from(error: ImageGenerationOptionsError) -> Self { + match error { + ImageGenerationOptionsError::InvalidModel(model) => Self::InvalidModel(model), + ImageGenerationOptionsError::InvalidAspectRatio(aspect_ratio) => { + Self::InvalidAspectRatio(aspect_ratio) + } + ImageGenerationOptionsError::InvalidImageSize { model, image_size } => { + Self::InvalidImageSize { model, image_size } + } + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GenerateIconSpritesheetToolArgs { pub reference_image_id: ImageId, + #[serde(default = "default_image_model")] + pub model: String, #[serde(default)] pub reference_image_ids: Vec, pub icon_descriptions: Vec, - #[serde(default)] + #[serde(default = "default_image_aspect_ratio")] pub aspect_ratio: Option, - #[serde(default)] + #[serde(default = "default_image_size")] pub image_size: Option, } @@ -84,10 +125,11 @@ impl Tool for GenerateIconSpritesheetTool { "type": "object", "properties": { "reference_image_id": { "type": "string", "description": "必填的图标规范或风格参考图 ID。" }, + "model": image_model_parameter_schema(), "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选的额外参考图 ID。" }, - "icon_descriptions": { "type": "array", "items": { "type": "string" }, "description": "要生成的多个图标描述。" }, - "aspect_ratio": { "type": "string", "description": "可选图集比例。" }, - "image_size": { "type": "string", "description": "可选清晰度，例如 1K 或 2K。" } + "icon_descriptions": { "type": "array", "items": { "type": "string" }, "minItems": 1, "maxItems": GenerateIconSpritesheetTool::MAX_ICON_DESCRIPTIONS, "description": "要生成的多个图标描述，数量必须在 1 到 100 个之间。" }, + "aspect_ratio": image_aspect_ratio_parameter_schema(), + "image_size": image_size_parameter_schema() }, "required": ["reference_image_id", "icon_descriptions"], "additionalProperties": false @@ -115,8 +157,12 @@ impl Tool for GenerateIconSpritesheetTool { GenerateIconSpritesheetError::AssetNotFound(_) => { ToolFailure::new(ToolFailureKind::NotFound, error.to_string()) } - GenerateIconSpritesheetError::ReferenceNotProvided - | GenerateIconSpritesheetError::DescriptionsNotProvided => { + GenerateIconSpritesheetError::InvalidModel(_) + | GenerateIconSpritesheetError::InvalidAspectRatio(_) + | GenerateIconSpritesheetError::InvalidImageSize { .. } + | GenerateIconSpritesheetError::ReferenceNotProvided + | GenerateIconSpritesheetError::DescriptionsNotProvided + | GenerateIconSpritesheetError::TooManyDescriptions(_) => { ToolFailure::invalid_args(error.to_string()) } } @@ -124,10 +170,17 @@ impl Tool for GenerateIconSpritesheetTool { } impl GenerateIconSpritesheetTool { - fn validate_args( + pub const MAX_ICON_DESCRIPTIONS: usize = 100; + + pub fn validate_args( &self, args: &GenerateIconSpritesheetToolArgs, ) -> Result<(), GenerateIconSpritesheetError> { + validate_image_generation_options( + args.model.as_str(), + args.aspect_ratio.as_deref(), + args.image_size.as_deref(), + )?; if args.reference_image_id.id.trim().is_empty() { return Err(GenerateIconSpritesheetError::ReferenceNotProvided); } @@ -136,13 +189,19 @@ impl GenerateIconSpritesheetTool { args.reference_image_id.clone(), )); } - if args + let description_count = args .icon_descriptions .iter() - .all(|description| description.trim().is_empty()) - { + .filter(|description| !description.trim().is_empty()) + .count(); + if description_count == 0 { return Err(GenerateIconSpritesheetError::DescriptionsNotProvided); } + if description_count > Self::MAX_ICON_DESCRIPTIONS { + return Err(GenerateIconSpritesheetError::TooManyDescriptions( + description_count, + )); + } for image_id in &args.reference_image_ids { if !self.context.contains_image(image_id) { return Err(GenerateIconSpritesheetError::AssetNotFound( @@ -174,3 +233,70 @@ impl GenerateIconSpritesheetTool { ) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::agent::asset::ImageMetadata; + use std::collections::HashMap; + + fn tool_and_args( + icon_descriptions: Vec, + ) -> (GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs) { + let reference_image_id = ImageId { + id: "reference-image".to_string(), + }; + let tool = GenerateIconSpritesheetTool { + context: EditorToolContext { + images: HashMap::from([( + reference_image_id.clone(), + ImageMetadata { + tag: "image".to_string(), + data_key: "asset://reference-image".to_string(), + }, + )]), + }, + }; + let args = GenerateIconSpritesheetToolArgs { + reference_image_id, + model: NANOBANANA_2_MODEL.to_string(), + reference_image_ids: Vec::new(), + icon_descriptions, + aspect_ratio: Some("1:1".to_string()), + image_size: Some("1K".to_string()), + }; + (tool, args) + } + + #[test] + fn validates_non_empty_icon_description_count() { + let too_many = (0..=GenerateIconSpritesheetTool::MAX_ICON_DESCRIPTIONS) + .map(|index| format!("图标{index}")) + .collect::>(); + let (tool, args) = tool_and_args(too_many); + assert!(matches!( + tool.validate_args(&args), + Err(GenerateIconSpritesheetError::TooManyDescriptions(101)) + )); + + let (tool, args) = tool_and_args(vec![" ".to_string()]); + assert!(matches!( + tool.validate_args(&args), + Err(GenerateIconSpritesheetError::DescriptionsNotProvided) + )); + } + + #[test] + fn schema_caps_icon_descriptions() { + let (tool, _) = tool_and_args(vec!["背包".to_string()]); + let schema = tool.parameters(); + assert_eq!( + schema["properties"]["icon_descriptions"]["minItems"], + json!(1) + ); + assert_eq!( + schema["properties"]["icon_descriptions"]["maxItems"], + json!(100) + ); + } +} diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs index e3c7a687e..09512a42c 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs @@ -1,6 +1,12 @@ use crate::agent::asset::ImageId; use crate::agent::tools::context::EditorToolContext; +use crate::agent::tools::image_generation_options::{ + ImageGenerationOptionsError, default_image_aspect_ratio, default_image_model, + default_image_size, image_aspect_ratio_parameter_schema, image_model_parameter_schema, + image_size_parameter_schema, validate_image_generation_options, +}; use crate::framework::tool::{Tool, ToolFailure, ToolFailureKind}; +use platform_image::{GPT_IMAGE_2_MODEL, NANOBANANA_2_MODEL}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::error::Error; @@ -12,6 +18,10 @@ pub struct GenerateImageTool { #[derive(Debug, Clone)] pub enum GenerateImageError { + InvalidModel(String), + UnsupportedUiDesignModel(String), + InvalidAspectRatio(String), + InvalidImageSize { model: String, image_size: String }, PromptNotProvided, AssetNotFound(ImageId), } @@ -19,6 +29,20 @@ pub enum GenerateImageError { impl Display for GenerateImageError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Self::InvalidModel(model) => write!( + f, + "{model} is not a valid image model; supported models: {NANOBANANA_2_MODEL}, {GPT_IMAGE_2_MODEL}" + ), + Self::UnsupportedUiDesignModel(model) => write!( + f, + "{model} is not supported for UI design generation; required model: {GPT_IMAGE_2_MODEL}" + ), + Self::InvalidAspectRatio(aspect_ratio) => { + write!(f, "invalid aspect ratio: {aspect_ratio}") + } + Self::InvalidImageSize { model, image_size } => { + write!(f, "invalid image size {image_size} for model {model}") + } Self::PromptNotProvided => write!(f, "prompt not provided"), Self::AssetNotFound(image_id) => write!(f, "asset {image_id} not found in context"), } @@ -27,15 +51,30 @@ impl Display for GenerateImageError { impl Error for GenerateImageError {} +impl From for GenerateImageError { + fn from(error: ImageGenerationOptionsError) -> Self { + match error { + ImageGenerationOptionsError::InvalidModel(model) => Self::InvalidModel(model), + ImageGenerationOptionsError::InvalidAspectRatio(aspect_ratio) => { + Self::InvalidAspectRatio(aspect_ratio) + } + ImageGenerationOptionsError::InvalidImageSize { model, image_size } => { + Self::InvalidImageSize { model, image_size } + } + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GenerateImageToolArgs { pub prompt: String, + #[serde(default = "default_image_model")] + pub model: String, #[serde(default)] pub reference_image_ids: Vec, - #[serde(default)] - // TODO restrict to a set of possible values + #[serde(default = "default_image_aspect_ratio")] pub aspect_ratio: Option, - #[serde(default)] + #[serde(default = "default_image_size")] pub image_size: Option, } @@ -63,19 +102,14 @@ impl Tool for GenerateImageTool { "type": "string", "description": "完整的生图提示词，包含主体、场景、风格、构图和背景。" }, + "model": image_model_parameter_schema(), "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选参考图 ID 列表，用于提供画风或元素参考。" }, - "aspect_ratio": { - "type": "string", - "description": "可选画面比例，例如 1:1、16:9、9:16。" - }, - "image_size": { - "type": "string", - "description": "可选图片清晰度，例如 1K、2K、4K。" - } + "aspect_ratio": image_aspect_ratio_parameter_schema(), + "image_size": image_size_parameter_schema() }, "required": ["prompt"], "additionalProperties": false @@ -103,7 +137,11 @@ impl Tool for GenerateImageTool { GenerateImageError::AssetNotFound(_) => { ToolFailure::new(ToolFailureKind::NotFound, error.to_string()) } - GenerateImageError::PromptNotProvided => ToolFailure::invalid_args(error.to_string()), + GenerateImageError::InvalidModel(_) + | GenerateImageError::UnsupportedUiDesignModel(_) + | GenerateImageError::InvalidAspectRatio(_) + | GenerateImageError::InvalidImageSize { .. } + | GenerateImageError::PromptNotProvided => ToolFailure::invalid_args(error.to_string()), } } } @@ -129,6 +167,11 @@ pub struct EditorImageGenerationResult { impl GenerateImageTool { pub fn validate_args(&self, args: &GenerateImageToolArgs) -> Result<(), GenerateImageError> { + validate_image_generation_options( + args.model.as_str(), + args.aspect_ratio.as_deref(), + args.image_size.as_deref(), + )?; if args.prompt.trim().is_empty() { return Err(GenerateImageError::PromptNotProvided); } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs index 59e5c77d9..dba207011 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs @@ -1,4 +1,5 @@ use crate::framework::tool::{Tool, ToolFailure}; +use platform_audio::VIDU_AUDIO_MODEL; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use shared_contracts::assets::EditorAudioGenerateResponse; @@ -9,12 +10,29 @@ pub struct GenerateSoundEffectTool; #[derive(Debug, Clone)] pub enum GenerateSoundEffectError { + InvalidModel(String), + InvalidDuration(u8), PromptNotProvided, } impl Display for GenerateSoundEffectError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "sound effect prompt not provided") + match self { + Self::InvalidModel(model) => write!( + f, + "{model} is not a valid sound effect model; only {VIDU_AUDIO_MODEL} is supported" + ), + Self::InvalidDuration(duration) => write!( + f, + "{duration} is not a valid sound effect duration; supported durations: {}", + GenerateSoundEffectTool::SUPPORTED_DURATIONS + .iter() + .map(u8::to_string) + .collect::>() + .join(", ") + ), + Self::PromptNotProvided => write!(f, "sound effect prompt not provided"), + } } } @@ -23,10 +41,20 @@ impl Error for GenerateSoundEffectError {} #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GenerateSoundEffectToolArgs { pub prompt: String, - #[serde(default)] + #[serde(default = "default_sound_effect_model")] + pub model: String, + #[serde(default = "default_sound_effect_duration")] pub duration: Option, } +fn default_sound_effect_model() -> String { + VIDU_AUDIO_MODEL.to_string() +} + +fn default_sound_effect_duration() -> Option { + Some(GenerateSoundEffectTool::DEFAULT_DURATION) +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GenerateSoundEffectToolOutput { pub message: String, @@ -47,8 +75,8 @@ impl Tool for GenerateSoundEffectTool { "type": "object", "properties": { "prompt": { "type": "string", "description": "音效内容、材质、节奏和情绪描述。" }, - "duration": { "type": "integer", "description": "可选时长（秒）。" }, - // "model": { "type": "string", "description": "可选音效模型。" } + "model": { "type": "string", "enum": [VIDU_AUDIO_MODEL], "default": VIDU_AUDIO_MODEL, "description": "音效模型。" }, + "duration": { "type": "integer", "enum": GenerateSoundEffectTool::SUPPORTED_DURATIONS, "default": GenerateSoundEffectTool::DEFAULT_DURATION, "description": "音效时长（秒）。" }, }, "required": ["prompt"], "additionalProperties": false @@ -60,9 +88,7 @@ impl Tool for GenerateSoundEffectTool { args: Self::Args, ) -> impl Future> + Send { async move { - if args.prompt.trim().is_empty() { - return Err(GenerateSoundEffectError::PromptNotProvided); - } + self.validate_args(&args)?; Ok(GenerateSoundEffectToolOutput { message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string(), }) @@ -79,7 +105,27 @@ impl Tool for GenerateSoundEffectTool { } impl GenerateSoundEffectTool { - // pub const DEFAULT_MODEL: &'static str = "audio1.0"; + pub const DEFAULT_MODEL: &'static str = VIDU_AUDIO_MODEL; + pub const DEFAULT_DURATION: u8 = 5; + pub const SUPPORTED_DURATIONS: &'static [u8] = &[2, 3, 4, 5, 6, 7, 8, 9, 10]; + + pub fn validate_args( + &self, + args: &GenerateSoundEffectToolArgs, + ) -> Result<(), GenerateSoundEffectError> { + if args.prompt.trim().is_empty() { + return Err(GenerateSoundEffectError::PromptNotProvided); + } + if args.model != VIDU_AUDIO_MODEL { + return Err(GenerateSoundEffectError::InvalidModel(args.model.clone())); + } + if let Some(duration) = args.duration + && !Self::SUPPORTED_DURATIONS.contains(&duration) + { + return Err(GenerateSoundEffectError::InvalidDuration(duration)); + } + Ok(()) + } pub fn format_execute_message( &self, @@ -94,3 +140,39 @@ impl GenerateSoundEffectTool { ) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn args(duration: Option) -> GenerateSoundEffectToolArgs { + GenerateSoundEffectToolArgs { + prompt: "按钮点击声".to_string(), + model: GenerateSoundEffectTool::DEFAULT_MODEL.to_string(), + duration, + } + } + + #[test] + fn validates_schema_durations_and_default() { + assert_eq!(GenerateSoundEffectTool::DEFAULT_DURATION, 5); + assert!(GenerateSoundEffectTool.validate_args(&args(None)).is_ok()); + for duration in GenerateSoundEffectTool::SUPPORTED_DURATIONS { + assert!( + GenerateSoundEffectTool + .validate_args(&args(Some(*duration))) + .is_ok() + ); + } + } + + #[tokio::test] + async fn planning_rejects_durations_outside_schema() { + for duration in [1, 11, u8::MAX] { + assert!(matches!( + GenerateSoundEffectTool.call(args(Some(duration))).await, + Err(GenerateSoundEffectError::InvalidDuration(value)) if value == duration + )); + } + } +} diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs index 49660a6de..d67f9e3eb 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs @@ -1,19 +1,43 @@ +use crate::agent::asset::ImageId; use crate::agent::tools::context::EditorToolContext; use crate::agent::tools::generate_image::{ - EditorImageGenerationResult, GenerateImageError, GenerateImageTool, GenerateImageToolArgs, - GenerateImageToolOutput, + EditorImageGenerationResult, GenerateImageError, GenerateImageToolOutput, }; +use crate::agent::tools::image_generation_options::{ + default_image_aspect_ratio, default_image_size, image_aspect_ratio_parameter_schema, + image_size_parameter_schema, validate_image_generation_options, +}; +use crate::framework::tool::ToolFailureKind; use crate::framework::tool::{Tool, ToolFailure}; +use platform_image::GPT_IMAGE_2_MODEL; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; pub struct GenerateUiDesignTool { pub context: EditorToolContext, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateUiDesignToolArgs { + pub prompt: String, + #[serde(default = "default_ui_design_model")] + pub model: String, + #[serde(default)] + pub reference_image_ids: Vec, + #[serde(default = "default_image_aspect_ratio")] + pub aspect_ratio: Option, + #[serde(default = "default_image_size")] + pub image_size: Option, +} + +fn default_ui_design_model() -> String { + GPT_IMAGE_2_MODEL.to_string() +} + impl Tool for GenerateUiDesignTool { const NAME: &'static str = "generate-ui-design"; type Error = GenerateImageError; - type Args = GenerateImageToolArgs; + type Args = GenerateUiDesignToolArgs; type Output = GenerateImageToolOutput; fn description(&self) -> String { @@ -25,11 +49,17 @@ impl Tool for GenerateUiDesignTool { "type": "object", "properties": { "prompt": { "type": "string", "description": "完整 UI 画面、信息层级、视觉风格和构图描述。" }, + "model": { + "type": "string", + "enum": [GPT_IMAGE_2_MODEL], + "default": GPT_IMAGE_2_MODEL, + "description": "UI 设计图固定使用 gpt-image-2。" + }, "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选 UI 风格或布局参考图 ID。" }, - "aspect_ratio": { "type": "string", "description": "可选画面比例。" }, - "image_size": { "type": "string", "description": "可选清晰度，例如 1K 或 2K。" } + "aspect_ratio": image_aspect_ratio_parameter_schema(), + "image_size": image_size_parameter_schema() }, - "required": ["prompt"], + "required": ["prompt", "model"], "additionalProperties": false }) } @@ -39,10 +69,7 @@ impl Tool for GenerateUiDesignTool { args: Self::Args, ) -> impl Future> + Send { async move { - GenerateImageTool { - context: self.context.clone(), - } - .validate_args(&args)?; + self.validate_args(&args)?; Ok(GenerateImageToolOutput { message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string() }) } } @@ -52,17 +79,41 @@ impl Tool for GenerateUiDesignTool { } fn classify_error(&self, error: &Self::Error) -> ToolFailure { - GenerateImageTool { - context: self.context.clone(), + match error { + GenerateImageError::AssetNotFound(_) => { + ToolFailure::new(ToolFailureKind::NotFound, error.to_string()) + } + _ => ToolFailure::invalid_args(error.to_string()), } - .classify_error(error) } } impl GenerateUiDesignTool { + pub fn validate_args(&self, args: &GenerateUiDesignToolArgs) -> Result<(), GenerateImageError> { + if args.model != GPT_IMAGE_2_MODEL { + return Err(GenerateImageError::UnsupportedUiDesignModel( + args.model.clone(), + )); + } + validate_image_generation_options( + args.model.as_str(), + args.aspect_ratio.as_deref(), + args.image_size.as_deref(), + )?; + if args.prompt.trim().is_empty() { + return Err(GenerateImageError::PromptNotProvided); + } + for image_id in &args.reference_image_ids { + if !self.context.contains_image(image_id) { + return Err(GenerateImageError::AssetNotFound(image_id.clone())); + } + } + Ok(()) + } + pub fn format_execute_message( &self, - args: &GenerateImageToolArgs, + args: &GenerateUiDesignToolArgs, result: &EditorImageGenerationResult, ) -> String { let tool_name = Self::NAME; @@ -80,3 +131,61 @@ impl GenerateUiDesignTool { ) } } + +#[cfg(test)] +mod tests { + use super::*; + use platform_image::NANOBANANA_2_MODEL; + + fn tool() -> GenerateUiDesignTool { + GenerateUiDesignTool { + context: EditorToolContext::default(), + } + } + + fn args(model: &str) -> GenerateUiDesignToolArgs { + GenerateUiDesignToolArgs { + prompt: "生成游戏主界面".to_string(), + model: model.to_string(), + reference_image_ids: Vec::new(), + aspect_ratio: Some("16:9".to_string()), + image_size: Some("1K".to_string()), + } + } + + #[test] + fn schema_and_validation_lock_ui_design_to_gpt_image_2() { + let tool = tool(); + let parameters = tool.parameters(); + + assert_eq!( + parameters["properties"]["model"]["enum"], + json!([GPT_IMAGE_2_MODEL]) + ); + assert_eq!( + parameters["properties"]["model"]["default"], + GPT_IMAGE_2_MODEL + ); + assert!( + parameters["required"] + .as_array() + .is_some_and(|required| required.contains(&json!("model"))) + ); + assert!(tool.validate_args(&args(GPT_IMAGE_2_MODEL)).is_ok()); + assert!(matches!( + tool.validate_args(&args(NANOBANANA_2_MODEL)), + Err(GenerateImageError::UnsupportedUiDesignModel(model)) if model == NANOBANANA_2_MODEL + )); + } + + #[test] + fn legacy_args_without_model_default_to_gpt_image_2() { + let args: GenerateUiDesignToolArgs = serde_json::from_value(json!({ + "prompt": "生成游戏主界面" + })) + .expect("旧版 UI 设计参数应能反序列化"); + + assert_eq!(args.model, GPT_IMAGE_2_MODEL); + assert!(tool().validate_args(&args).is_ok()); + } +} diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs index b03ad5c4b..d802aec42 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs @@ -13,12 +13,46 @@ pub struct GenerateVideoTool { #[derive(Debug, Clone)] pub enum GenerateVideoError { + InvalidModel(String), + InvalidAspectRatio(String), + InvalidDurationSeconds(u32), + InvalidResolution(String), + InvalidSound(String), + UnsupportedModelResolution { model: String, resolution: String }, + ReferencesUnsupportedForModel(String), + TooManyReferenceImages(usize), PromptNotProvided, AssetNotFound(ImageId), } impl Display for GenerateVideoError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Self::InvalidModel(model) => write!(f, "{model} is not a supported video model"), + Self::InvalidAspectRatio(aspect_ratio) => { + write!(f, "{aspect_ratio} is not a supported video aspect ratio") + } + Self::InvalidDurationSeconds(duration_seconds) => write!( + f, + "{duration_seconds} is not a supported video duration in seconds" + ), + Self::InvalidResolution(resolution) => { + write!(f, "{resolution} is not a supported video resolution") + } + Self::InvalidSound(sound) => { + write!(f, "{sound} is not a supported video sound option") + } + Self::UnsupportedModelResolution { model, resolution } => { + write!(f, "{model} does not support {resolution} resolution") + } + Self::ReferencesUnsupportedForModel(model) => write!( + f, + "reference images are not supported by video model {model}" + ), + Self::TooManyReferenceImages(count) => write!( + f, + "video generation accepts at most {} reference images, got {count}", + GenerateVideoTool::MAX_REFERENCE_IMAGES + ), Self::PromptNotProvided => write!(f, "video prompt not provided"), Self::AssetNotFound(image_id) => write!(f, "asset {image_id} not found in context"), } @@ -31,17 +65,37 @@ pub struct GenerateVideoToolArgs { pub prompt: String, #[serde(default)] pub reference_image_ids: Vec, - #[serde(default)] + #[serde(default = "default_video_aspect_ratio")] pub aspect_ratio: Option, - #[serde(default)] + #[serde(default = "default_video_duration_seconds")] pub duration_seconds: Option, - #[serde(default)] - pub model: Option, - #[serde(default)] + #[serde(default = "default_video_model")] + pub model: String, + #[serde(default = "default_video_resolution")] pub resolution: Option, - #[serde(default)] + #[serde(default = "default_video_sound")] pub sound: Option, } + +fn default_video_model() -> String { + GenerateVideoTool::DEFAULT_VIDEO_MODEL.to_string() +} + +fn default_video_aspect_ratio() -> Option { + Some(GenerateVideoTool::DEFAULT_VIDEO_ASPECT_RATIO.to_string()) +} + +fn default_video_duration_seconds() -> Option { + Some(GenerateVideoTool::DEFAULT_VIDEO_DURATION_SECONDS) +} + +fn default_video_resolution() -> Option { + Some(GenerateVideoTool::DEFAULT_VIDEO_RESOLUTION.to_string()) +} + +fn default_video_sound() -> Option { + Some(GenerateVideoTool::DEFAULT_VIDEO_SOUND.to_string()) +} #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GenerateVideoToolOutput { pub message: String, @@ -59,13 +113,35 @@ impl Tool for GenerateVideoTool { json!({ "type": "object", "properties": { "prompt": { "type": "string", "description": "镜头、主体动作、场景和风格描述。" }, - "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选图片参考图 ID。" }, - "aspect_ratio": { "type": "string", "description": "可选比例，默认 16:9。" }, - "duration_seconds": { "type": "integer", "description": "可选时长，默认 4 秒。" }, - "model": { "type": "string", "description": "可选视频模型。" }, - "resolution": { "type": "string", "description": "可选清晰度，默认 720p。" }, - "sound": { "type": "string", "description": "是否生成声音，默认 off。" } - }, "required": ["prompt"], "additionalProperties": false + "reference_image_ids": { "type": "array", "items": { "type": "string" }, "maxItems": GenerateVideoTool::MAX_REFERENCE_IMAGES, "description": "可选图片参考图 ID，最多 9 张；仅 Seedance 2.0 系列模型支持参考图。" }, + "aspect_ratio": { "type": "string", "enum": GenerateVideoTool::SUPPORTED_ASPECT_RATIOS, "default": GenerateVideoTool::DEFAULT_VIDEO_ASPECT_RATIO, "description": "视频比例。" }, + "duration_seconds": { "type": "integer", "enum": GenerateVideoTool::SUPPORTED_DURATION_SECONDS, "default": GenerateVideoTool::DEFAULT_VIDEO_DURATION_SECONDS, "description": "视频时长（秒）。" }, + "model": { "type": "string", "enum": GenerateVideoTool::SUPPORTED_VIDEO_MODELS, "default": GenerateVideoTool::DEFAULT_VIDEO_MODEL, "description": "视频模型。" }, + "resolution": { "type": "string", "enum": GenerateVideoTool::SUPPORTED_RESOLUTIONS, "default": GenerateVideoTool::DEFAULT_VIDEO_RESOLUTION, "description": "视频分辨率；seedance2.0-fast 仅支持 480p、720p。" }, + "sound": { "type": "string", "enum": GenerateVideoTool::SUPPORTED_SOUND_OPTIONS, "default": GenerateVideoTool::DEFAULT_VIDEO_SOUND, "description": "是否生成声音。" } + }, + "required": ["prompt"], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { "model": { "const": GenerateVideoTool::SEEDANCE_2_FAST_MODEL } }, + "required": ["model"] + }, + "then": { + "properties": { "resolution": { "enum": GenerateVideoTool::SEEDANCE_2_FAST_RESOLUTIONS } } + } + }, + { + "if": { + "properties": { "reference_image_ids": { "minItems": 1 } }, + "required": ["reference_image_ids"] + }, + "then": { + "properties": { "model": { "enum": GenerateVideoTool::REFERENCE_IMAGE_MODELS } } + } + } + ] }) } fn call( @@ -73,14 +149,7 @@ impl Tool for GenerateVideoTool { args: Self::Args, ) -> impl Future> + Send { async move { - if args.prompt.trim().is_empty() { - return Err(GenerateVideoError::PromptNotProvided); - } - for id in &args.reference_image_ids { - if !self.context.contains_image(id) { - return Err(GenerateVideoError::AssetNotFound(id.clone())); - } - } + self.validate_args(&args)?; Ok(GenerateVideoToolOutput { message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string(), }) @@ -95,15 +164,98 @@ impl Tool for GenerateVideoTool { GenerateVideoError::AssetNotFound(_) => { ToolFailure::new(ToolFailureKind::NotFound, error.to_string()) } - GenerateVideoError::PromptNotProvided => ToolFailure::invalid_args(error.to_string()), + _ => ToolFailure::invalid_args(error.to_string()), } } } impl GenerateVideoTool { - pub const DEFAULT_VIDEO_MODEL: &'static str = "seedance2.0-fast"; + pub const SEEDANCE_2_FAST_MODEL: &'static str = "seedance2.0-fast"; + pub const SEEDANCE_2_MODEL: &'static str = "seedance2.0"; + pub const DEFAULT_VIDEO_MODEL: &'static str = Self::SEEDANCE_2_FAST_MODEL; + pub const SUPPORTED_VIDEO_MODELS: &'static [&'static str] = &[ + Self::SEEDANCE_2_FAST_MODEL, + Self::SEEDANCE_2_MODEL, + "kling3.0", + "kling3.0-omni", + ]; + pub const REFERENCE_IMAGE_MODELS: &'static [&'static str] = + &[Self::SEEDANCE_2_FAST_MODEL, Self::SEEDANCE_2_MODEL]; + pub const SUPPORTED_ASPECT_RATIOS: &'static [&'static str] = + &["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"]; + pub const SUPPORTED_DURATION_SECONDS: &'static [u32] = + &[4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + pub const SUPPORTED_RESOLUTIONS: &'static [&'static str] = &["480p", "720p", "1080p"]; + pub const SEEDANCE_2_FAST_RESOLUTIONS: &'static [&'static str] = &["480p", "720p"]; + pub const SUPPORTED_SOUND_OPTIONS: &'static [&'static str] = &["on", "off"]; + pub const MAX_REFERENCE_IMAGES: usize = 9; + pub const DEFAULT_VIDEO_ASPECT_RATIO: &'static str = "16:9"; pub const DEFAULT_VIDEO_RESOLUTION: &'static str = "720p"; pub const DEFAULT_VIDEO_DURATION_SECONDS: u32 = 4; + pub const DEFAULT_VIDEO_SOUND: &'static str = "on"; + + pub fn validate_args(&self, args: &GenerateVideoToolArgs) -> Result<(), GenerateVideoError> { + if args.prompt.trim().is_empty() { + return Err(GenerateVideoError::PromptNotProvided); + } + if !Self::SUPPORTED_VIDEO_MODELS.contains(&args.model.as_str()) { + return Err(GenerateVideoError::InvalidModel(args.model.clone())); + } + let aspect_ratio = args + .aspect_ratio + .as_deref() + .unwrap_or(Self::DEFAULT_VIDEO_ASPECT_RATIO); + if !Self::SUPPORTED_ASPECT_RATIOS.contains(&aspect_ratio) { + return Err(GenerateVideoError::InvalidAspectRatio( + aspect_ratio.to_string(), + )); + } + let duration_seconds = args + .duration_seconds + .unwrap_or(Self::DEFAULT_VIDEO_DURATION_SECONDS); + if !Self::SUPPORTED_DURATION_SECONDS.contains(&duration_seconds) { + return Err(GenerateVideoError::InvalidDurationSeconds(duration_seconds)); + } + let resolution = args + .resolution + .as_deref() + .unwrap_or(Self::DEFAULT_VIDEO_RESOLUTION); + if !Self::SUPPORTED_RESOLUTIONS.contains(&resolution) { + return Err(GenerateVideoError::InvalidResolution( + resolution.to_string(), + )); + } + let sound = args.sound.as_deref().unwrap_or(Self::DEFAULT_VIDEO_SOUND); + if !Self::SUPPORTED_SOUND_OPTIONS.contains(&sound) { + return Err(GenerateVideoError::InvalidSound(sound.to_string())); + } + if args.model == Self::SEEDANCE_2_FAST_MODEL + && !Self::SEEDANCE_2_FAST_RESOLUTIONS.contains(&resolution) + { + return Err(GenerateVideoError::UnsupportedModelResolution { + model: args.model.clone(), + resolution: resolution.to_string(), + }); + } + if !args.reference_image_ids.is_empty() + && !Self::REFERENCE_IMAGE_MODELS.contains(&args.model.as_str()) + { + return Err(GenerateVideoError::ReferencesUnsupportedForModel( + args.model.clone(), + )); + } + if args.reference_image_ids.len() > Self::MAX_REFERENCE_IMAGES { + return Err(GenerateVideoError::TooManyReferenceImages( + args.reference_image_ids.len(), + )); + } + for id in &args.reference_image_ids { + if !self.context.contains_image(id) { + return Err(GenerateVideoError::AssetNotFound(id.clone())); + } + } + Ok(()) + } pub fn format_execute_message( &self, @@ -119,3 +271,101 @@ impl GenerateVideoTool { ) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn args() -> GenerateVideoToolArgs { + GenerateVideoToolArgs { + prompt: "镜头向前推进".to_string(), + reference_image_ids: Vec::new(), + aspect_ratio: Some("16:9".to_string()), + duration_seconds: Some(4), + model: GenerateVideoTool::SEEDANCE_2_FAST_MODEL.to_string(), + resolution: Some("720p".to_string()), + sound: Some("on".to_string()), + } + } + + fn tool() -> GenerateVideoTool { + GenerateVideoTool { + context: EditorToolContext::default(), + } + } + + #[test] + fn validates_each_video_option() { + let mut invalid_aspect_ratio = args(); + invalid_aspect_ratio.aspect_ratio = Some("2:1".to_string()); + assert!(matches!( + tool().validate_args(&invalid_aspect_ratio), + Err(GenerateVideoError::InvalidAspectRatio(value)) if value == "2:1" + )); + + let mut invalid_duration = args(); + invalid_duration.duration_seconds = Some(16); + assert!(matches!( + tool().validate_args(&invalid_duration), + Err(GenerateVideoError::InvalidDurationSeconds(16)) + )); + + let mut invalid_resolution = args(); + invalid_resolution.resolution = Some("4K".to_string()); + assert!(matches!( + tool().validate_args(&invalid_resolution), + Err(GenerateVideoError::InvalidResolution(value)) if value == "4K" + )); + + let mut invalid_sound = args(); + invalid_sound.sound = Some("auto".to_string()); + assert!(matches!( + tool().validate_args(&invalid_sound), + Err(GenerateVideoError::InvalidSound(value)) if value == "auto" + )); + } + + #[test] + fn rejects_invalid_model_combinations_and_reference_count() { + let mut invalid = args(); + invalid.resolution = Some("1080p".to_string()); + assert!(matches!( + tool().validate_args(&invalid), + Err(GenerateVideoError::UnsupportedModelResolution { model, resolution }) + if model == GenerateVideoTool::SEEDANCE_2_FAST_MODEL && resolution == "1080p" + )); + + let reference = || ImageId { + id: "reference-image".to_string(), + }; + let mut kling = args(); + kling.model = "kling3.0".to_string(); + kling.reference_image_ids = vec![reference()]; + assert!(matches!( + tool().validate_args(&kling), + Err(GenerateVideoError::ReferencesUnsupportedForModel(model)) if model == "kling3.0" + )); + + let mut too_many = args(); + too_many.reference_image_ids = (0..=GenerateVideoTool::MAX_REFERENCE_IMAGES) + .map(|index| ImageId { + id: format!("reference-{index}"), + }) + .collect(); + assert!(matches!( + tool().validate_args(&too_many), + Err(GenerateVideoError::TooManyReferenceImages(10)) + )); + } + + #[test] + fn schema_exposes_cross_field_constraints_and_current_defaults() { + let schema = tool().parameters(); + assert_eq!( + schema["properties"]["reference_image_ids"]["maxItems"], + json!(9) + ); + assert_eq!(schema["properties"]["sound"]["default"], json!("on")); + assert_eq!(schema["allOf"].as_array().map(Vec::len), Some(2)); + } +} diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/image_generation_options.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/image_generation_options.rs new file mode 100644 index 000000000..8cf725664 --- /dev/null +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/image_generation_options.rs @@ -0,0 +1,167 @@ +use platform_image::{GPT_IMAGE_2_MODEL, NANOBANANA_2_MODEL}; +use serde_json::{Value, json}; +use std::error::Error; +use std::fmt::Display; + +pub const DEFAULT_IMAGE_ASPECT_RATIO: &str = "1:1"; +pub const DEFAULT_IMAGE_SIZE: &str = "1K"; +pub const SUPPORTED_IMAGE_ASPECT_RATIOS: &[&str] = &["1:1", "4:3", "3:2", "2:3", "9:16", "16:9"]; +const NANOBANANA_2_IMAGE_SIZES: &[&str] = &["0.5K", "1K", "2K"]; +const GPT_IMAGE_2_IMAGE_SIZES: &[&str] = &["1K", "2K"]; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ImageGenerationOptionsError { + InvalidModel(String), + InvalidAspectRatio(String), + InvalidImageSize { model: String, image_size: String }, +} + +impl Display for ImageGenerationOptionsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidModel(model) => write!( + f, + "{model} is not a valid image model; supported models: {NANOBANANA_2_MODEL}, {GPT_IMAGE_2_MODEL}" + ), + Self::InvalidAspectRatio(aspect_ratio) => write!( + f, + "{aspect_ratio} is not a valid aspect ratio; supported values: {}", + SUPPORTED_IMAGE_ASPECT_RATIOS.join(", ") + ), + Self::InvalidImageSize { model, image_size } => write!( + f, + "{image_size} is not a valid image size for {model}; supported values: {}", + supported_image_sizes(model).unwrap_or_default().join(", ") + ), + } + } +} + +impl Error for ImageGenerationOptionsError {} + +pub fn default_image_model() -> String { + NANOBANANA_2_MODEL.to_string() +} + +pub fn default_image_aspect_ratio() -> Option { + Some(DEFAULT_IMAGE_ASPECT_RATIO.to_string()) +} + +pub fn default_image_size() -> Option { + Some(DEFAULT_IMAGE_SIZE.to_string()) +} + +pub fn validate_image_generation_options( + model: &str, + aspect_ratio: Option<&str>, + image_size: Option<&str>, +) -> Result<(), ImageGenerationOptionsError> { + let supported_sizes = supported_image_sizes(model) + .ok_or_else(|| ImageGenerationOptionsError::InvalidModel(model.to_string()))?; + let aspect_ratio = aspect_ratio.unwrap_or(DEFAULT_IMAGE_ASPECT_RATIO); + if !SUPPORTED_IMAGE_ASPECT_RATIOS.contains(&aspect_ratio) { + return Err(ImageGenerationOptionsError::InvalidAspectRatio( + aspect_ratio.to_string(), + )); + } + let image_size = image_size.unwrap_or(DEFAULT_IMAGE_SIZE); + if !supported_sizes.contains(&image_size) { + return Err(ImageGenerationOptionsError::InvalidImageSize { + model: model.to_string(), + image_size: image_size.to_string(), + }); + } + Ok(()) +} + +pub fn image_model_parameter_schema() -> Value { + json!({ + "type": "string", + "enum": [NANOBANANA_2_MODEL, GPT_IMAGE_2_MODEL], + "default": NANOBANANA_2_MODEL, + "description": "生图模型。默认 gemini-3.1-flash-image-preview（user may call it nanobanana2）；也可选择 gpt-image-2。" + }) +} + +pub fn image_aspect_ratio_parameter_schema() -> Value { + json!({ + "type": "string", + "enum": SUPPORTED_IMAGE_ASPECT_RATIOS, + "default": DEFAULT_IMAGE_ASPECT_RATIO, + "description": "画面宽高比。可选 1:1、4:3、3:2、2:3、9:16、16:9，默认 1:1。" + }) +} + +pub fn image_size_parameter_schema() -> Value { + json!({ + "type": "string", + "enum": ["0.5K", "1K", "2K"], + "default": DEFAULT_IMAGE_SIZE, + "description": "图片尺寸档位。nanobanana2 支持 0.5K、1K、2K；gpt-image-2 仅支持 1K、2K；默认 1K。" + }) +} + +fn supported_image_sizes(model: &str) -> Option<&'static [&'static str]> { + match model { + NANOBANANA_2_MODEL => Some(NANOBANANA_2_IMAGE_SIZES), + GPT_IMAGE_2_MODEL => Some(GPT_IMAGE_2_IMAGE_SIZES), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_frontend_image_model_dimension_matrix() { + for aspect_ratio in SUPPORTED_IMAGE_ASPECT_RATIOS { + for image_size in NANOBANANA_2_IMAGE_SIZES { + assert!( + validate_image_generation_options( + NANOBANANA_2_MODEL, + Some(aspect_ratio), + Some(image_size), + ) + .is_ok() + ); + } + for image_size in GPT_IMAGE_2_IMAGE_SIZES { + assert!( + validate_image_generation_options( + GPT_IMAGE_2_MODEL, + Some(aspect_ratio), + Some(image_size), + ) + .is_ok() + ); + } + } + + assert!(matches!( + validate_image_generation_options(GPT_IMAGE_2_MODEL, Some("1:1"), Some("0.5K")), + Err(ImageGenerationOptionsError::InvalidImageSize { .. }) + )); + assert!(matches!( + validate_image_generation_options(NANOBANANA_2_MODEL, Some("21:9"), Some("1K")), + Err(ImageGenerationOptionsError::InvalidAspectRatio(_)) + )); + } + + #[test] + fn image_option_schemas_expose_frontend_values_and_defaults() { + assert_eq!( + image_aspect_ratio_parameter_schema()["enum"], + json!(["1:1", "4:3", "3:2", "2:3", "9:16", "16:9"]) + ); + assert_eq!( + image_aspect_ratio_parameter_schema()["default"], + DEFAULT_IMAGE_ASPECT_RATIO + ); + assert_eq!( + image_size_parameter_schema()["enum"], + json!(["0.5K", "1K", "2K"]) + ); + assert_eq!(image_size_parameter_schema()["default"], DEFAULT_IMAGE_SIZE); + } +} diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs index f3ea99cc1..57f355e51 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs @@ -7,3 +7,170 @@ pub mod generate_image; pub mod generate_sound_effect; pub mod generate_ui_design; pub mod generate_video; +pub mod image_generation_options; + +#[cfg(test)] +mod tests { + use super::context::EditorToolContext; + use super::edit_image::{EditImageTool, EditImageToolArgs}; + use super::generate_background_music::{ + GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, + }; + use super::generate_character::{GenerateCharacterTool, GenerateCharacterToolArgs}; + use super::generate_icon_spritesheet::GenerateIconSpritesheetToolArgs; + use super::generate_image::{GenerateImageError, GenerateImageTool, GenerateImageToolArgs}; + use super::generate_sound_effect::{GenerateSoundEffectTool, GenerateSoundEffectToolArgs}; + use super::generate_ui_design::{GenerateUiDesignTool, GenerateUiDesignToolArgs}; + use super::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; + use crate::framework::tool::Tool; + use platform_audio::{SUNO_DEFAULT_MODEL, VIDU_AUDIO_MODEL}; + use platform_image::{GPT_IMAGE_2_MODEL, NANOBANANA_2_MODEL}; + use serde_json::json; + + #[test] + fn tool_args_apply_stable_default_models() { + let image: GenerateImageToolArgs = serde_json::from_value(json!({ + "prompt": "生成森林场景" + })) + .expect("image args should deserialize"); + let edit: EditImageToolArgs = serde_json::from_value(json!({ + "object_image_id": "image-1", + "prompt": "改成夜景" + })) + .expect("edit args should deserialize"); + let character: GenerateCharacterToolArgs = serde_json::from_value(json!({ + "prompt": "生成冒险者角色" + })) + .expect("character args should deserialize"); + let ui_design: GenerateUiDesignToolArgs = serde_json::from_value(json!({ + "prompt": "生成游戏主界面" + })) + .expect("legacy UI design args should deserialize"); + let icon: GenerateIconSpritesheetToolArgs = serde_json::from_value(json!({ + "reference_image_id": "image-1", + "icon_descriptions": ["背包"] + })) + .expect("icon args should deserialize"); + let video: GenerateVideoToolArgs = serde_json::from_value(json!({ + "prompt": "镜头向前推进" + })) + .expect("video args should deserialize"); + let sound: GenerateSoundEffectToolArgs = serde_json::from_value(json!({ + "prompt": "按钮点击声" + })) + .expect("sound args should deserialize"); + let music: GenerateBackgroundMusicToolArgs = serde_json::from_value(json!({ + "prompt": "轻快冒险音乐" + })) + .expect("music args should deserialize"); + + assert_eq!(image.model, NANOBANANA_2_MODEL); + assert_eq!(image.aspect_ratio.as_deref(), Some("1:1")); + assert_eq!(image.image_size.as_deref(), Some("1K")); + assert_eq!(edit.model, GPT_IMAGE_2_MODEL); + assert_eq!(character.model, NANOBANANA_2_MODEL); + assert_eq!(ui_design.model, GPT_IMAGE_2_MODEL); + assert_eq!(icon.model, NANOBANANA_2_MODEL); + assert_eq!(video.model, GenerateVideoTool::DEFAULT_VIDEO_MODEL); + assert_eq!(video.sound.as_deref(), Some("on")); + assert_eq!(sound.model, VIDU_AUDIO_MODEL); + assert_eq!(sound.duration, Some(5)); + assert_eq!(music.model, SUNO_DEFAULT_MODEL); + } + + #[test] + fn generate_image_accepts_nanobanana_and_image2_only() { + let tool = GenerateImageTool { + context: EditorToolContext::default(), + }; + let args = |model: &str| GenerateImageToolArgs { + prompt: "生成森林场景".to_string(), + model: model.to_string(), + reference_image_ids: Vec::new(), + aspect_ratio: None, + image_size: None, + }; + + assert!(tool.validate_args(&args(NANOBANANA_2_MODEL)).is_ok()); + assert!(tool.validate_args(&args(GPT_IMAGE_2_MODEL)).is_ok()); + assert!(matches!( + tool.validate_args(&args("unknown-image-model")), + Err(GenerateImageError::InvalidModel(_)) + )); + assert_eq!( + tool.parameters()["properties"]["model"]["enum"], + json!([NANOBANANA_2_MODEL, GPT_IMAGE_2_MODEL]) + ); + } + + #[test] + fn character_and_ui_tools_validate_their_own_reference_images() { + let missing_image = crate::agent::asset::ImageId { + id: "missing-image".to_string(), + }; + let character_args = GenerateCharacterToolArgs { + prompt: "生成角色".to_string(), + model: NANOBANANA_2_MODEL.to_string(), + reference_image_ids: vec![missing_image.clone()], + aspect_ratio: Some("2:3".to_string()), + image_size: Some("1K".to_string()), + }; + + assert!(matches!( + GenerateCharacterTool { + context: EditorToolContext::default(), + } + .validate_args(&character_args), + Err(GenerateImageError::AssetNotFound(image_id)) if image_id == missing_image + )); + let ui_args = GenerateUiDesignToolArgs { + prompt: "生成游戏主界面".to_string(), + model: GPT_IMAGE_2_MODEL.to_string(), + reference_image_ids: vec![missing_image.clone()], + aspect_ratio: Some("16:9".to_string()), + image_size: Some("1K".to_string()), + }; + assert!(matches!( + GenerateUiDesignTool { + context: EditorToolContext::default(), + } + .validate_args(&ui_args), + Err(GenerateImageError::AssetNotFound(image_id)) if image_id == missing_image + )); + } + + #[test] + fn confirmation_schemas_expose_all_finite_choices_as_enums() { + let context = EditorToolContext::default(); + let edit = EditImageTool { + context: context.clone(), + } + .parameters(); + let video = GenerateVideoTool { + context: context.clone(), + } + .parameters(); + let sound = GenerateSoundEffectTool.parameters(); + let music = GenerateBackgroundMusicTool.parameters(); + + assert_eq!( + edit["properties"]["model"]["enum"], + json!([GPT_IMAGE_2_MODEL]) + ); + assert_eq!( + video["properties"]["duration_seconds"]["enum"], + json!([4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + ); + assert_eq!( + video["properties"]["resolution"]["enum"], + json!(["480p", "720p", "1080p"]) + ); + assert_eq!( + sound["properties"]["duration"]["enum"], + json!([2, 3, 4, 5, 6, 7, 8, 9, 10]) + ); + assert_eq!(sound["properties"]["duration"]["default"], json!(5)); + assert_eq!(video["properties"]["sound"]["default"], json!("on")); + assert!(music["properties"].get("make_instrumental").is_none()); + } +} diff --git a/server-rs/crates/platform-image/src/lib.rs b/server-rs/crates/platform-image/src/lib.rs index 95495b218..ccef9f192 100644 --- a/server-rs/crates/platform-image/src/lib.rs +++ b/server-rs/crates/platform-image/src/lib.rs @@ -3,8 +3,8 @@ pub mod generated_assets; pub mod vector_engine; pub use vector_engine::{ - DownloadedImage, GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL, GeneratedImages, PlatformImageError, - PlatformImageFailureAudit, PlatformImageStatusHint, ReferenceImage, + DownloadedImage, GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL, GeneratedImages, NANOBANANA_2_MODEL, + PlatformImageError, PlatformImageFailureAudit, PlatformImageStatusHint, ReferenceImage, VECTOR_ENGINE_GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER, VectorEngineImageSettings, build_vector_engine_image_http_client, build_vector_engine_image_request_body, build_vector_engine_nanobanana_generate_content_request_body, create_vector_engine_image_edit, diff --git a/server-rs/crates/platform-image/src/vector_engine/constants.rs b/server-rs/crates/platform-image/src/vector_engine/constants.rs index afbe17d03..2da30eef3 100644 --- a/server-rs/crates/platform-image/src/vector_engine/constants.rs +++ b/server-rs/crates/platform-image/src/vector_engine/constants.rs @@ -1,4 +1,5 @@ pub const GPT_IMAGE_2_MODEL: &str = "gpt-image-2"; pub const GPT_IMAGE_2_C_MODEL: &str = "gpt-image-2-c"; +pub const NANOBANANA_2_MODEL: &str = "gemini-3.1-flash-image-preview"; pub const VECTOR_ENGINE_GPT_IMAGE_2_MODEL: &str = GPT_IMAGE_2_MODEL; pub const VECTOR_ENGINE_PROVIDER: &str = "vector-engine"; diff --git a/server-rs/crates/platform-image/src/vector_engine/mod.rs b/server-rs/crates/platform-image/src/vector_engine/mod.rs index 99511b9ac..f64cba54a 100644 --- a/server-rs/crates/platform-image/src/vector_engine/mod.rs +++ b/server-rs/crates/platform-image/src/vector_engine/mod.rs @@ -20,7 +20,8 @@ pub use client::{ create_vector_engine_nanobanana_generate_content, }; pub use constants::{ - GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL, VECTOR_ENGINE_GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER, + GPT_IMAGE_2_C_MODEL, GPT_IMAGE_2_MODEL, NANOBANANA_2_MODEL, VECTOR_ENGINE_GPT_IMAGE_2_MODEL, + VECTOR_ENGINE_PROVIDER, }; pub use error::{PlatformImageError, PlatformImageStatusHint}; pub use image_source::download_remote_image; -- 2.52.0 From 9949e8d06b80cf802edd6d25cff31512f93519e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Tue, 21 Jul 2026 20:28:32 +0800 Subject: [PATCH 02/25] display user friendly model name --- .../【编辑器】画布Agent对话面板-2026-07-03.md | 2 +- .../PendingToolCall.tsx | 7 ++- .../ImageCanvasGenerationModel.ts | 44 ++++++++++++++----- 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/docs/【编辑器】画布Agent对话面板-2026-07-03.md b/docs/【编辑器】画布Agent对话面板-2026-07-03.md index 63749082d..2d6f642c3 100644 --- a/docs/【编辑器】画布Agent对话面板-2026-07-03.md +++ b/docs/【编辑器】画布Agent对话面板-2026-07-03.md @@ -84,7 +84,7 @@ - `GET /conversation` 会在同一个 conversation lock 内扫描 `status=not_completed` 且已有 `externalJobId` 的工具消息：只对这些消息按 job id 定向读取主任务；任务完成后复用对应工具的 `format_execute_message` 替换 system text、回填轻量图片 / 视频 / 音频引用并写为 `completed`，任务失败则回填 `error` 并写为 `failed`。任务结果读取或 completed payload 解析 / formatter 回填失败时，必须在同一次 GET 内完成首次尝试及最多 3 次重试，三次重试各间隔 100ms 并重新读取任务结果；仍失败才把该工具消息写为 `failed` 并保存最后错误。该重试不依赖前端再次刷新。排队和执行中都保持 `not_completed`，整轮扫描结果一次性写回 OSS。 - `EditorAgentToolCall.args` 保留为工具返回的原始 JSON，是确认接口重新反序列化并执行工具的唯一参数真相。图片参数继续只保存由真实 data key 计算出的 opaque SHA-256 `imageId`；不得为了前端预览把 `args` 中的图片 ID 改写成 `objectKey`、URL 或展示对象，也不得由前端重组或回传一份新的执行参数。 - `EditorAgentToolCall.displayArgs` 是必填、只读的用户确认展示投影，与 `args` 分离： - - `stringArgs` 保存提示词、比例、清晰度、模型、时长等可展示参数的稳定名称、用户可见标题和值； + - `stringArgs` 保存提示词、比例、清晰度、模型、时长等可展示参数的稳定名称、用户可见标题和值；前端渲染模型字段时复用图片编辑器公共展示名映射，`gemini-3.1-flash-image-preview` 显示为 `nanobanana2`、`audio1.0` 显示为 `Vidu`、`chirp-v5` 显示为 `Suno`，视频模型显示现有产品标签，不得改写后端参数真相； - `imageArgs` 按“目标图片 / 参考图片”等参数分组，每个 `refs` 项包含与原始参数对应的 `imageId`，以及后端从已校验会话上下文解析出的 `objectKey`、`imageSrc`、可选 `thumbnailSrc` / `label` / `width` / `height`。 - `extras.priceMudPoints` 保存创建待确认消息时按后端运行时模型定价快照计算的预计泥点消耗；前端统一展示为“预计消耗 N泥点”，不自行计算价格。 - `displayArgs` 只能由 api-server 按已注册 tool 白名单，基于已经通过 ToolArgs 校验的 `args` 和当前 OSS 会话文档中的附件 / 历史生成结果构建；不能信任 LLM 自报的展示地址、标题或素材元数据。展示投影不参与确认执行，确认接口仍只读取同一条持久化 tool call 的 `args`，避免“看到的素材”和“实际执行的素材”分叉。 diff --git a/src/components/image-editor/EditorAgentConversation/PendingToolCall.tsx b/src/components/image-editor/EditorAgentConversation/PendingToolCall.tsx index cc53e1341..25881d1ec 100644 --- a/src/components/image-editor/EditorAgentConversation/PendingToolCall.tsx +++ b/src/components/image-editor/EditorAgentConversation/PendingToolCall.tsx @@ -2,6 +2,7 @@ import { Check, Coins, Loader2, Pencil, X } from 'lucide-react'; import type { EditorAgentToolCall } from '@/packages/shared/src/contracts'; import { editorAgentToolLabel } from '@/src/components/image-editor/EditorAgentConversation/toolCallPresentation.ts'; +import { getEditorGenerationModelDisplayName } from '@/src/components/image-editor/ImageCanvasGenerationModel.ts'; import { ResolvedAssetImage } from '@/src/components/ResolvedAssetImage.tsx'; type PendingToolCallAction = 'confirm' | 'cancel' | null; @@ -18,6 +19,10 @@ function readString(value: unknown) { return typeof value === 'string' && value.trim() ? value.trim() : null; } +function displayStringArgument(name: string, value: string) { + return name === 'model' ? getEditorGenerationModelDisplayName(value) : value; +} + export function PendingToolCall({ messageId, toolCall, @@ -64,7 +69,7 @@ export function PendingToolCall({ {argument.label}
- {argument.value} + {displayStringArgument(argument.name, argument.value)}
))} diff --git a/src/components/image-editor/ImageCanvasGenerationModel.ts b/src/components/image-editor/ImageCanvasGenerationModel.ts index e04b43e91..96e7ec07a 100644 --- a/src/components/image-editor/ImageCanvasGenerationModel.ts +++ b/src/components/image-editor/ImageCanvasGenerationModel.ts @@ -582,6 +582,28 @@ export function getEditorImageModelDisplayName( ); } +export function getEditorGenerationModelDisplayName( + model: string | null | undefined, +) { + const normalizedModel = model?.trim(); + if (normalizedModel === BACKGROUND_MUSIC_MODEL_SUNO) { + return 'Suno'; + } + const soundEffectLabel = EDITOR_SOUND_EFFECT_MODEL_OPTIONS.find( + (option) => option.value === normalizedModel, + )?.label; + if (soundEffectLabel) { + return soundEffectLabel; + } + const videoLabel = EDITOR_VIDEO_MODEL_OPTIONS.find( + (option) => option.value === normalizedModel, + )?.label; + if (videoLabel) { + return videoLabel; + } + return getEditorImageModelDisplayName(normalizedModel); +} + export function buildQuickEditModelOptions(currentModel: string) { void currentModel; const options = [...QUICK_EDIT_MODEL_OPTIONS]; @@ -1292,17 +1314,17 @@ export function isCanvasGenerationDialog( ): dialog is CanvasGenerationDialogState { return Boolean( dialog?.id && - (dialog.mode === 'generate' || - dialog.mode === 'spec' || - dialog.mode === 'character' || - dialog.mode === 'icon' || - dialog.mode === 'publication' || - dialog.mode === 'ui-design' || - dialog.mode === 'quick-edit' || - dialog.mode === 'character-animation' || - dialog.mode === 'video' || - dialog.mode === 'audio-sound-effect' || - dialog.mode === 'audio-background-music'), + (dialog.mode === 'generate' || + dialog.mode === 'spec' || + dialog.mode === 'character' || + dialog.mode === 'icon' || + dialog.mode === 'publication' || + dialog.mode === 'ui-design' || + dialog.mode === 'quick-edit' || + dialog.mode === 'character-animation' || + dialog.mode === 'video' || + dialog.mode === 'audio-sound-effect' || + dialog.mode === 'audio-background-music'), ); } -- 2.52.0 From be3fe47187e25725c100d33c5efd16012a0868bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Tue, 21 Jul 2026 20:28:52 +0800 Subject: [PATCH 03/25] adjust system prompt --- .../crates/api-server/src/editor_agent/api.rs | 15 ++++++++++----- .../src/agent/tools/generate_character.rs | 2 +- .../src/agent/tools/generate_icon_spritesheet.rs | 2 +- .../src/agent/tools/generate_image.rs | 2 +- .../src/agent/tools/generate_ui_design.rs | 2 +- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index 274706e87..dccd599aa 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -157,9 +157,9 @@ pub async fn editor_agent_message( let now = now_rfc3339(); if !attachments.is_empty() { let mut attachment_info = String::new(); - attachment_info.push_str("user has just uploaded attachments of the order: "); - for attachment in &attachments { - attachment_info.push_str(&format!("{} ,", attachment.clone().into_image_id())); + attachment_info.push_str("user added these image ids to context: "); + for (i,attachment) in attachments.iter().enumerate() { + attachment_info.push_str(&format!("[{i}]: {}, ", attachment.clone().into_image_id())); } document.messages.push(EditorAgentMessage { id: document.messages.len(), @@ -625,8 +625,13 @@ mod tests { } fn editor_agent_system_prompt() -> &'static str { r#" -你是 Genarrative 图片画布 Agent，只负责帮助用户理解、规划和触发画布生成工具。 -对话回复要简短，不能承诺免费生成；实际生成工具由后端按模型定价扣泥点。 +* image_id str format is like: sha256:* +* when user referenced/uploaded image, a system message will notify you the image id(s). + YOU MUST USE THESE IMAGE IDs(or more from former context) IN YOUR TOOL CALLS.(or why user upload them?) +* 用户所说的 规范图/参考图/生成的图/... 没有本质区别，all can be some image_id +* 实际生成工具由后端按模型定价扣泥点, 不能承诺免费生成 + +你是 Genarrative 图片画布 Agent，负责帮助用户理解、规划和触发画布生成工具. 对话回复要简短. "# } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs index 70f41692a..0f7ddd588 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs @@ -47,7 +47,7 @@ impl Tool for GenerateCharacterTool { "properties": { "prompt": { "type": "string", "description": "角色外貌、服装、姿势、画风和构图的完整描述。" }, "model": image_model_parameter_schema(), - "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选角色或画风参考图 ID。" }, + "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "image_id(s) for desc 角色或画风" }, "aspect_ratio": image_aspect_ratio_parameter_schema(), "image_size": image_size_parameter_schema(), }, diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs index 9e7cff650..56df93ebb 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs @@ -126,7 +126,7 @@ impl Tool for GenerateIconSpritesheetTool { "properties": { "reference_image_id": { "type": "string", "description": "必填的图标规范或风格参考图 ID。" }, "model": image_model_parameter_schema(), - "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选的额外参考图 ID。" }, + "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "image_id(s) for desc" }, "icon_descriptions": { "type": "array", "items": { "type": "string" }, "minItems": 1, "maxItems": GenerateIconSpritesheetTool::MAX_ICON_DESCRIPTIONS, "description": "要生成的多个图标描述，数量必须在 1 到 100 个之间。" }, "aspect_ratio": image_aspect_ratio_parameter_schema(), "image_size": image_size_parameter_schema() diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs index 09512a42c..b8272c3e7 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs @@ -106,7 +106,7 @@ impl Tool for GenerateImageTool { "reference_image_ids": { "type": "array", "items": { "type": "string" }, - "description": "可选参考图 ID 列表，用于提供画风或元素参考。" + "description": "image_id(s) for desc 画风或元素。" }, "aspect_ratio": image_aspect_ratio_parameter_schema(), "image_size": image_size_parameter_schema() diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs index d67f9e3eb..9be752206 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs @@ -55,7 +55,7 @@ impl Tool for GenerateUiDesignTool { "default": GPT_IMAGE_2_MODEL, "description": "UI 设计图固定使用 gpt-image-2。" }, - "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选 UI 风格或布局参考图 ID。" }, + "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "image_id(s) for desc UI 风格或布局" }, "aspect_ratio": image_aspect_ratio_parameter_schema(), "image_size": image_size_parameter_schema() }, -- 2.52.0 From 93c872907b43a24eaf8ae83a72b84156dcfff2f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Wed, 22 Jul 2026 10:52:38 +0800 Subject: [PATCH 04/25] adjust tool call error message --- server-rs/crates/api-server/src/editor_agent/reconcile.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server-rs/crates/api-server/src/editor_agent/reconcile.rs b/server-rs/crates/api-server/src/editor_agent/reconcile.rs index 4b48bd1c0..8351b2429 100644 --- a/server-rs/crates/api-server/src/editor_agent/reconcile.rs +++ b/server-rs/crates/api-server/src/editor_agent/reconcile.rs @@ -158,7 +158,11 @@ fn mark_job_failed(message: &mut EditorAgentMessage, error: String) { .expect("reconcile candidate contains a tool call"); tool_call.error = Some(error.clone()); tool_call.status = EditorAgentToolCallStatus::Failed; - message.text = format!("[tool_call:{}] output: {error}", tool_call.tool_name); + let tool_name = tool_call.tool_name.clone(); + let args = tool_call.args.to_string(); + message.text = format!( + "[tool_call:{tool_name}] args:{args} failed because: {error} if this is upstream/temp error, please retry it if user asked for", + ); } fn mark_fatal_result_reconcile_failed(message: &mut EditorAgentMessage, error: String) { -- 2.52.0 From a871b5f8c4e716a884bc54f86f0574e0a75bd5ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Wed, 22 Jul 2026 11:10:48 +0800 Subject: [PATCH 05/25] refactor: editor agent extract common tool prompt --- .../platform-editor-agent/src/agent/tools/edit_image.rs | 3 ++- .../src/agent/tools/generate_background_music.rs | 5 +++-- .../src/agent/tools/generate_character.rs | 5 ++++- .../src/agent/tools/generate_icon_spritesheet.rs | 3 ++- .../platform-editor-agent/src/agent/tools/generate_image.rs | 3 ++- .../src/agent/tools/generate_sound_effect.rs | 3 ++- .../src/agent/tools/generate_ui_design.rs | 5 ++++- .../platform-editor-agent/src/agent/tools/generate_video.rs | 3 ++- .../crates/platform-editor-agent/src/agent/tools/mod.rs | 1 + 9 files changed, 22 insertions(+), 9 deletions(-) diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs index d4fcc6df5..e1aebda49 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::error::Error; use std::fmt::Display; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct EditImageTool { pub context: EditorToolContext, @@ -108,7 +109,7 @@ impl Tool for EditImageTool { return Err(error); } Ok(EditImageToolOutput { - message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string(), + message: TOOL_CALL_PENDING_MESSAGE.to_string(), }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs index 8cdfb529a..8d659cf5f 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs @@ -1,10 +1,11 @@ use crate::framework::tool::{Tool, ToolFailure}; use platform_audio::SUNO_DEFAULT_MODEL; use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; +use serde_json::{json, Value}; use shared_contracts::assets::EditorAudioGenerateResponse; use std::error::Error; use std::fmt::Display; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct GenerateBackgroundMusicTool; @@ -64,7 +65,7 @@ impl Tool for GenerateBackgroundMusicTool { async move { self.validate_args(&args)?; Ok(GenerateBackgroundMusicToolOutput { - message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string(), + message: TOOL_CALL_PENDING_MESSAGE.to_string(), }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs index 0f7ddd588..47aa4d640 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs @@ -12,6 +12,7 @@ use crate::framework::tool::ToolFailureKind; use crate::framework::tool::{Tool, ToolFailure}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct GenerateCharacterTool { pub context: EditorToolContext, @@ -62,7 +63,9 @@ impl Tool for GenerateCharacterTool { ) -> impl Future> + Send { async move { self.validate_args(&args)?; - Ok(GenerateImageToolOutput { message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string() }) + Ok(GenerateImageToolOutput { + message: TOOL_CALL_PENDING_MESSAGE.to_string(), + }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs index 56df93ebb..ea0cab0ad 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs @@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::error::Error; use std::fmt::Display; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct GenerateIconSpritesheetTool { pub context: EditorToolContext, @@ -143,7 +144,7 @@ impl Tool for GenerateIconSpritesheetTool { async move { self.validate_args(&args)?; Ok(GenerateIconSpritesheetToolOutput { - message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string(), + message: TOOL_CALL_PENDING_MESSAGE.to_string(), }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs index b8272c3e7..fd257612e 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs @@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::error::Error; use std::fmt::Display; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct GenerateImageTool { pub context: EditorToolContext, @@ -123,7 +124,7 @@ impl Tool for GenerateImageTool { async move { self.validate_args(&args)?; Ok(GenerateImageToolOutput { - message: "this tool call is pending user confirmation.".to_string(), + message: TOOL_CALL_PENDING_MESSAGE.to_string(), }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs index dba207011..479c71717 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs @@ -5,6 +5,7 @@ use serde_json::{Value, json}; use shared_contracts::assets::EditorAudioGenerateResponse; use std::error::Error; use std::fmt::Display; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct GenerateSoundEffectTool; @@ -90,7 +91,7 @@ impl Tool for GenerateSoundEffectTool { async move { self.validate_args(&args)?; Ok(GenerateSoundEffectToolOutput { - message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string(), + message:TOOL_CALL_PENDING_MESSAGE.to_string(), }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs index 9be752206..152d9a99a 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs @@ -1,4 +1,5 @@ use crate::agent::asset::ImageId; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; use crate::agent::tools::context::EditorToolContext; use crate::agent::tools::generate_image::{ EditorImageGenerationResult, GenerateImageError, GenerateImageToolOutput, @@ -70,7 +71,9 @@ impl Tool for GenerateUiDesignTool { ) -> impl Future> + Send { async move { self.validate_args(&args)?; - Ok(GenerateImageToolOutput { message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string() }) + Ok(GenerateImageToolOutput { + message: TOOL_CALL_PENDING_MESSAGE.to_string(), + }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs index d802aec42..21935959e 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs @@ -1,4 +1,5 @@ use crate::agent::asset::ImageId; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; use crate::agent::tools::context::EditorToolContext; use crate::framework::tool::{Tool, ToolFailure, ToolFailureKind}; use serde::{Deserialize, Serialize}; @@ -151,7 +152,7 @@ impl Tool for GenerateVideoTool { async move { self.validate_args(&args)?; Ok(GenerateVideoToolOutput { - message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string(), + message: TOOL_CALL_PENDING_MESSAGE.to_string(), }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs index 57f355e51..a8ef88678 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs @@ -8,6 +8,7 @@ pub mod generate_sound_effect; pub mod generate_ui_design; pub mod generate_video; pub mod image_generation_options; +mod common; #[cfg(test)] mod tests { -- 2.52.0 From 8c85b2f8be371776cb493eae9e20f07976c69a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Wed, 22 Jul 2026 11:14:39 +0800 Subject: [PATCH 06/25] refactor: editor agent extract common tool prompt --- .../crates/platform-editor-agent/src/agent/tools/common.rs | 2 ++ .../platform-editor-agent/src/agent/tools/edit_image.rs | 2 +- .../src/agent/tools/generate_background_music.rs | 4 ++-- .../src/agent/tools/generate_character.rs | 4 ++-- .../src/agent/tools/generate_icon_spritesheet.rs | 4 ++-- .../platform-editor-agent/src/agent/tools/generate_image.rs | 4 ++-- .../src/agent/tools/generate_sound_effect.rs | 4 ++-- server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs | 2 +- 8 files changed, 14 insertions(+), 12 deletions(-) create mode 100644 server-rs/crates/platform-editor-agent/src/agent/tools/common.rs diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/common.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/common.rs new file mode 100644 index 000000000..31fd237ce --- /dev/null +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/common.rs @@ -0,0 +1,2 @@ +pub const TOOL_CALL_PENDING_MESSAGE: &'static str = + "this tool call is pending user confirmation. if all is pending, just end this turn"; diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs index e1aebda49..2d52be188 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs @@ -1,4 +1,5 @@ use crate::agent::asset::ImageId; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; use crate::agent::tools::context::EditorToolContext; use crate::framework::tool::{Tool, ToolFailure, ToolFailureKind}; use platform_image::GPT_IMAGE_2_MODEL; @@ -6,7 +7,6 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::error::Error; use std::fmt::Display; -use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct EditImageTool { pub context: EditorToolContext, diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs index 8d659cf5f..8a1c12f47 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs @@ -1,11 +1,11 @@ +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; use crate::framework::tool::{Tool, ToolFailure}; use platform_audio::SUNO_DEFAULT_MODEL; use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; +use serde_json::{Value, json}; use shared_contracts::assets::EditorAudioGenerateResponse; use std::error::Error; use std::fmt::Display; -use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct GenerateBackgroundMusicTool; diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs index 47aa4d640..e3fa88810 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs @@ -1,4 +1,5 @@ use crate::agent::asset::ImageId; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; use crate::agent::tools::context::EditorToolContext; use crate::agent::tools::generate_image::{ EditorImageGenerationResult, GenerateImageError, GenerateImageToolOutput, @@ -12,7 +13,6 @@ use crate::framework::tool::ToolFailureKind; use crate::framework::tool::{Tool, ToolFailure}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; -use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct GenerateCharacterTool { pub context: EditorToolContext, @@ -64,7 +64,7 @@ impl Tool for GenerateCharacterTool { async move { self.validate_args(&args)?; Ok(GenerateImageToolOutput { - message: TOOL_CALL_PENDING_MESSAGE.to_string(), + message: TOOL_CALL_PENDING_MESSAGE.to_string(), }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs index ea0cab0ad..fde5ba420 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs @@ -1,4 +1,5 @@ use crate::agent::asset::ImageId; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; use crate::agent::tools::context::EditorToolContext; use crate::agent::tools::image_generation_options::{ ImageGenerationOptionsError, default_image_aspect_ratio, default_image_model, @@ -11,7 +12,6 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::error::Error; use std::fmt::Display; -use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct GenerateIconSpritesheetTool { pub context: EditorToolContext, @@ -144,7 +144,7 @@ impl Tool for GenerateIconSpritesheetTool { async move { self.validate_args(&args)?; Ok(GenerateIconSpritesheetToolOutput { - message: TOOL_CALL_PENDING_MESSAGE.to_string(), + message: TOOL_CALL_PENDING_MESSAGE.to_string(), }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs index fd257612e..2e3e44386 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs @@ -1,4 +1,5 @@ use crate::agent::asset::ImageId; +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; use crate::agent::tools::context::EditorToolContext; use crate::agent::tools::image_generation_options::{ ImageGenerationOptionsError, default_image_aspect_ratio, default_image_model, @@ -11,7 +12,6 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::error::Error; use std::fmt::Display; -use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct GenerateImageTool { pub context: EditorToolContext, @@ -124,7 +124,7 @@ impl Tool for GenerateImageTool { async move { self.validate_args(&args)?; Ok(GenerateImageToolOutput { - message: TOOL_CALL_PENDING_MESSAGE.to_string(), + message: TOOL_CALL_PENDING_MESSAGE.to_string(), }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs index 479c71717..9496003bd 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs @@ -1,3 +1,4 @@ +use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; use crate::framework::tool::{Tool, ToolFailure}; use platform_audio::VIDU_AUDIO_MODEL; use serde::{Deserialize, Serialize}; @@ -5,7 +6,6 @@ use serde_json::{Value, json}; use shared_contracts::assets::EditorAudioGenerateResponse; use std::error::Error; use std::fmt::Display; -use crate::agent::tools::common::TOOL_CALL_PENDING_MESSAGE; pub struct GenerateSoundEffectTool; @@ -91,7 +91,7 @@ impl Tool for GenerateSoundEffectTool { async move { self.validate_args(&args)?; Ok(GenerateSoundEffectToolOutput { - message:TOOL_CALL_PENDING_MESSAGE.to_string(), + message: TOOL_CALL_PENDING_MESSAGE.to_string(), }) } } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs index a8ef88678..5aa5a3537 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/mod.rs @@ -1,3 +1,4 @@ +mod common; pub mod context; pub mod edit_image; pub mod generate_background_music; @@ -8,7 +9,6 @@ pub mod generate_sound_effect; pub mod generate_ui_design; pub mod generate_video; pub mod image_generation_options; -mod common; #[cfg(test)] mod tests { -- 2.52.0 From b11631ce952b80c07c5325e6eb64ba04b2982d70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Wed, 22 Jul 2026 11:15:05 +0800 Subject: [PATCH 07/25] format --- server-rs/crates/api-server/src/editor_agent/api.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index dccd599aa..32ad7de44 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -158,8 +158,9 @@ pub async fn editor_agent_message( if !attachments.is_empty() { let mut attachment_info = String::new(); attachment_info.push_str("user added these image ids to context: "); - for (i,attachment) in attachments.iter().enumerate() { - attachment_info.push_str(&format!("[{i}]: {}, ", attachment.clone().into_image_id())); + for (i, attachment) in attachments.iter().enumerate() { + attachment_info + .push_str(&format!("[{i}]: {}, ", attachment.clone().into_image_id())); } document.messages.push(EditorAgentMessage { id: document.messages.len(), -- 2.52.0 From 511dd98ed51fa8cc90cf78d9d1d3a701b7ce82d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Wed, 22 Jul 2026 11:26:57 +0800 Subject: [PATCH 08/25] fix: editor agent adjust tool call to tip user to use the confirm button --- server-rs/crates/api-server/src/editor_agent/api.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index 32ad7de44..3e075a3bb 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -629,6 +629,8 @@ fn editor_agent_system_prompt() -> &'static str { * image_id str format is like: sha256:* * when user referenced/uploaded image, a system message will notify you the image id(s). YOU MUST USE THESE IMAGE IDs(or more from former context) IN YOUR TOOL CALLS.(or why user upload them?) +* to confirm a pending tool call, user should click a confirm button in their UI, instead of tell you "ok"/"confirm". + If in that case, you should tip the user to use the confirm button, instead of repeat that pending tool call. * 用户所说的 规范图/参考图/生成的图/... 没有本质区别，all can be some image_id * 实际生成工具由后端按模型定价扣泥点, 不能承诺免费生成 -- 2.52.0 From 53a66a3650e094d9b7aceba3fe62c941f5ea7c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Wed, 22 Jul 2026 13:12:31 +0800 Subject: [PATCH 09/25] update doc --- docs/project-memory/shared-memory/decision-log.md | 4 ++-- docs/【编辑器】画布Agent对话面板-2026-07-03.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 01345b3cb..c2be358ba 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -81,8 +81,8 @@ ## 2026-07-10 画布 Agent 工具确认分离执行参数与展示投影 -- 背景：画布 Agent 已在实际生成前进入 `pending_confirmation`，但 `EditorAgentToolCall.args` 只保存工具私有 JSON，其中图片参数是保护真实 data key 的 SHA-256 opaque ID。前端直接解析 raw args 只能显示内部哈希或图片数量，无法向用户准确展示即将使用的目标图、参考图和完整参数；若直接把图片 URL 或对象塞回 raw args，又会破坏确认执行反序列化和 LLM 不可见真实 data key 的安全边界。 -- 决策：`EditorAgentToolCall.args` 继续作为确认执行唯一真相，不允许前端改写或回传替代参数；新增必填 `displayArgs` 只读展示投影，内含 `stringArgs`、`imageArgs` 和 `extras.priceMudPoints`。`stringArgs` 承载提示词与规格等用户可见字段，`imageArgs.refs` 承载 `imageId` 及后端解析出的 `objectKey`、`imageSrc`、可选缩略图、标签和尺寸；`extras.priceMudPoints` 由 api-server 在创建待确认消息时使用后端运行时模型定价快照计算，前端只显示“预计消耗 N泥点”，不自行计算或回传价格。api-server 必须按已注册 tool 白名单，从已校验 args 与 OSS 会话文档的附件 / 历史生成结果构建该投影；前端只渲染投影，以 `ResolvedAssetImage` 换签显示图片，不解析 tool 私有 schema、不展示 SHA-256 ID。展示价格不参与确认执行或实际扣费，确认后仍由既有生成 BFF 按后端运行时定价预扣费。删除只重复 `args` 且没有稳定语义的 `EditorAgentToolCall.summary`。模块尚未上线，不保留缺少 `displayArgs` 时读取 raw `args` 的旧消息降级路径。 +- 背景：画布 Agent 已在实际生成前进入 `pending_confirmation`，但 `EditorAgentToolCall.args` 只保存工具私有的规范参数 JSON，其中图片参数是保护真实 data key 的 SHA-256 opaque ID。前端直接解析 `args` 只能显示内部哈希或图片数量，无法向用户准确展示即将使用的目标图、参考图和完整参数；若直接把图片 URL 或对象塞回 `args`，又会破坏确认执行反序列化和 LLM 不可见真实 data key 的安全边界。 +- 决策：LLM 返回的原始工具参数只作为 api-server 本次处理的瞬时输入；后端按已注册 ToolArgs 反序列化、补齐默认值、删除未知 / 退役字段、完成工具参数校验并重新序列化后，才把结果写入 `EditorAgentToolCall.args`。校验失败的调用不得持久化为待确认消息。该规范 `args` 是确认执行唯一真相，不允许前端改写或回传替代参数；新增必填 `displayArgs` 只读展示投影，内含 `stringArgs`、`imageArgs` 和 `extras.priceMudPoints`。`stringArgs` 承载提示词与规格等用户可见字段，`imageArgs.refs` 承载规范 `args` 中的 `imageId` 及后端解析出的 `objectKey`、`imageSrc`、可选缩略图、标签和尺寸；`extras.priceMudPoints` 由 api-server 在创建待确认消息时使用后端运行时模型定价快照计算，前端只显示“预计消耗 N泥点”，不自行计算或回传价格。api-server 必须按已注册 tool 白名单，从规范 `args` 与 OSS 会话文档的附件 / 历史生成结果构建该投影；前端只渲染投影，以 `ResolvedAssetImage` 换签显示图片，不解析 tool 私有 schema、不展示 SHA-256 ID。展示价格不参与确认执行或实际扣费，确认后仍由既有生成 BFF 按后端运行时定价预扣费。删除只重复 `args` 且没有稳定语义的 `EditorAgentToolCall.summary`。模块尚未上线，不保留缺少 `displayArgs` 时读取 `args` 的旧消息降级路径。 - 影响范围：`shared-contracts` / `packages/shared` 的 `editorAgent` DTO、`api-server/src/editor_agent/api.rs` 的待确认消息构建、画布 Agent 待确认卡、OSS 会话消息文档与相关测试。 - 验证方式：`cargo test -p shared-contracts --manifest-path server-rs/Cargo.toml editor_agent`、`cargo test -p api-server --manifest-path server-rs/Cargo.toml editor_agent`、`npm run test -- src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx src/services/image-editor/editorAgentClient.test.ts`、`npm run typecheck`、`npm run check:encoding`、`git diff --check`。 - 关联文档：`docs/【编辑器】画布Agent对话面板-2026-07-03.md`、`docs/adr/【ADR】画布Agent会话消息存OSS-2026-07-03.md`。 diff --git a/docs/【编辑器】画布Agent对话面板-2026-07-03.md b/docs/【编辑器】画布Agent对话面板-2026-07-03.md index 2d6f642c3..2bd30c961 100644 --- a/docs/【编辑器】画布Agent对话面板-2026-07-03.md +++ b/docs/【编辑器】画布Agent对话面板-2026-07-03.md @@ -82,10 +82,10 @@ - 确认接口必须先把工具参数转换为既有编辑器 worker payload，再使用 `editor-agent:{conversationId}:{messageId}:{toolName}` 稳定 dedupe key 入队；同一确认的请求重试只能得到同一个 external job。入队成功后把返回的 job id 写回同一条 OSS 工具消息，不新增 Agent 工具执行关联表。 - 前端根据 `externalJobId` 查询通用 external-generation job 状态；worker 继续通过 `canvasCompletion` 把生成结果写回工程与素材库。浏览器断线、刷新或 api-server 重启不得导致确认接口重新扣费或重新提交 provider。 - `GET /conversation` 会在同一个 conversation lock 内扫描 `status=not_completed` 且已有 `externalJobId` 的工具消息：只对这些消息按 job id 定向读取主任务；任务完成后复用对应工具的 `format_execute_message` 替换 system text、回填轻量图片 / 视频 / 音频引用并写为 `completed`，任务失败则回填 `error` 并写为 `failed`。任务结果读取或 completed payload 解析 / formatter 回填失败时，必须在同一次 GET 内完成首次尝试及最多 3 次重试，三次重试各间隔 100ms 并重新读取任务结果；仍失败才把该工具消息写为 `failed` 并保存最后错误。该重试不依赖前端再次刷新。排队和执行中都保持 `not_completed`，整轮扫描结果一次性写回 OSS。 -- `EditorAgentToolCall.args` 保留为工具返回的原始 JSON，是确认接口重新反序列化并执行工具的唯一参数真相。图片参数继续只保存由真实 data key 计算出的 opaque SHA-256 `imageId`；不得为了前端预览把 `args` 中的图片 ID 改写成 `objectKey`、URL 或展示对象，也不得由前端重组或回传一份新的执行参数。 +- `EditorAgentToolCall.args` 的正式持久化契约是**校验后的规范参数 JSON**，不是 LLM 返回的原始 JSON。api-server 收到工具调用后，必须先按已注册的 ToolArgs 反序列化、补齐字段默认值、删除未进入 ToolArgs 的未知 / 退役字段、执行工具参数校验，再重新序列化并写入 `args`；校验失败的调用不得持久化为待确认消息。LLM 原始参数只作为本次规范化的瞬时输入，不作为执行或审计真相；确认、取消、任务回填与后续上下文统一读取同一条消息中的规范 `args`。图片参数继续只保存由真实 data key 计算出的 opaque SHA-256 `imageId`；不得为了前端预览把 `args` 中的图片 ID 改写成 `objectKey`、URL 或展示对象，也不得由前端重组或回传一份新的执行参数。 - `EditorAgentToolCall.displayArgs` 是必填、只读的用户确认展示投影，与 `args` 分离： - `stringArgs` 保存提示词、比例、清晰度、模型、时长等可展示参数的稳定名称、用户可见标题和值；前端渲染模型字段时复用图片编辑器公共展示名映射，`gemini-3.1-flash-image-preview` 显示为 `nanobanana2`、`audio1.0` 显示为 `Vidu`、`chirp-v5` 显示为 `Suno`，视频模型显示现有产品标签，不得改写后端参数真相； - - `imageArgs` 按“目标图片 / 参考图片”等参数分组，每个 `refs` 项包含与原始参数对应的 `imageId`，以及后端从已校验会话上下文解析出的 `objectKey`、`imageSrc`、可选 `thumbnailSrc` / `label` / `width` / `height`。 + - `imageArgs` 按“目标图片 / 参考图片”等参数分组，每个 `refs` 项包含与规范参数对应的 `imageId`，以及后端从已校验会话上下文解析出的 `objectKey`、`imageSrc`、可选 `thumbnailSrc` / `label` / `width` / `height`。 - `extras.priceMudPoints` 保存创建待确认消息时按后端运行时模型定价快照计算的预计泥点消耗；前端统一展示为“预计消耗 N泥点”，不自行计算价格。 - `displayArgs` 只能由 api-server 按已注册 tool 白名单，基于已经通过 ToolArgs 校验的 `args` 和当前 OSS 会话文档中的附件 / 历史生成结果构建；不能信任 LLM 自报的展示地址、标题或素材元数据。展示投影不参与确认执行，确认接口仍只读取同一条持久化 tool call 的 `args`，避免“看到的素材”和“实际执行的素材”分叉。 - `extras.priceMudPoints` 同样只属于展示投影，不作为扣费输入；确认后仍由既有生成 BFF 按后端运行时定价执行预扣费，因此该字段表达用户确认时看到的价格快照，而不是前端可提交或覆盖的计费真相。 -- 2.52.0 From 2540c13ddb122c25bc67a05727aaf75977690237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Wed, 22 Jul 2026 14:01:21 +0800 Subject: [PATCH 10/25] feat: editor agent include attachment human readable label to context --- packages/shared/src/contracts/editorAgent.ts | 25 +++++++++++- .../crates/api-server/src/editor_agent/api.rs | 10 ++++- .../AttachmentChip.tsx | 2 +- .../EditorAgentConversationPanelView.test.tsx | 7 ++-- .../EditorAgentConversationPanelView.tsx | 38 ++++++++++--------- .../useEditorAgentConversation.test.tsx | 16 ++++---- 6 files changed, 67 insertions(+), 31 deletions(-) diff --git a/packages/shared/src/contracts/editorAgent.ts b/packages/shared/src/contracts/editorAgent.ts index b85143eae..34185b756 100644 --- a/packages/shared/src/contracts/editorAgent.ts +++ b/packages/shared/src/contracts/editorAgent.ts @@ -1,6 +1,7 @@ // 画布Agent对话契约：会话元数据存 SpacetimeDB，消息正文整体存 OSS（editor-agent/{conversationId}.json）。 export const EDITOR_AGENT_MAX_ATTACHMENTS = 9; +export const EDITOR_AGENT_ATTACHMENT_LABEL_MAX_CODE_POINTS = 10; export const EDITOR_AGENT_ERROR_MESSAGE_PREFIX = 'ERROR '; export type EditorAgentMessageRole = 'user' | 'assistant' | 'system'; @@ -13,17 +14,39 @@ export type EditorAgentToolCallStatus = export type EditorAgentAttachmentSource = 'canvas_resource' | 'library_asset'; +function normalizeEditorAgentAttachmentLabel( + label: string | null | undefined, + fallback: string, +) { + const normalized = label?.trim() || fallback.trim() || '图片'; + return Array.from(normalized) + .slice(0, EDITOR_AGENT_ATTACHMENT_LABEL_MAX_CODE_POINTS) + .join(''); +} + export interface EditorAgentAttachmentRef { source: EditorAgentAttachmentSource; referenceId: string; objectKey?: string | null; imageSrc: string; thumbnailSrc?: string | null; - label?: string | null; + label: string; width?: number | null; height?: number | null; } +export function createEditorAgentAttachmentRef( + input: Omit & { + label?: string | null; + }, + fallbackLabel: string = input.referenceId, +): EditorAgentAttachmentRef { + return { + ...input, + label: normalizeEditorAgentAttachmentLabel(input.label, fallbackLabel), + }; +} + export interface EditorAgentGeneratedImage { resourceId?: string | null; objectKey?: string | null; diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index 3e075a3bb..a95d9870c 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -159,8 +159,14 @@ pub async fn editor_agent_message( let mut attachment_info = String::new(); attachment_info.push_str("user added these image ids to context: "); for (i, attachment) in attachments.iter().enumerate() { - attachment_info - .push_str(&format!("[{i}]: {}, ", attachment.clone().into_image_id())); + let image_label_str = match &attachment.label { + None => "".to_string(), + Some(label) => { + format!(" {label}") + } + }; + let image_id = attachment.clone().into_image_id(); + attachment_info.push_str(&format!("({i}{image_label_str}): {image_id}, ")); } document.messages.push(EditorAgentMessage { id: document.messages.len(), diff --git a/src/components/image-editor/EditorAgentConversation/AttachmentChip.tsx b/src/components/image-editor/EditorAgentConversation/AttachmentChip.tsx index 81b327f57..5d5d1f4a7 100644 --- a/src/components/image-editor/EditorAgentConversation/AttachmentChip.tsx +++ b/src/components/image-editor/EditorAgentConversation/AttachmentChip.tsx @@ -10,7 +10,7 @@ function AttachmentChip({ attachment: EditorAgentAttachmentRef; onRemove?: () => void; }) { - const label = attachment.label?.trim() || attachment.referenceId; + const label = attachment.label; return (