diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index c42398316..b9f967013 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -32,7 +32,21 @@ use crate::auth::AuthenticatedAccessToken; use crate::editor_agent::agent::LlmChatAgentBuilder; use crate::editor_agent::editor_tools::common::EditorToolContext; use crate::editor_agent::editor_tools::edit_image::{EditImageTool, EditImageToolArgs}; -use crate::editor_agent::editor_tools::generate_image::{GenerateImageTool, GenerateImageToolArgs}; +use crate::editor_agent::editor_tools::generate_background_music::{ + GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, +}; +use crate::editor_agent::editor_tools::generate_character::GenerateCharacterTool; +use crate::editor_agent::editor_tools::generate_icon_spritesheet::{ + GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, +}; +use crate::editor_agent::editor_tools::generate_image::{ + EditorImageGenerationResult, GenerateImageTool, GenerateImageToolArgs, +}; +use crate::editor_agent::editor_tools::generate_sound_effect::{ + GenerateSoundEffectTool, GenerateSoundEffectToolArgs, +}; +use crate::editor_agent::editor_tools::generate_ui_design::GenerateUiDesignTool; +use crate::editor_agent::editor_tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; use crate::editor_agent::utils::{ ImageId, ImageMetadata, IntoImageId, build_editor_agent_canvas_completion, conversation_detail_from_record, conversation_summary_from_record, editor_agent_bad_request, @@ -151,6 +165,20 @@ pub async fn editor_agent_message( context: tool_context.clone(), }) .tool(GenerateImageTool { + context: tool_context.clone(), + }) + .tool(GenerateCharacterTool { + context: tool_context.clone(), + }) + .tool(GenerateIconSpritesheetTool { + context: tool_context.clone(), + }) + .tool(GenerateSoundEffectTool) + .tool(GenerateBackgroundMusicTool) + .tool(GenerateVideoTool { + context: tool_context.clone(), + }) + .tool(GenerateUiDesignTool { context: tool_context, }) .max_turns(3) @@ -509,26 +537,281 @@ pub async fn confirm_editor_agent_tool_call( let tool_args = tc.args.clone(); // Each confirmed tool reuses its existing editor-generation entry point. - match tool_name.as_str() { + match tc.tool_name.as_str() { + GenerateSoundEffectTool::NAME => { + let args: GenerateSoundEffectToolArgs = serde_json::from_value(tc.args.clone()) + .map_err(|error| { + editor_agent_bad_request(format!("invalid tool call args: {error}")) + })?; + let project = load_editor_agent_project(&state, &conversation).await?; + let title = args.prompt.clone(); + let tool = GenerateSoundEffectTool; + let result = tool + .execute( + &state, + &request_context, + conversation.owner_user_id.clone(), + args.clone(), + conversation.project_id.clone(), + Some(editor_agent_media_generation_inputs( + &conversation, + message_id, + "用户指令", + &args.prompt, + )), + Some(title.clone()), + build_editor_agent_canvas_completion( + &project, + GenerateSoundEffectTool::NAME, + &title, + ), + ) + .await?; + complete_editor_agent_tool_call( + &state, + &request_context, + &conversation, + &mut document, + message_id, + tool.format_execute_message(&args, &result), + vec![editor_agent_generated_media( + result.resource.as_ref(), + result.object_key, + result.asset_object_id, + result.audio_src, + None, + result.width, + result.height, + )], + ) + .await + } + GenerateBackgroundMusicTool::NAME => { + let args: GenerateBackgroundMusicToolArgs = serde_json::from_value(tc.args.clone()) + .map_err(|error| { + editor_agent_bad_request(format!("invalid tool call args: {error}")) + })?; + let project = load_editor_agent_project(&state, &conversation).await?; + let title = "生成背景音乐".to_string(); + let tool = GenerateBackgroundMusicTool; + let result = tool + .execute( + &state, + &request_context, + conversation.owner_user_id.clone(), + args.clone(), + conversation.project_id.clone(), + Some(editor_agent_media_generation_inputs( + &conversation, + message_id, + "用户指令", + &args.prompt, + )), + Some(title.clone()), + build_editor_agent_canvas_completion( + &project, + GenerateBackgroundMusicTool::NAME, + &title, + ), + ) + .await?; + complete_editor_agent_tool_call( + &state, + &request_context, + &conversation, + &mut document, + message_id, + tool.format_execute_message(&args, &result), + vec![editor_agent_generated_media( + result.resource.as_ref(), + result.object_key, + result.asset_object_id, + result.audio_src, + None, + result.width, + result.height, + )], + ) + .await + } + GenerateVideoTool::NAME => { + let args: GenerateVideoToolArgs = + serde_json::from_value(tc.args.clone()).map_err(|error| { + editor_agent_bad_request(format!("invalid tool call args: {error}")) + })?; + let project = load_editor_agent_project(&state, &conversation).await?; + let title = args.prompt.clone(); + let tool = GenerateVideoTool { + context: build_tool_context(&document), + }; + let result = tool + .execute( + &state, + &request_context, + conversation.owner_user_id.clone(), + args.clone(), + conversation.project_id.clone(), + Some(editor_agent_media_generation_inputs( + &conversation, + message_id, + "用户指令", + &args.prompt, + )), + Some(title.clone()), + build_editor_agent_canvas_completion(&project, GenerateVideoTool::NAME, &title), + ) + .await?; + complete_editor_agent_tool_call( + &state, + &request_context, + &conversation, + &mut document, + message_id, + tool.format_execute_message(&args, &result), + vec![editor_agent_generated_media( + result.resource.as_ref(), + result.object_key, + result.asset_object_id, + result.video_src, + result.thumbnail_src, + result.width, + result.height, + )], + ) + .await + } + GenerateIconSpritesheetTool::NAME => { + let args: GenerateIconSpritesheetToolArgs = serde_json::from_value(tc.args.clone()) + .map_err(|e| editor_agent_bad_request(format!("invalid tool call args: {e}")))?; + let project = load_editor_agent_project(&state, &conversation).await?; + let title = "图标素材图集".to_string(); + let canvas_completion = build_editor_agent_canvas_completion( + &project, + GenerateIconSpritesheetTool::NAME, + &title, + ); + let tool = GenerateIconSpritesheetTool { + context: build_tool_context(&document), + }; + let result = tool + .execute( + &state, + &request_context, + editor_agent_generation_caller(&conversation), + args.clone(), + conversation.project_id.clone(), + Some(json!({ + "source": "editor-agent", + "conversationId": conversation.conversation_id, + "toolCallMessageId": message_id, + "fields": [{ "title": "图标描述", "value": args.icon_descriptions }], + })), + canvas_completion, + ) + .await?; + let image = EditorAgentGeneratedImage { + resource_id: result + .spritesheet_resource + .as_ref() + .and_then(|resource| resource.get("resourceId")) + .and_then(Value::as_str) + .map(ToOwned::to_owned), + object_key: result + .spritesheet_resource + .as_ref() + .and_then(|resource| resource.get("objectKey")) + .and_then(Value::as_str) + .map(ToOwned::to_owned), + asset_object_id: result + .spritesheet_resource + .as_ref() + .and_then(|resource| resource.get("assetObjectId")) + .and_then(Value::as_str) + .map(ToOwned::to_owned), + image_src: result.spritesheet_image_src.clone(), + thumbnail_src: None, + width: Some(result.spritesheet_width), + height: Some(result.spritesheet_height), + }; + complete_editor_agent_tool_call( + &state, + &request_context, + &conversation, + &mut document, + message_id, + tool.format_execute_message(&args, &result), + vec![image], + ) + .await + } + GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => { + let args: GenerateImageToolArgs = serde_json::from_value(tc.args.clone()) + .map_err(|e| editor_agent_bad_request(format!("invalid tool call args: {e}")))?; + let project = load_editor_agent_project(&state, &conversation).await?; + let caller = editor_agent_generation_caller(&conversation); + let title = args.prompt.clone(); + let canvas_completion = + build_editor_agent_canvas_completion(&project, tc.tool_name.as_str(), &title); + let generation_inputs = + editor_agent_generation_inputs(&conversation, message_id, &args); + let result = if tc.tool_name == GenerateCharacterTool::NAME { + GenerateCharacterTool { + context: build_tool_context(&document), + } + .execute( + &state, + &request_context, + caller, + args.clone(), + conversation.project_id.clone(), + generation_inputs, + Some(title), + canvas_completion, + ) + .await? + } else { + GenerateUiDesignTool { + context: build_tool_context(&document), + } + .execute( + &state, + &request_context, + caller, + args.clone(), + conversation.project_id.clone(), + generation_inputs, + Some(title), + canvas_completion, + ) + .await? + }; + let formatted_text = if tc.tool_name == GenerateCharacterTool::NAME { + GenerateCharacterTool { + context: build_tool_context(&document), + } + .format_execute_message(&args, &result) + } else { + GenerateUiDesignTool { + context: build_tool_context(&document), + } + .format_execute_message(&args, &result) + }; + complete_editor_agent_image_tool_call( + &state, + &request_context, + &conversation, + &mut document, + message_id, + formatted_text, + result, + ) + .await + } GenerateImageTool::NAME => { let args: GenerateImageToolArgs = serde_json::from_value(tc.args.clone()) .map_err(|e| editor_agent_bad_request(format!("invalid tool call args: {e}")))?; - let project = state - .spacetime_client() - .get_editor_project(EditorProjectGetRecordInput { - project_id: conversation.project_id.clone(), - owner_user_id: conversation.owner_user_id.clone(), - }) - .await - .map_err(|e| { - AppError::from_status(axum::http::StatusCode::NOT_FOUND) - .with_details(json!({ "message": format!("project not found: {e}") })) - })?; - let caller = EditorGenerationCaller { - owner_user_id: conversation.owner_user_id.clone(), - audit_subject_user_id: Some(conversation.owner_user_id.clone()), - audit_project_id: Some(conversation.project_id.clone()), - }; + let project = load_editor_agent_project(&state, &conversation).await?; + let caller = editor_agent_generation_caller(&conversation); let title = args.prompt.clone(); let canvas_completion = build_editor_agent_canvas_completion(&project, GenerateImageTool::NAME, &title); @@ -543,42 +826,22 @@ pub async fn confirm_editor_agent_tool_call( caller, args.clone(), conversation.project_id.clone(), - Some(json!({ - "source": "editor-agent", - "conversationId": conversation.conversation_id, - "toolCallMessageId": message_id, - "fields": [{ "title": "用户指令", "value": args.prompt }], - })), + editor_agent_generation_inputs(&conversation, message_id, &args), Some(title), canvas_completion, ) .await?; - let generated_images = vec![EditorAgentGeneratedImage { - resource_id: result - .resource - .as_ref() - .and_then(|resource| resource.get("resourceId")) - .and_then(Value::as_str) - .map(ToOwned::to_owned), - object_key: result.object_key.clone(), - asset_object_id: result.asset_object_id.clone(), - image_src: result.image_src.clone(), - thumbnail_src: None, - width: Some(result.width), - height: Some(result.height), - }]; let formatted_text = generate_tool.format_execute_message(&args, &result); - let msg = &mut document.messages[message_id]; - msg.text = formatted_text; - if let Some(tool_call) = &mut msg.tool_call { - tool_call.status = EditorAgentToolCallStatus::Completed; - tool_call.images = generated_images; - } - write_messages_document(&state, &conversation, &document).await?; - Ok(json_success_body( - Some(&request_context), - &document.messages[message_id], - )) + complete_editor_agent_image_tool_call( + &state, + &request_context, + &conversation, + &mut document, + message_id, + formatted_text, + result, + ) + .await } EditImageTool::NAME => { let args: EditImageToolArgs = serde_json::from_value(tool_args) @@ -685,3 +948,137 @@ pub async fn confirm_editor_agent_tool_call( ))), } } + +async fn load_editor_agent_project( + state: &AppState, + conversation: &EditorAgentConversationRecord, +) -> Result { + state + .spacetime_client() + .get_editor_project(EditorProjectGetRecordInput { + project_id: conversation.project_id.clone(), + owner_user_id: conversation.owner_user_id.clone(), + }) + .await + .map_err(|error| { + AppError::from_status(axum::http::StatusCode::NOT_FOUND) + .with_details(json!({ "message": format!("project not found: {error}") })) + }) +} + +fn editor_agent_generation_caller( + conversation: &EditorAgentConversationRecord, +) -> EditorGenerationCaller { + EditorGenerationCaller { + owner_user_id: conversation.owner_user_id.clone(), + audit_subject_user_id: Some(conversation.owner_user_id.clone()), + audit_project_id: Some(conversation.project_id.clone()), + } +} + +fn editor_agent_generation_inputs( + conversation: &EditorAgentConversationRecord, + message_id: usize, + args: &GenerateImageToolArgs, +) -> Option { + Some(json!({ + "source": "editor-agent", + "conversationId": conversation.conversation_id, + "toolCallMessageId": message_id, + "fields": [{ "title": "用户指令", "value": args.prompt }], + })) +} + +fn editor_agent_media_generation_inputs( + conversation: &EditorAgentConversationRecord, + message_id: usize, + title: &str, + value: &str, +) -> Value { + json!({ + "source": "editor-agent", + "conversationId": conversation.conversation_id, + "toolCallMessageId": message_id, + "fields": [{ "title": title, "value": value }], + }) +} + +fn editor_agent_generated_media( + resource: Option<&Value>, + object_key: Option, + asset_object_id: Option, + image_src: String, + thumbnail_src: Option, + width: u32, + height: u32, +) -> EditorAgentGeneratedImage { + EditorAgentGeneratedImage { + resource_id: resource + .and_then(|value| value.get("resourceId")) + .and_then(Value::as_str) + .map(ToOwned::to_owned), + object_key, + asset_object_id, + image_src, + thumbnail_src, + width: Some(width), + height: Some(height), + } +} + +async fn complete_editor_agent_image_tool_call( + state: &AppState, + request_context: &RequestContext, + conversation: &EditorAgentConversationRecord, + document: &mut EditorAgentConversationMessagesDocument, + message_id: usize, + formatted_text: String, + result: EditorImageGenerationResult, +) -> Result, AppError> { + let generated_images = vec![EditorAgentGeneratedImage { + resource_id: result + .resource + .as_ref() + .and_then(|resource| resource.get("resourceId")) + .and_then(Value::as_str) + .map(ToOwned::to_owned), + object_key: result.object_key, + asset_object_id: result.asset_object_id, + image_src: result.image_src, + thumbnail_src: None, + width: Some(result.width), + height: Some(result.height), + }]; + complete_editor_agent_tool_call( + state, + request_context, + conversation, + document, + message_id, + formatted_text, + generated_images, + ) + .await +} + +async fn complete_editor_agent_tool_call( + state: &AppState, + request_context: &RequestContext, + conversation: &EditorAgentConversationRecord, + document: &mut EditorAgentConversationMessagesDocument, + message_id: usize, + formatted_text: String, + images: Vec, +) -> Result, AppError> { + let message = &mut document.messages[message_id]; + message.text = formatted_text; + if let Some(tool_call) = &mut message.tool_call { + tool_call.status = EditorAgentToolCallStatus::Completed; + tool_call.images = images; + } + write_messages_document(state, conversation, document).await?; + Ok(json_success_body( + Some(request_context), + &document.messages[message_id], + )) +} diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs index e39e372a9..0c99dd43f 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs @@ -1,6 +1,6 @@ +use crate::editor_agent::utils::{ImageId, ImageMetadata}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use crate::editor_agent::utils::{ImageId, ImageMetadata}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EditorToolContext { diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs index a1bb5d207..561633108 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs @@ -155,10 +155,7 @@ impl EditImageTool { } /// Validate that all referenced images exist in the context. - fn validate_context_images( - &self, - args: &EditImageToolArgs, - ) -> Result<(), EditImageError> { + fn validate_context_images(&self, args: &EditImageToolArgs) -> Result<(), EditImageError> { if !self.context.contains_image(&args.object_image_id) { return Err(EditImageError::AssetNotFound(args.object_image_id.clone())); } @@ -209,7 +206,7 @@ impl EditImageTool { source_resource_id: Option, canvas_completion: Option, ) -> Result { - // TODO id should not be assumed as src key + // TODO id should not be assumed as src key let source_image_src = args.object_image_id.id; let reference_image_srcs: Vec = args .reference_image_ids diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_background_music.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_background_music.rs index e69de29bb..a76a965c6 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_background_music.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_background_music.rs @@ -0,0 +1,130 @@ +use crate::editor_agent::editor_tools::generate_sound_effect::{ + map_media_response_error, parse_media_response, pending_message, +}; +use crate::http_error::AppError; +use crate::request_context::RequestContext; +use crate::state::AppState; +use crate::vector_engine_audio_generation::generate_editor_background_music_for_owner; +use axum::Json; +use axum::http::StatusCode; +use module_editor_agent::agent::tool::{Tool, ToolFailure}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use shared_contracts::assets::{ + EditorAudioGenerateResponse, EditorBackgroundMusicGenerateRequest, + EditorCanvasGenerationCompletionPayload, +}; +use std::error::Error; +use std::fmt::Display; + +pub struct GenerateBackgroundMusicTool; + +#[derive(Debug, Clone)] +pub enum GenerateBackgroundMusicError { + PromptNotProvided, +} +impl Display for GenerateBackgroundMusicError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "background music prompt not provided") + } +} +impl Error for GenerateBackgroundMusicError {} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateBackgroundMusicToolArgs { + pub prompt: String, + #[serde(default = "default_instrumental")] + pub make_instrumental: bool, +} +fn default_instrumental() -> bool { + true +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateBackgroundMusicToolOutput { + pub message: String, +} + +impl Tool for GenerateBackgroundMusicTool { + const NAME: &'static str = "generate-background-music"; + type Error = GenerateBackgroundMusicError; + type Args = GenerateBackgroundMusicToolArgs; + type Output = GenerateBackgroundMusicToolOutput; + fn description(&self) -> String { + "根据文字描述生成背景音乐。默认生成纯音乐，除非明确要求歌词或人声。".to_string() + } + fn parameters(&self) -> Value { + json!({ + "type": "object", "properties": { + "prompt": { "type": "string", "description": "音乐风格、情绪、乐器和节奏描述。" }, + "make_instrumental": { "type": "boolean", "description": "是否生成纯音乐，默认 true。" } + }, "required": ["prompt"], "additionalProperties": false + }) + } + fn call( + &self, + args: Self::Args, + ) -> impl Future> + Send { + async move { + if args.prompt.trim().is_empty() { + return Err(GenerateBackgroundMusicError::PromptNotProvided); + } + Ok(GenerateBackgroundMusicToolOutput { + message: pending_message(), + }) + } + } + fn classify_error(&self, error: &Self::Error) -> ToolFailure { + ToolFailure::invalid_args(error.to_string()) + } +} + +impl GenerateBackgroundMusicTool { + #[allow(clippy::too_many_arguments)] + pub async fn execute( + &self, + state: &AppState, + request_context: &RequestContext, + owner_user_id: String, + args: GenerateBackgroundMusicToolArgs, + project_id: String, + generation_inputs: Option, + asset_label: Option, + canvas_completion: EditorCanvasGenerationCompletionPayload, + ) -> Result { + if args.prompt.trim().is_empty() { + return Err(AppError::from_status(StatusCode::BAD_REQUEST).with_details( + json!({ "provider": "editor-agent", "message": "背景音乐提示词不能为空" }), + )); + } + let response = generate_editor_background_music_for_owner( + state.clone(), + request_context.clone(), + owner_user_id, + Ok(Json(EditorBackgroundMusicGenerateRequest { + gpt_description_prompt: args.prompt, + make_instrumental: args.make_instrumental, + project_id: Some(project_id), + canvas_completion: Some(canvas_completion), + generation_inputs, + asset_folder_id: Some("project".to_string()), + asset_label, + })), + ) + .await + .map_err(map_media_response_error)?; + parse_media_response(request_context, response.0) + } + pub fn format_execute_message( + &self, + args: &GenerateBackgroundMusicToolArgs, + result: &EditorAudioGenerateResponse, + ) -> String { + format!( + "[tool_call:{}] args: {} output: generated audio saved as: {}", + Self::NAME, + serde_json::to_string(args).unwrap_or_default(), + result.audio_src + ) + } +} diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_character.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_character.rs index e69de29bb..28f065500 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_character.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_character.rs @@ -0,0 +1,117 @@ +use crate::editor_agent::editor_tools::common::EditorToolContext; +use crate::editor_agent::editor_tools::generate_image::{ + EditorImageGenerationResult, GenerateImageError, GenerateImageTool, GenerateImageToolArgs, + GenerateImageToolOutput, +}; +use crate::editor_project::EditorGenerationCaller; +use crate::http_error::AppError; +use crate::request_context::RequestContext; +use crate::state::AppState; +use module_editor_agent::agent::tool::{Tool, ToolFailure}; +use serde_json::{Value, json}; +use shared_contracts::assets::EditorCanvasGenerationCompletionPayload; + +pub struct GenerateCharacterTool { + pub context: EditorToolContext, +} + +impl Tool for GenerateCharacterTool { + const NAME: &'static str = "generate-character"; + type Error = GenerateImageError; + type Args = GenerateImageToolArgs; + type Output = GenerateImageToolOutput; + + fn description(&self) -> String { + "生成一个新的角色形象、人物立绘或角色设定图。修改已有角色图片时请使用 edit-image。" + .to_string() + } + + fn parameters(&self) -> Value { + json!({ + "type": "object", + "properties": { + "prompt": { "type": "string", "description": "角色外貌、服装、姿势、画风和构图的完整描述。" }, + "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选角色或画风参考图 ID。" }, + "aspect_ratio": { "type": "string", "description": "可选比例，例如 2:3 或 9:16。" }, + "image_size": { "type": "string", "description": "可选清晰度，例如 1K 或 2K。" } + }, + "required": ["prompt"], + "additionalProperties": false + }) + } + + fn call( + &self, + args: Self::Args, + ) -> impl Future> + Send { + async move { + GenerateImageTool { + context: self.context.clone(), + } + .validate_args(&args)?; + Ok(GenerateImageToolOutput { message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string() }) + } + } + + fn classify_error(&self, error: &Self::Error) -> ToolFailure { + GenerateImageTool { + context: self.context.clone(), + } + .classify_error(error) + } +} + +impl GenerateCharacterTool { + #[allow(clippy::too_many_arguments)] + pub async fn execute( + &self, + state: &AppState, + request_context: &RequestContext, + caller: EditorGenerationCaller, + args: GenerateImageToolArgs, + project_id: String, + generation_inputs: Option, + asset_label: Option, + canvas_completion: EditorCanvasGenerationCompletionPayload, + ) -> Result { + GenerateImageTool { + context: self.context.clone(), + } + .execute_with_kind( + state, + request_context, + caller, + args, + project_id, + generation_inputs, + asset_label, + canvas_completion, + Some("character"), + "character", + ) + .await + } + + pub fn format_execute_message( + &self, + args: &GenerateImageToolArgs, + result: &EditorImageGenerationResult, + ) -> String { + format_tool_message(Self::NAME, args, result) + } +} + +pub(crate) fn format_tool_message( + tool_name: &str, + args: &GenerateImageToolArgs, + result: &EditorImageGenerationResult, +) -> String { + let args = serde_json::to_string(args).unwrap_or_default(); + let image_id = result + .object_key + .as_deref() + .unwrap_or(result.image_src.as_str()); + format!( + "[tool_call:{tool_name}] args: {args} output: generated result saved as image: {image_id}" + ) +} diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_icon_spritesheet.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_icon_spritesheet.rs index e69de29bb..b92f46324 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_icon_spritesheet.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_icon_spritesheet.rs @@ -0,0 +1,221 @@ +use crate::editor_agent::editor_tools::common::EditorToolContext; +use crate::editor_agent::utils::ImageId; +use crate::editor_project::{ + EditorGenerationCaller, EditorIconSpritesheetGenerationRequest, + generate_editor_icon_spritesheet_for_owner, +}; +use crate::http_error::AppError; +use crate::request_context::RequestContext; +use crate::state::AppState; +use axum::http::StatusCode; +use module_editor_agent::agent::tool::{Tool, ToolFailure, ToolFailureKind}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use shared_contracts::api::ApiSuccessEnvelope; +use shared_contracts::assets::EditorCanvasGenerationCompletionPayload; +use std::error::Error; +use std::fmt::Display; + +pub struct GenerateIconSpritesheetTool { + pub context: EditorToolContext, +} + +#[derive(Debug, Clone)] +pub enum GenerateIconSpritesheetError { + ReferenceNotProvided, + DescriptionsNotProvided, + AssetNotFound(ImageId), +} + +impl Display for GenerateIconSpritesheetError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::ReferenceNotProvided => write!(f, "reference image not provided"), + Self::DescriptionsNotProvided => write!(f, "icon descriptions not provided"), + Self::AssetNotFound(image_id) => write!(f, "asset {image_id} not found in context"), + } + } +} + +impl Error for GenerateIconSpritesheetError {} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateIconSpritesheetToolArgs { + pub reference_image_id: ImageId, + #[serde(default)] + pub reference_image_ids: Vec, + pub icon_descriptions: Vec, + #[serde(default)] + pub aspect_ratio: Option, + #[serde(default)] + pub image_size: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateIconSpritesheetToolOutput { + pub message: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EditorIconSpritesheetResult { + pub spritesheet_image_src: String, + pub spritesheet_width: u32, + pub spritesheet_height: u32, + pub task_id: String, + pub spritesheet_resource: Option, + pub spritesheet_asset: Option, + pub project: Option, +} + +impl Tool for GenerateIconSpritesheetTool { + const NAME: &'static str = "generate-icon-spritesheet"; + type Error = GenerateIconSpritesheetError; + type Args = GenerateIconSpritesheetToolArgs; + type Output = GenerateIconSpritesheetToolOutput; + + fn description(&self) -> String { + "根据一张图标规范参考图生成一组图标素材图集。必须提供参考图和多个图标描述。".to_string() + } + + fn parameters(&self) -> Value { + json!({ + "type": "object", + "properties": { + "reference_image_id": { "type": "string", "description": "必填的图标规范或风格参考图 ID。" }, + "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选的额外参考图 ID。" }, + "icon_descriptions": { "type": "array", "items": { "type": "string" }, "description": "要生成的多个图标描述。" }, + "aspect_ratio": { "type": "string", "description": "可选图集比例。" }, + "image_size": { "type": "string", "description": "可选清晰度，例如 1K 或 2K。" } + }, + "required": ["reference_image_id", "icon_descriptions"], + "additionalProperties": false + }) + } + + fn call( + &self, + args: Self::Args, + ) -> impl Future> + Send { + async move { + self.validate_args(&args)?; + Ok(GenerateIconSpritesheetToolOutput { + message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string(), + }) + } + } + + fn classify_error(&self, error: &Self::Error) -> ToolFailure { + match error { + GenerateIconSpritesheetError::AssetNotFound(_) => { + ToolFailure::new(ToolFailureKind::NotFound, error.to_string()) + } + GenerateIconSpritesheetError::ReferenceNotProvided + | GenerateIconSpritesheetError::DescriptionsNotProvided => { + ToolFailure::invalid_args(error.to_string()) + } + } + } +} + +impl GenerateIconSpritesheetTool { + fn validate_args( + &self, + args: &GenerateIconSpritesheetToolArgs, + ) -> Result<(), GenerateIconSpritesheetError> { + if args.reference_image_id.id.trim().is_empty() { + return Err(GenerateIconSpritesheetError::ReferenceNotProvided); + } + if !self.context.contains_image(&args.reference_image_id) { + return Err(GenerateIconSpritesheetError::AssetNotFound( + args.reference_image_id.clone(), + )); + } + if args + .icon_descriptions + .iter() + .all(|description| description.trim().is_empty()) + { + return Err(GenerateIconSpritesheetError::DescriptionsNotProvided); + } + for image_id in &args.reference_image_ids { + if !self.context.contains_image(image_id) { + return Err(GenerateIconSpritesheetError::AssetNotFound( + image_id.clone(), + )); + } + } + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + pub async fn execute( + &self, + state: &AppState, + request_context: &RequestContext, + caller: EditorGenerationCaller, + args: GenerateIconSpritesheetToolArgs, + project_id: String, + generation_inputs: Option, + canvas_completion: EditorCanvasGenerationCompletionPayload, + ) -> Result { + self.validate_args(&args).map_err(|error| { + AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ + "provider": "editor-agent", + "message": error.to_string(), + })) + })?; + let result = generate_editor_icon_spritesheet_for_owner( + state, + request_context, + caller, + EditorIconSpritesheetGenerationRequest { + reference_image_src: args.reference_image_id.id, + reference_image_srcs: Some( + args.reference_image_ids + .into_iter() + .map(|id| id.id) + .collect(), + ), + icon_descriptions: args.icon_descriptions, + model: None, + screen_color: Some("auto".to_string()), + seg_model: Some("birefnet".to_string()), + aspect_ratio: args.aspect_ratio, + image_size: args.image_size, + project_id: Some(project_id), + generation_inputs, + asset_folder_id: Some("project".to_string()), + canvas_completion: Some(canvas_completion), + }, + ) + .await? + .0; + let data = if request_context.wants_envelope() { + serde_json::from_value::>(result) + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details( + json!({ "message": format!("failed to parse success envelope: {error}") }), + ) + })? + .data + } else { + result + }; + serde_json::from_value(data).map_err(|error| AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_details(json!({ "message": format!("failed to deserialize icon spritesheet result: {error}") }))) + } + + pub fn format_execute_message( + &self, + args: &GenerateIconSpritesheetToolArgs, + result: &EditorIconSpritesheetResult, + ) -> String { + let args = serde_json::to_string(args).unwrap_or_default(); + format!( + "[tool_call:{}] args: {args} output: generated spritesheet saved as image: {}", + Self::NAME, + result.spritesheet_image_src + ) + } +} diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_image.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_image.rs index 7a0850923..c6d581df8 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_image.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_image.rs @@ -60,7 +60,8 @@ impl Tool for GenerateImageTool { type Output = GenerateImageToolOutput; fn description(&self) -> String { - "根据文字描述生成一张新图片；可选参考图仅用于借鉴画风、构图或元素，不会修改参考图本身。".to_string() + "根据文字描述生成一张新图片；可选参考图仅用于借鉴画风、构图或元素，不会修改参考图本身。" + .to_string() } fn parameters(&self) -> Value { @@ -132,7 +133,10 @@ pub struct EditorImageGenerationResult { } impl GenerateImageTool { - fn validate_args(&self, args: &GenerateImageToolArgs) -> Result<(), GenerateImageError> { + pub(crate) fn validate_args( + &self, + args: &GenerateImageToolArgs, + ) -> Result<(), GenerateImageError> { if args.prompt.trim().is_empty() { return Err(GenerateImageError::PromptNotProvided); } @@ -173,6 +177,35 @@ impl GenerateImageTool { generation_inputs: Option, asset_label: Option, canvas_completion: EditorCanvasGenerationCompletionPayload, + ) -> Result { + self.execute_with_kind( + state, + request_context, + caller, + args, + project_id, + generation_inputs, + asset_label, + canvas_completion, + None, + "editor_agent_generated_image", + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub(crate) async fn execute_with_kind( + &self, + state: &AppState, + request_context: &RequestContext, + caller: EditorGenerationCaller, + args: GenerateImageToolArgs, + project_id: String, + generation_inputs: Option, + asset_label: Option, + canvas_completion: EditorCanvasGenerationCompletionPayload, + kind: Option<&str>, + asset_kind: &str, ) -> Result { self.validate_args(&args).map_err(|error| { AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ @@ -188,10 +221,14 @@ impl GenerateImageTool { EditorImageGenerationRequest { prompt: args.prompt, size: None, - kind: None, + kind: kind.map(ToOwned::to_owned), model: None, - screen_color: None, - seg_model: None, + screen_color: kind + .is_some_and(|value| value == "character") + .then(|| "auto".to_string()), + seg_model: kind + .is_some_and(|value| value == "character") + .then(|| "birefnet".to_string()), aspect_ratio: args.aspect_ratio, image_size: args.image_size, reference_image_srcs: Some( @@ -201,7 +238,7 @@ impl GenerateImageTool { .collect(), ), project_id: Some(project_id), - asset_kind: Some("editor_agent_generated_image".to_string()), + asset_kind: Some(asset_kind.to_string()), generation_inputs, asset_folder_id: Some("project".to_string()), asset_label, @@ -241,7 +278,9 @@ mod tests { GenerateImageTool { context: EditorToolContext { images: HashMap::from([( - ImageId { id: "reference.png".to_string() }, + ImageId { + id: "reference.png".to_string(), + }, crate::editor_agent::utils::ImageMetadata { tag: String::new() }, )]), }, @@ -253,7 +292,9 @@ mod tests { let output = tool() .call(GenerateImageToolArgs { prompt: "雨夜的霓虹街道".to_string(), - reference_image_ids: vec![ImageId { id: "reference.png".to_string() }], + reference_image_ids: vec![ImageId { + id: "reference.png".to_string(), + }], aspect_ratio: Some("16:9".to_string()), image_size: Some("2K".to_string()), }) @@ -274,17 +315,25 @@ mod tests { }) .await .expect_err("blank prompt must fail"); - assert!(matches!(prompt_error, GenerateImageError::PromptNotProvided)); + assert!(matches!( + prompt_error, + GenerateImageError::PromptNotProvided + )); let reference_error = tool() .call(GenerateImageToolArgs { prompt: "一棵树".to_string(), - reference_image_ids: vec![ImageId { id: "missing.png".to_string() }], + reference_image_ids: vec![ImageId { + id: "missing.png".to_string(), + }], aspect_ratio: None, image_size: None, }) .await .expect_err("unknown reference must fail"); - assert!(matches!(reference_error, GenerateImageError::AssetNotFound(_))); + assert!(matches!( + reference_error, + GenerateImageError::AssetNotFound(_) + )); } } diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_sound_effect.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_sound_effect.rs index e69de29bb..895672c40 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_sound_effect.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_sound_effect.rs @@ -0,0 +1,173 @@ +use crate::http_error::AppError; +use crate::request_context::RequestContext; +use crate::state::AppState; +use crate::vector_engine_audio_generation::generate_editor_sound_effect_for_owner; +use axum::Json; +use axum::http::StatusCode; +use axum::response::Response; +use module_editor_agent::agent::tool::{Tool, ToolFailure}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use shared_contracts::api::ApiSuccessEnvelope; +use shared_contracts::assets::{ + EditorAudioGenerateResponse, EditorCanvasGenerationCompletionPayload, + EditorSoundEffectGenerateRequest, +}; +use std::error::Error; +use std::fmt::Display; + +pub struct GenerateSoundEffectTool; + +#[derive(Debug, Clone)] +pub enum GenerateSoundEffectError { + PromptNotProvided, +} + +impl Display for GenerateSoundEffectError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "sound effect prompt not provided") + } +} + +impl Error for GenerateSoundEffectError {} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateSoundEffectToolArgs { + pub prompt: String, + #[serde(default)] + pub duration: Option, + #[serde(default)] + pub model: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateSoundEffectToolOutput { + pub message: String, +} + +impl Tool for GenerateSoundEffectTool { + const NAME: &'static str = "generate-sound-effect"; + type Error = GenerateSoundEffectError; + type Args = GenerateSoundEffectToolArgs; + type Output = GenerateSoundEffectToolOutput; + + fn description(&self) -> String { + "根据文字描述生成一段音效，例如脚步声、按钮点击声或环境音。".to_string() + } + + fn parameters(&self) -> Value { + json!({ + "type": "object", + "properties": { + "prompt": { "type": "string", "description": "音效内容、材质、节奏和情绪描述。" }, + "duration": { "type": "integer", "description": "可选时长（秒）。" }, + "model": { "type": "string", "description": "可选音效模型。" } + }, + "required": ["prompt"], + "additionalProperties": false + }) + } + + fn call( + &self, + args: Self::Args, + ) -> impl Future> + Send { + async move { + if args.prompt.trim().is_empty() { + return Err(GenerateSoundEffectError::PromptNotProvided); + } + Ok(GenerateSoundEffectToolOutput { + message: pending_message(), + }) + } + } + + fn classify_error(&self, error: &Self::Error) -> ToolFailure { + ToolFailure::invalid_args(error.to_string()) + } +} + +impl GenerateSoundEffectTool { + #[allow(clippy::too_many_arguments)] + pub async fn execute( + &self, + state: &AppState, + request_context: &RequestContext, + owner_user_id: String, + args: GenerateSoundEffectToolArgs, + project_id: String, + generation_inputs: Option, + asset_label: Option, + canvas_completion: EditorCanvasGenerationCompletionPayload, + ) -> Result { + if args.prompt.trim().is_empty() { + return Err(AppError::from_status(StatusCode::BAD_REQUEST).with_details( + json!({ "provider": "editor-agent", "message": "音效提示词不能为空" }), + )); + } + let response = generate_editor_sound_effect_for_owner( + state.clone(), + request_context.clone(), + owner_user_id, + Ok(Json(EditorSoundEffectGenerateRequest { + prompt: args.prompt, + model: args.model, + duration: args.duration.unwrap_or(3), + project_id: Some(project_id), + canvas_completion: Some(canvas_completion), + generation_inputs, + asset_folder_id: Some("project".to_string()), + asset_label, + })), + ) + .await + .map_err(map_media_response_error)?; + parse_media_response(request_context, response.0) + } + + pub fn format_execute_message( + &self, + args: &GenerateSoundEffectToolArgs, + result: &EditorAudioGenerateResponse, + ) -> String { + format!( + "[tool_call:{}] args: {} output: generated audio saved as: {}", + Self::NAME, + serde_json::to_string(args).unwrap_or_default(), + result.audio_src + ) + } +} + +pub(crate) fn pending_message() -> String { + "this tool call is pending user confirmation. if all is pending, just end this turn".to_string() +} + +pub(crate) fn map_media_response_error(response: Response) -> AppError { + AppError::from_status(response.status()).with_details(json!({ + "provider": "editor-agent", + "message": "媒体生成请求失败", + })) +} + +pub(crate) fn parse_media_response Deserialize<'de>>( + request_context: &RequestContext, + value: Value, +) -> Result { + let data = if request_context.wants_envelope() { + serde_json::from_value::>(value) + .map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details( + json!({ "message": format!("failed to parse success envelope: {error}") }), + ) + })? + .data + } else { + value + }; + serde_json::from_value(data).map_err(|error| { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details( + json!({ "message": format!("failed to deserialize media generation result: {error}") }), + ) + }) +} diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_ui_design.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_ui_design.rs index e69de29bb..7b3900200 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_ui_design.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_ui_design.rs @@ -0,0 +1,102 @@ +use crate::editor_agent::editor_tools::common::EditorToolContext; +use crate::editor_agent::editor_tools::generate_character::format_tool_message; +use crate::editor_agent::editor_tools::generate_image::{ + EditorImageGenerationResult, GenerateImageError, GenerateImageTool, GenerateImageToolArgs, + GenerateImageToolOutput, +}; +use crate::editor_project::EditorGenerationCaller; +use crate::http_error::AppError; +use crate::request_context::RequestContext; +use crate::state::AppState; +use module_editor_agent::agent::tool::{Tool, ToolFailure}; +use serde_json::{Value, json}; +use shared_contracts::assets::EditorCanvasGenerationCompletionPayload; + +pub struct GenerateUiDesignTool { + pub context: EditorToolContext, +} + +impl Tool for GenerateUiDesignTool { + const NAME: &'static str = "generate-ui-design"; + type Error = GenerateImageError; + type Args = GenerateImageToolArgs; + type Output = GenerateImageToolOutput; + + fn description(&self) -> String { + "生成一张完整 UI 设计图或界面稿，适用于 HUD、弹窗、面板、按钮组合和整页界面。".to_string() + } + + fn parameters(&self) -> Value { + json!({ + "type": "object", + "properties": { + "prompt": { "type": "string", "description": "完整 UI 画面、信息层级、视觉风格和构图描述。" }, + "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选 UI 风格或布局参考图 ID。" }, + "aspect_ratio": { "type": "string", "description": "可选画面比例。" }, + "image_size": { "type": "string", "description": "可选清晰度，例如 1K 或 2K。" } + }, + "required": ["prompt"], + "additionalProperties": false + }) + } + + fn call( + &self, + args: Self::Args, + ) -> impl Future> + Send { + async move { + GenerateImageTool { + context: self.context.clone(), + } + .validate_args(&args)?; + Ok(GenerateImageToolOutput { message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string() }) + } + } + + fn classify_error(&self, error: &Self::Error) -> ToolFailure { + GenerateImageTool { + context: self.context.clone(), + } + .classify_error(error) + } +} + +impl GenerateUiDesignTool { + #[allow(clippy::too_many_arguments)] + pub async fn execute( + &self, + state: &AppState, + request_context: &RequestContext, + caller: EditorGenerationCaller, + args: GenerateImageToolArgs, + project_id: String, + generation_inputs: Option, + asset_label: Option, + canvas_completion: EditorCanvasGenerationCompletionPayload, + ) -> Result { + GenerateImageTool { + context: self.context.clone(), + } + .execute_with_kind( + state, + request_context, + caller, + args, + project_id, + generation_inputs, + asset_label, + canvas_completion, + Some("ui-design"), + "ui-design", + ) + .await + } + + pub fn format_execute_message( + &self, + args: &GenerateImageToolArgs, + result: &EditorImageGenerationResult, + ) -> String { + format_tool_message(Self::NAME, args, result) + } +} diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs index e69de29bb..fe93647c6 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs @@ -0,0 +1,174 @@ +use crate::character_animation_assets::generate_editor_video_for_owner; +use crate::editor_agent::editor_tools::generate_sound_effect::{ + map_media_response_error, parse_media_response, pending_message, +}; +use crate::editor_agent::utils::ImageId; +use crate::http_error::AppError; +use crate::request_context::RequestContext; +use crate::state::AppState; +use axum::Json; +use axum::http::StatusCode; +use module_editor_agent::agent::tool::{Tool, ToolFailure, ToolFailureKind}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use shared_contracts::assets::{ + EditorCanvasGenerationCompletionPayload, EditorVideoGenerateRequest, + EditorVideoGenerateResponse, +}; +use std::error::Error; +use std::fmt::Display; + +pub struct GenerateVideoTool { + pub context: crate::editor_agent::editor_tools::common::EditorToolContext, +} + +#[derive(Debug, Clone)] +pub enum GenerateVideoError { + PromptNotProvided, + AssetNotFound(ImageId), +} +impl Display for GenerateVideoError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::PromptNotProvided => write!(f, "video prompt not provided"), + Self::AssetNotFound(image_id) => write!(f, "asset {image_id} not found in context"), + } + } +} +impl Error for GenerateVideoError {} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateVideoToolArgs { + pub prompt: String, + #[serde(default)] + pub reference_image_ids: Vec, + #[serde(default)] + pub aspect_ratio: Option, + #[serde(default)] + pub duration_seconds: Option, + #[serde(default)] + pub model: Option, + #[serde(default)] + pub resolution: Option, + #[serde(default)] + pub sound: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateVideoToolOutput { + pub message: String, +} + +impl Tool for GenerateVideoTool { + const NAME: &'static str = "generate-video"; + type Error = GenerateVideoError; + type Args = GenerateVideoToolArgs; + type Output = GenerateVideoToolOutput; + fn description(&self) -> String { + "根据文字描述生成一段短视频；可选图片参考图用于镜头风格或主体参考。".to_string() + } + fn parameters(&self) -> Value { + json!({ + "type": "object", "properties": { + "prompt": { "type": "string", "description": "镜头、主体动作、场景和风格描述。" }, + "reference_image_ids": { "type": "array", "items": { "type": "string" }, "description": "可选图片参考图 ID。" }, + "aspect_ratio": { "type": "string", "description": "可选比例，默认 16:9。" }, + "duration_seconds": { "type": "integer", "description": "可选时长，默认 4 秒。" }, + "model": { "type": "string", "description": "可选视频模型。" }, + "resolution": { "type": "string", "description": "可选清晰度，默认 720p。" }, + "sound": { "type": "string", "description": "是否生成声音，默认 off。" } + }, "required": ["prompt"], "additionalProperties": false + }) + } + fn call( + &self, + args: Self::Args, + ) -> impl Future> + Send { + async move { + if args.prompt.trim().is_empty() { + return Err(GenerateVideoError::PromptNotProvided); + } + for id in &args.reference_image_ids { + if !self.context.contains_image(id) { + return Err(GenerateVideoError::AssetNotFound(id.clone())); + } + } + Ok(GenerateVideoToolOutput { + message: pending_message(), + }) + } + } + fn classify_error(&self, error: &Self::Error) -> ToolFailure { + match error { + GenerateVideoError::AssetNotFound(_) => { + ToolFailure::new(ToolFailureKind::NotFound, error.to_string()) + } + GenerateVideoError::PromptNotProvided => ToolFailure::invalid_args(error.to_string()), + } + } +} + +impl GenerateVideoTool { + #[allow(clippy::too_many_arguments)] + pub async fn execute( + &self, + state: &AppState, + request_context: &RequestContext, + owner_user_id: String, + args: GenerateVideoToolArgs, + project_id: String, + generation_inputs: Option, + asset_label: Option, + canvas_completion: EditorCanvasGenerationCompletionPayload, + ) -> Result { + if args.prompt.trim().is_empty() { + return Err(AppError::from_status(StatusCode::BAD_REQUEST).with_details( + json!({ "provider": "editor-agent", "message": "视频提示词不能为空" }), + )); + } + let reference_image_srcs = args + .reference_image_ids + .into_iter() + .map(|id| id.id) + .collect(); + let response = generate_editor_video_for_owner( + state.clone(), + request_context.clone(), + owner_user_id, + Ok(Json(EditorVideoGenerateRequest { + prompt: args.prompt, + model: args.model.unwrap_or_else(|| "seedance2.0-fast".to_string()), + aspect_ratio: args.aspect_ratio.unwrap_or_else(|| "16:9".to_string()), + duration_seconds: args.duration_seconds.unwrap_or(4), + resolution: args.resolution.unwrap_or_else(|| "720p".to_string()), + mode: "std".to_string(), + sound: args.sound.unwrap_or_else(|| "off".to_string()), + web_search_enabled: false, + reference_image_srcs, + reference_video_srcs: Vec::new(), + reference_audio_srcs: Vec::new(), + project_id: Some(project_id), + canvas_completion: Some(canvas_completion), + generation_inputs, + source_resource_id: None, + asset_kind: Some("video".to_string()), + asset_folder_id: Some("project".to_string()), + asset_label, + })), + ) + .await + .map_err(map_media_response_error)?; + parse_media_response(request_context, response.0) + } + pub fn format_execute_message( + &self, + args: &GenerateVideoToolArgs, + result: &EditorVideoGenerateResponse, + ) -> String { + format!( + "[tool_call:{}] args: {} output: generated video saved as: {}", + Self::NAME, + serde_json::to_string(args).unwrap_or_default(), + result.video_src + ) + } +} diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/mod.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/mod.rs index 695076375..add644b07 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/mod.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/mod.rs @@ -1,3 +1,9 @@ pub mod common; pub mod edit_image; +pub mod generate_background_music; +pub mod generate_character; +pub mod generate_icon_spritesheet; pub mod generate_image; +pub mod generate_sound_effect; +pub mod generate_ui_design; +pub mod generate_video; diff --git a/server-rs/crates/api-server/src/editor_agent/utils.rs b/server-rs/crates/api-server/src/editor_agent/utils.rs index 6e2a9bd7b..e2a633f54 100644 --- a/server-rs/crates/api-server/src/editor_agent/utils.rs +++ b/server-rs/crates/api-server/src/editor_agent/utils.rs @@ -3,9 +3,11 @@ use crate::http_error::AppError; use crate::platform_errors::map_oss_error; use crate::state::AppState; use axum::http::StatusCode; -use platform_oss::{LegacyAssetPrefix, OssObjectAccess, OssPutObjectRequest, OssSignedGetObjectUrlRequest}; +use platform_oss::{ + LegacyAssetPrefix, OssObjectAccess, OssPutObjectRequest, OssSignedGetObjectUrlRequest, +}; use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; +use serde_json::{Value, json}; use shared_contracts::assets::{ EditorCanvasGenerationCompletionPayload, EditorCanvasGenerationPlaceholderPayload, }; @@ -271,70 +273,69 @@ pub async fn write_messages_document( const EDITOR_AGENT_MESSAGES_READ_EXPIRE_SECONDS: u64 = 60; pub async fn read_messages_document( - state: &AppState, - conversation: &EditorAgentConversationRecord, + state: &AppState, + conversation: &EditorAgentConversationRecord, ) -> Result { - let oss_client = state - .oss_client() - .ok_or_else(editor_agent_oss_unavailable)?; - let signed = oss_client - .sign_internal_get_object_url(OssSignedGetObjectUrlRequest { - object_key: conversation.messages_object_key.clone(), - expire_seconds: Some(EDITOR_AGENT_MESSAGES_READ_EXPIRE_SECONDS), - }) - .map_err(|error| map_oss_error(error, "aliyun-oss"))?; - let response = reqwest::Client::new() - .get(signed.signed_url.as_str()) - .send() - .await - .map_err(|error| editor_agent_oss_read_error(error.to_string()))?; - if response.status() == StatusCode::NOT_FOUND { - return Ok(empty_messages_document( - conversation.conversation_id.as_str(), - )); - } - if !response.status().is_success() { - return Err(editor_agent_oss_read_error(format!( - "OSS returned non-success status {}", - response.status().as_u16() - ))); - } - if response - .content_length() - .is_some_and(|size| size > EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES as u64) - { - return Err(editor_agent_messages_document_too_large()); - } - let bytes = response - .bytes() - .await - .map_err(|error| editor_agent_oss_read_error(error.to_string()))?; - if bytes.is_empty() { - return Ok(empty_messages_document( - conversation.conversation_id.as_str(), - )); - } - if bytes.len() > EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES { - return Err(editor_agent_messages_document_too_large()); - } - let document: EditorAgentConversationMessagesDocument = serde_json::from_slice(&bytes) - .map_err(|error| { - editor_agent_oss_read_error(format!("message document JSON invalid: {error}")) - })?; - if document.conversation_id != conversation.conversation_id { - return Err( - AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ + let oss_client = state + .oss_client() + .ok_or_else(editor_agent_oss_unavailable)?; + let signed = oss_client + .sign_internal_get_object_url(OssSignedGetObjectUrlRequest { + object_key: conversation.messages_object_key.clone(), + expire_seconds: Some(EDITOR_AGENT_MESSAGES_READ_EXPIRE_SECONDS), + }) + .map_err(|error| map_oss_error(error, "aliyun-oss"))?; + let response = reqwest::Client::new() + .get(signed.signed_url.as_str()) + .send() + .await + .map_err(|error| editor_agent_oss_read_error(error.to_string()))?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(empty_messages_document( + conversation.conversation_id.as_str(), + )); + } + if !response.status().is_success() { + return Err(editor_agent_oss_read_error(format!( + "OSS returned non-success status {}", + response.status().as_u16() + ))); + } + if response + .content_length() + .is_some_and(|size| size > EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES as u64) + { + return Err(editor_agent_messages_document_too_large()); + } + let bytes = response + .bytes() + .await + .map_err(|error| editor_agent_oss_read_error(error.to_string()))?; + if bytes.is_empty() { + return Ok(empty_messages_document( + conversation.conversation_id.as_str(), + )); + } + if bytes.len() > EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES { + return Err(editor_agent_messages_document_too_large()); + } + let document: EditorAgentConversationMessagesDocument = serde_json::from_slice(&bytes) + .map_err(|error| { + editor_agent_oss_read_error(format!("message document JSON invalid: {error}")) + })?; + if document.conversation_id != conversation.conversation_id { + return Err( + AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ "provider": "editor-agent", "message": "message document conversationId does not match metadata", "conversationId": conversation.conversation_id, "documentConversationId": document.conversation_id, })), - ); - } - Ok(document) + ); + } + Ok(document) } - pub async fn ensure_editor_project_access( state: &AppState, project_id: &str, @@ -571,7 +572,7 @@ pub fn build_editor_agent_canvas_completion( fn editor_agent_tool_display_size(tool_name: &str) -> (f64, f64) { match tool_name { - "ui-design" => (640.0, 360.0), + "ui-design" | "generate-ui-design" => (640.0, 360.0), "generate-character" => (512.0, 768.0), _ => (512.0, 512.0), } @@ -579,7 +580,11 @@ fn editor_agent_tool_display_size(tool_name: &str) -> (f64, f64) { const EDITOR_AGENT_CANVAS_RESULT_GAP: f64 = 32.0; -fn next_editor_agent_canvas_position(layers: serde_json::Value, _width: f64, _height: f64) -> (f64, f64) { +fn next_editor_agent_canvas_position( + layers: serde_json::Value, + _width: f64, + _height: f64, +) -> (f64, f64) { let mut max_right: Option = None; let mut min_y: Option = None; if let Value::Array(items) = layers {