diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index fd322e744..55b27d765 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use axum::extract::{Path, State}; use axum::{Extension, Json}; use module_editor_agent::agent::agent_builder::AgentBuilder; @@ -16,16 +14,15 @@ use serde::Serialize; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use shared_contracts::assets::{ - EditorBackgroundMusicGenerateRequest, EditorSoundEffectGenerateRequest, - EditorVideoGenerateRequest, + EditorAudioGenerateResponse, EditorBackgroundMusicGenerateRequest, + EditorSoundEffectGenerateRequest, EditorVideoGenerateRequest, EditorVideoGenerateResponse, }; use shared_contracts::editor_agent::{ CreateEditorAgentConversationRequest, EditorAgentConversationListResponse, EditorAgentConversationMessagesDocument, EditorAgentConversationResponse, EditorAgentConversationSummary, EditorAgentMessage, EditorAgentMessageRequest, EditorAgentMessageResponse, EditorAgentMessageRole, EditorAgentToolCall, - EditorAgentToolCallDisplayArgs, EditorAgentToolCallDisplayExtras, EditorAgentToolCallImageArg, - EditorAgentToolCallImageRef, EditorAgentToolCallStringArg, + EditorAgentToolCallStatus, }; use spacetime_client::{ EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput, @@ -37,7 +34,9 @@ use crate::api_response::json_success_body; use crate::auth::AuthenticatedAccessToken; use crate::editor_agent::agent::LlmChatAgentBuilder; use crate::editor_agent::editor_tools::common::{EditorAgentPricedTool, EditorToolContext}; -use crate::editor_agent::editor_tools::edit_image::{EditImageTool, EditImageToolArgs}; +use crate::editor_agent::editor_tools::edit_image::{ + EditImageTool, EditImageToolArgs, +}; use crate::editor_agent::editor_tools::generate_background_music::{ GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, }; @@ -45,19 +44,21 @@ use crate::editor_agent::editor_tools::generate_character::GenerateCharacterTool use crate::editor_agent::editor_tools::generate_icon_spritesheet::{ GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, }; -use crate::editor_agent::editor_tools::generate_image::{GenerateImageTool, GenerateImageToolArgs}; +use crate::editor_agent::editor_tools::generate_image::{ + GenerateImageTool, GenerateImageToolArgs, +}; use crate::editor_agent::editor_tools::generate_sound_effect::{ GenerateSoundEffectTool, GenerateSoundEffectToolArgs, }; use crate::editor_agent::editor_tools::generate_ui_design::GenerateUiDesignTool; use crate::editor_agent::editor_tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; use crate::editor_agent::utils::{ - ImageId, ImageMetadata, IntoDataKey, IntoImageId, build_editor_agent_canvas_completion, - conversation_detail_from_record, conversation_summary_from_record, editor_agent_bad_request, - empty_messages_document, ensure_editor_project_access, normalize_editor_agent_attachments, - now_rfc3339, read_messages_document, require_editor_agent_sidebar_enabled, - write_messages_document, + IntoImageId, build_editor_agent_canvas_completion, conversation_detail_from_record, + conversation_summary_from_record, editor_agent_bad_request, empty_messages_document, + ensure_editor_project_access, normalize_editor_agent_attachments, now_rfc3339, + read_messages_document, require_editor_agent_sidebar_enabled, write_messages_document, }; +use crate::editor_agent::{context, display_args}; use crate::editor_generation_config::EditorGenerationPricingConfig; use crate::editor_generation_queue::{ EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND, @@ -104,7 +105,6 @@ pub async fn editor_agent_message( let mut document: EditorAgentConversationMessagesDocument = read_messages_document(&state, &conversation).await?; - // Determine initialization before attachment bookkeeping adds a system message. let was_empty = document.messages.is_empty(); let now = now_rfc3339(); @@ -162,7 +162,7 @@ pub async fn editor_agent_message( let conversation_summary = conversation_summary_from_record(updated_conversation); // Build tool context from document - let tool_context = build_tool_context(&document); + let tool_context = context::build_tool_context(&document); // Build and run agent let llm_client = state.llm_client().ok_or_else(|| { @@ -220,10 +220,10 @@ pub async fn editor_agent_message( ) { Err(err) => Ok(Json(EditorAgentMessageResponse { conversation: conversation_summary, - delta_messages: vec![], + delta_messages: Vec::new(), error_message: Some(err.to_string()), })), - Ok(delta_messages) => { + Ok(mut delta_messages) => { for msg in &delta_messages { document.messages.push(msg.clone()); } @@ -269,7 +269,7 @@ fn build_delta_messages( }); } PromptOutput::Tool(tco) => { - let display_args = build_tool_call_display_args( + let display_args = display_args::build_tool_call_display_args( tco.tool_call.name.as_str(), &tco.tool_call.args, document, @@ -283,10 +283,10 @@ fn build_delta_messages( attachments: Vec::new(), tool_call: Some(EditorAgentToolCall { tool_name: tco.tool_call.name, + status: EditorAgentToolCallStatus::Pending, args: tco.tool_call.args, display_args, external_job_id: None, - cancelled_at: None, images: Vec::new(), videos: Vec::new(), audios: Vec::new(), @@ -301,467 +301,6 @@ fn build_delta_messages( Ok(messages) } -fn build_tool_call_display_args( - tool_name: &str, - args: &Value, - document: &EditorAgentConversationMessagesDocument, - tool_context: &EditorToolContext, - pricing: &EditorGenerationPricingConfig, -) -> Result { - let mut display_args = EditorAgentToolCallDisplayArgs::default(); - - let price_mud_points = match tool_name { - EditImageTool::NAME => { - let args: EditImageToolArgs = parse_display_tool_args(tool_name, args)?; - let price_mud_points = EditImageTool { - context: tool_context.clone(), - } - .pricing(pricing, &args); - push_string_display_arg(&mut display_args, "prompt", "修改要求", args.prompt); - push_image_display_arg( - &mut display_args, - document, - "object_image_id", - "目标图片", - std::slice::from_ref(&args.object_image_id), - )?; - push_image_display_arg( - &mut display_args, - document, - "reference_image_ids", - "参考图片", - args.reference_image_ids.as_slice(), - )?; - price_mud_points - } - GenerateImageTool::NAME | GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => { - let args: GenerateImageToolArgs = parse_display_tool_args(tool_name, args)?; - let price_mud_points = match tool_name { - GenerateImageTool::NAME => GenerateImageTool { - context: tool_context.clone(), - } - .pricing(pricing, &args), - GenerateCharacterTool::NAME => GenerateCharacterTool { - context: tool_context.clone(), - } - .pricing(pricing, &args), - GenerateUiDesignTool::NAME => GenerateUiDesignTool { - context: tool_context.clone(), - } - .pricing(pricing, &args), - _ => unreachable!("tool name was matched above"), - }; - push_string_display_arg(&mut display_args, "prompt", "提示词", args.prompt); - push_optional_string_display_arg( - &mut display_args, - "aspect_ratio", - "画面比例", - args.aspect_ratio, - ); - push_optional_string_display_arg( - &mut display_args, - "image_size", - "图片尺寸", - args.image_size, - ); - push_image_display_arg( - &mut display_args, - document, - "reference_image_ids", - "参考图片", - args.reference_image_ids.as_slice(), - )?; - price_mud_points - } - GenerateIconSpritesheetTool::NAME => { - let args: GenerateIconSpritesheetToolArgs = parse_display_tool_args(tool_name, args)?; - let price_mud_points = GenerateIconSpritesheetTool { - context: tool_context.clone(), - } - .pricing(pricing, &args); - push_string_display_arg( - &mut display_args, - "icon_descriptions", - "图标描述", - args.icon_descriptions.join("\n"), - ); - push_optional_string_display_arg( - &mut display_args, - "aspect_ratio", - "图集比例", - args.aspect_ratio, - ); - push_optional_string_display_arg( - &mut display_args, - "image_size", - "图片尺寸", - args.image_size, - ); - push_image_display_arg( - &mut display_args, - document, - "reference_image_id", - "规范参考图", - std::slice::from_ref(&args.reference_image_id), - )?; - push_image_display_arg( - &mut display_args, - document, - "reference_image_ids", - "额外参考图", - args.reference_image_ids.as_slice(), - )?; - price_mud_points - } - GenerateVideoTool::NAME => { - let args: GenerateVideoToolArgs = parse_display_tool_args(tool_name, args)?; - let price_mud_points = GenerateVideoTool { - context: tool_context.clone(), - } - .pricing(pricing, &args); - push_string_display_arg(&mut display_args, "prompt", "视频描述", args.prompt); - push_optional_string_display_arg( - &mut display_args, - "aspect_ratio", - "画面比例", - args.aspect_ratio, - ); - if let Some(duration_seconds) = args.duration_seconds { - push_string_display_arg( - &mut display_args, - "duration_seconds", - "时长（秒）", - duration_seconds.to_string(), - ); - } - push_optional_string_display_arg(&mut display_args, "model", "模型", args.model); - push_optional_string_display_arg( - &mut display_args, - "resolution", - "分辨率", - args.resolution, - ); - push_optional_string_display_arg(&mut display_args, "sound", "声音", args.sound); - push_image_display_arg( - &mut display_args, - document, - "reference_image_ids", - "参考图片", - args.reference_image_ids.as_slice(), - )?; - price_mud_points - } - GenerateSoundEffectTool::NAME => { - let args: GenerateSoundEffectToolArgs = parse_display_tool_args(tool_name, args)?; - let price_mud_points = GenerateSoundEffectTool.pricing(pricing, &args); - push_string_display_arg(&mut display_args, "prompt", "音效描述", args.prompt); - if let Some(duration) = args.duration { - push_string_display_arg( - &mut display_args, - "duration", - "时长（秒）", - duration.to_string(), - ); - } - push_optional_string_display_arg(&mut display_args, "model", "模型", args.model); - price_mud_points - } - GenerateBackgroundMusicTool::NAME => { - let args: GenerateBackgroundMusicToolArgs = parse_display_tool_args(tool_name, args)?; - let price_mud_points = GenerateBackgroundMusicTool.pricing(pricing, &args); - push_string_display_arg(&mut display_args, "prompt", "音乐描述", args.prompt); - push_string_display_arg( - &mut display_args, - "make_instrumental", - "纯音乐", - if args.make_instrumental { "是" } else { "否" }, - ); - price_mud_points - } - _ => { - return Err(PromptError::InternalError(format!( - "missing display argument mapping for tool {tool_name}" - ))); - } - }; - - display_args.extras = EditorAgentToolCallDisplayExtras { price_mud_points }; - - Ok(display_args) -} - -fn parse_display_tool_args(tool_name: &str, args: &Value) -> Result -where - T: serde::de::DeserializeOwned, -{ - serde_json::from_value(args.clone()).map_err(|error| { - PromptError::InternalError(format!( - "failed to build display arguments for {tool_name}: {error}" - )) - }) -} - -fn push_string_display_arg( - display_args: &mut EditorAgentToolCallDisplayArgs, - name: &str, - label: &str, - value: impl Into, -) { - let value = value.into(); - if value.trim().is_empty() { - return; - } - display_args.string_args.push(EditorAgentToolCallStringArg { - name: name.to_string(), - label: label.to_string(), - value, - }); -} - -fn push_optional_string_display_arg( - display_args: &mut EditorAgentToolCallDisplayArgs, - name: &str, - label: &str, - value: Option, -) { - if let Some(value) = value { - push_string_display_arg(display_args, name, label, value); - } -} - -fn push_image_display_arg( - display_args: &mut EditorAgentToolCallDisplayArgs, - document: &EditorAgentConversationMessagesDocument, - name: &str, - label: &str, - image_ids: &[ImageId], -) -> Result<(), PromptError> { - if image_ids.is_empty() { - return Ok(()); - } - - let refs = image_ids - .iter() - .map(|image_id| { - resolve_tool_call_image_ref(document, image_id).ok_or_else(|| { - PromptError::InternalError(format!( - "display image reference {} is missing from conversation context", - image_id.id - )) - }) - }) - .collect::, _>>()?; - - display_args.image_args.push(EditorAgentToolCallImageArg { - name: name.to_string(), - label: label.to_string(), - refs, - }); - Ok(()) -} - -fn resolve_tool_call_image_ref( - document: &EditorAgentConversationMessagesDocument, - image_id: &ImageId, -) -> Option { - for message in document.messages.iter().rev() { - for attachment in message.attachments.iter().rev() { - if attachment.clone().into_image_id() == *image_id { - return Some(EditorAgentToolCallImageRef { - image_id: image_id.id.clone(), - image_src: attachment.image_src.clone(), - object_key: attachment.object_key.clone(), - thumbnail_src: attachment.thumbnail_src.clone(), - label: attachment.label.clone(), - width: attachment.width, - height: attachment.height, - }); - } - } - - if let Some(tool_call) = &message.tool_call { - for image in tool_call.images.iter().rev() { - let data_key = image.clone().into_data_key(); - if ImageId::from_data_key(data_key) == *image_id { - return Some(EditorAgentToolCallImageRef { - image_id: image_id.id.clone(), - image_src: image.image_src.clone(), - object_key: image.object_key.clone(), - thumbnail_src: image.thumbnail_src.clone(), - label: None, - width: image.width, - height: image.height, - }); - } - } - } - } - - None -} - -fn build_tool_context(document: &EditorAgentConversationMessagesDocument) -> EditorToolContext { - let mut images: HashMap = HashMap::new(); - - for msg in document.messages.iter().rev() { - if let Some(tc) = &msg.tool_call { - for img in &tc.images { - let data_key = img.clone().into_data_key(); - let image_id = ImageId::from_data_key(&data_key); - let metadata = ImageMetadata { - tag: String::new(), - data_key, - }; - images.insert(image_id, metadata); - } - } - for a in &msg.attachments { - let data_key = a.clone().into_data_key(); - let image_id = ImageId::from_data_key(&data_key); - let metadata = ImageMetadata { - tag: String::new(), - data_key, - }; - images.insert(image_id, metadata); - } - } - - EditorToolContext { images } -} - -#[cfg(test)] -mod tool_context_tests { - use super::*; - use crate::editor_generation_config::load_editor_generation_pricing_from_paths; - use shared_contracts::editor_agent::{ - EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, EditorAgentAttachmentRef, - EditorAgentAttachmentSource, EditorAgentGeneratedImage, EditorAgentMessageRole, - }; - - #[test] - fn tool_context_maps_hashed_ids_back_to_real_data_keys() { - let data_key = "generated-images/proj-1/output.png"; - let document = EditorAgentConversationMessagesDocument { - version: EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, - conversation_id: "editor-agent-conversation-1".to_string(), - messages: vec![EditorAgentMessage { - id: 0, - role: EditorAgentMessageRole::System, - text: String::new(), - attachments: Vec::new(), - tool_call: Some(EditorAgentToolCall { - tool_name: "generate-image".to_string(), - args: json!({}), - display_args: EditorAgentToolCallDisplayArgs::default(), - external_job_id: Some("task-existing".to_string()), - cancelled_at: None, - images: vec![EditorAgentGeneratedImage { - resource_id: None, - object_key: Some(data_key.to_string()), - asset_object_id: None, - image_src: "/generated-images/proj-1/output.png".to_string(), - thumbnail_src: None, - width: None, - height: None, - }], - videos: Vec::new(), - audios: Vec::new(), - error: None, - }), - created_at: String::new(), - }], - }; - - let context = build_tool_context(&document); - let image_id = ImageId::from_data_key(data_key); - - assert_eq!(context.image_data_key(&image_id), Some(data_key)); - assert!(!image_id.id.contains(data_key)); - - let display_ref = resolve_tool_call_image_ref(&document, &image_id) - .expect("generated image should have a display reference"); - assert_eq!(display_ref.image_src, "/generated-images/proj-1/output.png"); - assert_eq!(display_ref.object_key.as_deref(), Some(data_key)); - } - - #[test] - fn tool_call_display_args_resolve_target_and_reference_image_previews() { - let target = EditorAgentAttachmentRef { - source: EditorAgentAttachmentSource::CanvasResource, - reference_id: "resource-target".to_string(), - object_key: Some("editor-projects/proj-1/target.png".to_string()), - image_src: "/target.png".to_string(), - thumbnail_src: Some("/target-thumbnail.png".to_string()), - label: Some("目标角色".to_string()), - width: Some(640), - height: Some(960), - }; - let reference = EditorAgentAttachmentRef { - source: EditorAgentAttachmentSource::LibraryAsset, - reference_id: "asset-reference".to_string(), - object_key: Some("editor-assets/reference.png".to_string()), - image_src: "/reference.png".to_string(), - thumbnail_src: None, - label: Some("像素风参考".to_string()), - width: Some(512), - height: Some(512), - }; - let target_id = target.clone().into_image_id(); - let reference_id = reference.clone().into_image_id(); - let document = EditorAgentConversationMessagesDocument { - version: EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, - conversation_id: "editor-agent-conversation-1".to_string(), - messages: vec![EditorAgentMessage { - id: 0, - role: EditorAgentMessageRole::User, - text: "改成像素风".to_string(), - attachments: vec![target, reference], - tool_call: None, - created_at: String::new(), - }], - }; - - let tool_context = build_tool_context(&document); - let pricing = load_editor_generation_pricing_from_paths(None) - .expect("default editor pricing should load"); - let display_args = build_tool_call_display_args( - EditImageTool::NAME, - &json!({ - "object_image_id": target_id.id, - "reference_image_ids": [reference_id.id], - "prompt": "把角色换成像素风" - }), - &document, - &tool_context, - &pricing, - ) - .expect("validated tool args should build a display projection"); - - assert_eq!(display_args.string_args.len(), 1); - assert_eq!(display_args.string_args[0].label, "修改要求"); - assert_eq!(display_args.string_args[0].value, "把角色换成像素风"); - assert_eq!(display_args.image_args.len(), 2); - assert_eq!(display_args.image_args[0].label, "目标图片"); - assert_eq!( - display_args.image_args[0].refs[0].label.as_deref(), - Some("目标角色") - ); - assert_eq!(display_args.image_args[1].label, "参考图片"); - assert_eq!( - display_args.image_args[1].refs[0].label.as_deref(), - Some("像素风参考") - ); - assert!( - display_args - .image_args - .iter() - .flat_map(|arg| &arg.refs) - .all(|image| image.image_id.starts_with("sha256:")) - ); - assert_eq!(display_args.extras.price_mud_points, 3); - } -} - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct EditorAgentConversationDeleteResponse { @@ -941,13 +480,13 @@ pub async fn cancel_editor_agent_tool_call( .tool_call .as_mut() .ok_or_else(|| editor_agent_bad_request("message has no tool call"))?; - if tc.external_job_id.is_some() || tc.cancelled_at.is_some() { + if tc.status != EditorAgentToolCallStatus::Pending || tc.external_job_id.is_some() { return Err(editor_agent_bad_request( "tool call is no longer pending confirmation", )); } - tc.cancelled_at = Some(now_rfc3339()); + tc.status = EditorAgentToolCallStatus::Canceled; let arg_json = tc.args.to_string(); msg.text = format!( "[tool_call:{tool_name}] args: {arg_json} output: 用户已取消该操作", @@ -996,10 +535,12 @@ pub async fn confirm_editor_agent_tool_call( .tool_call .as_ref() .ok_or_else(|| editor_agent_bad_request("message has no tool call"))?; - if tool_call.cancelled_at.is_some() { + if tool_call.status == EditorAgentToolCallStatus::Canceled { return Err(editor_agent_bad_request("tool call was cancelled")); } - if tool_call.external_job_id.is_some() { + if tool_call.status != EditorAgentToolCallStatus::Pending + || tool_call.external_job_id.is_some() + { return Ok(json_success_body(Some(&request_context), message)); } @@ -1012,14 +553,14 @@ pub async fn confirm_editor_agent_tool_call( })) })?; let project = load_editor_agent_project(&state, &conversation).await?; - let context = build_tool_context(&document); + let context = context::build_tool_context(&document); let generation_inputs = |title: &str, value: &str| { - Some(editor_agent_media_generation_inputs( - &conversation, - message_id, - title, - value, - )) + Some(json!({ + "source": "editor-agent", + "conversationId": conversation.conversation_id, + "toolCallMessageId": message_id, + "fields": [{ "title": title, "value": value }], + })) }; let (job_kind, request_label, price_mud_points, payload) = match tool_name.as_str() { @@ -1072,16 +613,7 @@ pub async fn confirm_editor_agent_tool_call( reference_image_srcs: Some(reference_image_srcs), project_id: Some(conversation.project_id.clone()), asset_kind: Some(kind.unwrap_or("editor_agent_generated_image").to_string()), - generation_inputs: editor_agent_generation_inputs( - &conversation, - message_id, - &GenerateImageToolArgs { - prompt: title.clone(), - reference_image_ids: Vec::new(), - aspect_ratio: None, - image_size: None, - }, - ), + generation_inputs: generation_inputs("用户指令", &title), asset_folder_id: Some("project".to_string()), asset_label: Some(title.clone()), source_resource_id: None, @@ -1329,6 +861,7 @@ pub async fn confirm_editor_agent_tool_call( .as_mut() .ok_or_else(|| editor_agent_bad_request("message has no tool call"))?; tool_call.external_job_id = Some(job.job_id); + tool_call.status = EditorAgentToolCallStatus::Executing; write_messages_document(&state, &conversation, &document).await?; Ok(json_success_body( Some(&request_context), @@ -1367,29 +900,3 @@ async fn load_editor_agent_project( .with_details(json!({ "message": format!("project not found: {error}") })) }) } -fn editor_agent_generation_inputs( - conversation: &EditorAgentConversationRecord, - message_id: usize, - args: &GenerateImageToolArgs, -) -> Option { - Some(json!({ - "source": "editor-agent", - "conversationId": conversation.conversation_id, - "toolCallMessageId": message_id, - "fields": [{ "title": "用户指令", "value": args.prompt }], - })) -} - -fn editor_agent_media_generation_inputs( - conversation: &EditorAgentConversationRecord, - message_id: usize, - title: &str, - value: &str, -) -> Value { - json!({ - "source": "editor-agent", - "conversationId": conversation.conversation_id, - "toolCallMessageId": message_id, - "fields": [{ "title": title, "value": value }], - }) -} diff --git a/server-rs/crates/api-server/src/editor_agent/context.rs b/server-rs/crates/api-server/src/editor_agent/context.rs new file mode 100644 index 000000000..87d1e3f5c --- /dev/null +++ b/server-rs/crates/api-server/src/editor_agent/context.rs @@ -0,0 +1,36 @@ +use crate::editor_agent::editor_tools::common::EditorToolContext; +use crate::editor_agent::utils::{ImageId, ImageMetadata, IntoDataKey}; +use shared_contracts::editor_agent::EditorAgentConversationMessagesDocument; +use std::collections::HashMap; + +pub fn build_tool_context(document: &EditorAgentConversationMessagesDocument) -> EditorToolContext { + let mut images: HashMap = HashMap::new(); + + for msg in document.messages.iter().rev() { + // tool gen + if let Some(tc) = &msg.tool_call { + for img in &tc.images { + let data_key = img.clone().into_data_key(); + let image_id = ImageId::from_data_key(&data_key); + let metadata = ImageMetadata { + tag: String::new(), + data_key, + }; + images.insert(image_id, metadata); + } + } + + // user pointed + for a in &msg.attachments { + let data_key = a.clone().into_data_key(); + let image_id = ImageId::from_data_key(&data_key); + let metadata = ImageMetadata { + tag: String::new(), + data_key, + }; + images.insert(image_id, metadata); + } + } + + EditorToolContext { images } +} \ No newline at end of file diff --git a/server-rs/crates/api-server/src/editor_agent/display_args.rs b/server-rs/crates/api-server/src/editor_agent/display_args.rs new file mode 100644 index 000000000..84ee705e4 --- /dev/null +++ b/server-rs/crates/api-server/src/editor_agent/display_args.rs @@ -0,0 +1,314 @@ +use serde_json::Value; +use module_editor_agent::agent::error::PromptError; +use module_editor_agent::agent::tool::Tool; +use shared_contracts::editor_agent::{EditorAgentConversationMessagesDocument, EditorAgentToolCallDisplayArgs, EditorAgentToolCallDisplayExtras, EditorAgentToolCallImageArg, EditorAgentToolCallImageRef, EditorAgentToolCallStringArg}; +use crate::editor_agent::editor_tools::common::{EditorAgentPricedTool, EditorToolContext}; +use crate::editor_agent::editor_tools::edit_image::{EditImageTool, EditImageToolArgs}; +use crate::editor_agent::editor_tools::generate_background_music::{GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs}; +use crate::editor_agent::editor_tools::generate_character::GenerateCharacterTool; +use crate::editor_agent::editor_tools::generate_icon_spritesheet::{GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs}; +use crate::editor_agent::editor_tools::generate_image::{GenerateImageTool, GenerateImageToolArgs}; +use crate::editor_agent::editor_tools::generate_sound_effect::{GenerateSoundEffectTool, GenerateSoundEffectToolArgs}; +use crate::editor_agent::editor_tools::generate_ui_design::GenerateUiDesignTool; +use crate::editor_agent::editor_tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; +use crate::editor_agent::utils::{ImageId, IntoDataKey, IntoImageId}; +use crate::editor_generation_config::EditorGenerationPricingConfig; + +pub fn build_tool_call_display_args( + tool_name: &str, + args: &Value, + document: &EditorAgentConversationMessagesDocument, + tool_context: &EditorToolContext, + pricing: &EditorGenerationPricingConfig, +) -> Result { + let mut display_args = EditorAgentToolCallDisplayArgs::default(); + + let price_mud_points = match tool_name { + EditImageTool::NAME => { + let args: EditImageToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = EditImageTool { + context: tool_context.clone(), + } + .pricing(pricing, &args); + push_string_display_arg(&mut display_args, "prompt", "修改要求", args.prompt); + push_image_display_arg( + &mut display_args, + document, + "object_image_id", + "目标图片", + std::slice::from_ref(&args.object_image_id), + )?; + push_image_display_arg( + &mut display_args, + document, + "reference_image_ids", + "参考图片", + args.reference_image_ids.as_slice(), + )?; + price_mud_points + } + GenerateImageTool::NAME | GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => { + let args: GenerateImageToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = match tool_name { + GenerateImageTool::NAME => GenerateImageTool { + context: tool_context.clone(), + } + .pricing(pricing, &args), + GenerateCharacterTool::NAME => GenerateCharacterTool { + context: tool_context.clone(), + } + .pricing(pricing, &args), + GenerateUiDesignTool::NAME => GenerateUiDesignTool { + context: tool_context.clone(), + } + .pricing(pricing, &args), + _ => unreachable!("tool name was matched above"), + }; + push_string_display_arg(&mut display_args, "prompt", "提示词", args.prompt); + push_optional_string_display_arg( + &mut display_args, + "aspect_ratio", + "画面比例", + args.aspect_ratio, + ); + push_optional_string_display_arg( + &mut display_args, + "image_size", + "图片尺寸", + args.image_size, + ); + push_image_display_arg( + &mut display_args, + document, + "reference_image_ids", + "参考图片", + args.reference_image_ids.as_slice(), + )?; + price_mud_points + } + GenerateIconSpritesheetTool::NAME => { + let args: GenerateIconSpritesheetToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = GenerateIconSpritesheetTool { + context: tool_context.clone(), + } + .pricing(pricing, &args); + push_string_display_arg( + &mut display_args, + "icon_descriptions", + "图标描述", + args.icon_descriptions.join("\n"), + ); + push_optional_string_display_arg( + &mut display_args, + "aspect_ratio", + "图集比例", + args.aspect_ratio, + ); + push_optional_string_display_arg( + &mut display_args, + "image_size", + "图片尺寸", + args.image_size, + ); + push_image_display_arg( + &mut display_args, + document, + "reference_image_id", + "规范参考图", + std::slice::from_ref(&args.reference_image_id), + )?; + push_image_display_arg( + &mut display_args, + document, + "reference_image_ids", + "额外参考图", + args.reference_image_ids.as_slice(), + )?; + price_mud_points + } + GenerateVideoTool::NAME => { + let args: GenerateVideoToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = GenerateVideoTool { + context: tool_context.clone(), + } + .pricing(pricing, &args); + push_string_display_arg(&mut display_args, "prompt", "视频描述", args.prompt); + push_optional_string_display_arg( + &mut display_args, + "aspect_ratio", + "画面比例", + args.aspect_ratio, + ); + if let Some(duration_seconds) = args.duration_seconds { + push_string_display_arg( + &mut display_args, + "duration_seconds", + "时长（秒）", + duration_seconds.to_string(), + ); + } + push_optional_string_display_arg(&mut display_args, "model", "模型", args.model); + push_optional_string_display_arg( + &mut display_args, + "resolution", + "分辨率", + args.resolution, + ); + push_optional_string_display_arg(&mut display_args, "sound", "声音", args.sound); + push_image_display_arg( + &mut display_args, + document, + "reference_image_ids", + "参考图片", + args.reference_image_ids.as_slice(), + )?; + price_mud_points + } + GenerateSoundEffectTool::NAME => { + let args: GenerateSoundEffectToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = GenerateSoundEffectTool.pricing(pricing, &args); + push_string_display_arg(&mut display_args, "prompt", "音效描述", args.prompt); + if let Some(duration) = args.duration { + push_string_display_arg( + &mut display_args, + "duration", + "时长（秒）", + duration.to_string(), + ); + } + push_optional_string_display_arg(&mut display_args, "model", "模型", args.model); + price_mud_points + } + GenerateBackgroundMusicTool::NAME => { + let args: GenerateBackgroundMusicToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = GenerateBackgroundMusicTool.pricing(pricing, &args); + push_string_display_arg(&mut display_args, "prompt", "音乐描述", args.prompt); + push_string_display_arg( + &mut display_args, + "make_instrumental", + "纯音乐", + if args.make_instrumental { "是" } else { "否" }, + ); + price_mud_points + } + _ => { + return Err(PromptError::InternalError(format!( + "missing display argument mapping for tool {tool_name}" + ))); + } + }; + + display_args.extras = EditorAgentToolCallDisplayExtras { price_mud_points }; + + Ok(display_args) +} + +fn parse_display_tool_args(tool_name: &str, args: &Value) -> Result +where + T: serde::de::DeserializeOwned, +{ + serde_json::from_value(args.clone()).map_err(|error| { + PromptError::InternalError(format!( + "failed to build display arguments for {tool_name}: {error}" + )) + }) +} + +fn push_string_display_arg( + display_args: &mut EditorAgentToolCallDisplayArgs, + name: &str, + label: &str, + value: impl Into, +) { + let value = value.into(); + if value.trim().is_empty() { + return; + } + display_args.string_args.push(EditorAgentToolCallStringArg { + name: name.to_string(), + label: label.to_string(), + value, + }); +} + +fn push_optional_string_display_arg( + display_args: &mut EditorAgentToolCallDisplayArgs, + name: &str, + label: &str, + value: Option, +) { + if let Some(value) = value { + push_string_display_arg(display_args, name, label, value); + } +} + +fn push_image_display_arg( + display_args: &mut EditorAgentToolCallDisplayArgs, + document: &EditorAgentConversationMessagesDocument, + name: &str, + label: &str, + image_ids: &[ImageId], +) -> Result<(), PromptError> { + if image_ids.is_empty() { + return Ok(()); + } + + let refs = image_ids + .iter() + .map(|image_id| { + resolve_tool_call_image_ref(document, image_id).ok_or_else(|| { + PromptError::InternalError(format!( + "display image reference {} is missing from conversation context", + image_id.id + )) + }) + }) + .collect::, _>>()?; + + display_args.image_args.push(EditorAgentToolCallImageArg { + name: name.to_string(), + label: label.to_string(), + refs, + }); + Ok(()) +} + +pub fn resolve_tool_call_image_ref( + document: &EditorAgentConversationMessagesDocument, + image_id: &ImageId, +) -> Option { + for message in document.messages.iter().rev() { + for attachment in message.attachments.iter().rev() { + if attachment.clone().into_image_id() == *image_id { + return Some(EditorAgentToolCallImageRef { + image_id: image_id.id.clone(), + image_src: attachment.image_src.clone(), + object_key: attachment.object_key.clone(), + thumbnail_src: attachment.thumbnail_src.clone(), + label: attachment.label.clone(), + width: attachment.width, + height: attachment.height, + }); + } + } + + if let Some(tool_call) = &message.tool_call { + for image in tool_call.images.iter().rev() { + let data_key = image.clone().into_data_key(); + if ImageId::from_data_key(data_key) == *image_id { + return Some(EditorAgentToolCallImageRef { + image_id: image_id.id.clone(), + image_src: image.image_src.clone(), + object_key: image.object_key.clone(), + thumbnail_src: image.thumbnail_src.clone(), + label: None, + width: image.width, + height: image.height, + }); + } + } + } + } + + None +} \ No newline at end of file diff --git a/server-rs/crates/api-server/src/editor_agent/mod.rs b/server-rs/crates/api-server/src/editor_agent/mod.rs index 16c3cb530..3fa73ebb1 100644 --- a/server-rs/crates/api-server/src/editor_agent/mod.rs +++ b/server-rs/crates/api-server/src/editor_agent/mod.rs @@ -2,6 +2,9 @@ mod editor_tools; mod utils; pub mod api; mod agent; +mod display_args; +mod context; +mod resp_to_asset; pub use api::{ create_editor_agent_conversation, delete_editor_agent_conversation, diff --git a/server-rs/crates/api-server/src/editor_agent/resp_to_asset.rs b/server-rs/crates/api-server/src/editor_agent/resp_to_asset.rs new file mode 100644 index 000000000..a365e72a1 --- /dev/null +++ b/server-rs/crates/api-server/src/editor_agent/resp_to_asset.rs @@ -0,0 +1,74 @@ +use serde_json::Value; +use shared_contracts::assets::{EditorAudioGenerateResponse, EditorVideoGenerateResponse}; +use shared_contracts::editor_agent::{EditorAgentGeneratedAudio, EditorAgentGeneratedImage, EditorAgentGeneratedVideo}; +use crate::editor_agent::editor_tools::edit_image::EditorImageEditResult; +use crate::editor_agent::editor_tools::generate_icon_spritesheet::EditorIconSpritesheetResult; +use crate::editor_agent::editor_tools::generate_image::EditorImageGenerationResult; + +fn value_string(value: Option<&Value>, field: &str) -> Option { + value + .and_then(|resource| resource.get(field)) + .and_then(Value::as_str) + .map(ToOwned::to_owned) +} + +pub fn editor_agent_generated_image(result: &EditorImageGenerationResult) -> EditorAgentGeneratedImage { + EditorAgentGeneratedImage { + resource_id: value_string(result.resource.as_ref(), "resourceId"), + object_key: result.object_key.clone(), + asset_object_id: result.asset_object_id.clone(), + image_src: result.image_src.clone(), + thumbnail_src: None, + width: Some(result.width), + height: Some(result.height), + } +} + +pub fn editor_agent_generated_edited_image( + result: &EditorImageEditResult, +) -> EditorAgentGeneratedImage { + EditorAgentGeneratedImage { + resource_id: value_string(result.resource.as_ref(), "resourceId"), + object_key: result.object_key.clone(), + asset_object_id: result.asset_object_id.clone(), + image_src: result.image_src.clone(), + thumbnail_src: None, + width: Some(result.width), + height: Some(result.height), + } +} + +pub fn editor_agent_generated_spritesheet( + result: &EditorIconSpritesheetResult, +) -> EditorAgentGeneratedImage { + EditorAgentGeneratedImage { + resource_id: value_string(result.spritesheet_resource.as_ref(), "resourceId"), + object_key: value_string(result.spritesheet_resource.as_ref(), "objectKey"), + asset_object_id: value_string(result.spritesheet_resource.as_ref(), "assetObjectId"), + image_src: result.spritesheet_image_src.clone(), + thumbnail_src: None, + width: Some(result.spritesheet_width), + height: Some(result.spritesheet_height), + } +} + +pub fn editor_agent_generated_video(result: &EditorVideoGenerateResponse) -> EditorAgentGeneratedVideo { + EditorAgentGeneratedVideo { + resource_id: value_string(result.resource.as_ref(), "resourceId"), + object_key: result.object_key.clone(), + asset_object_id: result.asset_object_id.clone(), + video_src: result.video_src.clone(), + thumbnail_src: result.thumbnail_src.clone(), + width: Some(result.width), + height: Some(result.height), + } +} + +pub fn editor_agent_generated_audio(result: &EditorAudioGenerateResponse) -> EditorAgentGeneratedAudio { + EditorAgentGeneratedAudio { + resource_id: value_string(result.resource.as_ref(), "resourceId"), + object_key: result.object_key.clone(), + asset_object_id: result.asset_object_id.clone(), + audio_src: result.audio_src.clone(), + } +}