diff --git a/server-rs/crates/api-server/src/editor_agent/agent.rs b/server-rs/crates/api-server/src/editor_agent/agent.rs index bf1aefc8f..8d1773734 100644 --- a/server-rs/crates/api-server/src/editor_agent/agent.rs +++ b/server-rs/crates/api-server/src/editor_agent/agent.rs @@ -8,7 +8,7 @@ use module_editor_agent::agent::run::ToolCallFlow; use module_editor_agent::agent::tool::{Tool, ToolCall, ToolDyn}; use platform_llm::{LlmClient, LlmMessage}; -struct LlmCompletionModel { +pub(crate) struct LlmCompletionModel { client: LlmClient, } diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index a0b3a8964..99791362a 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -13,27 +13,31 @@ use module_editor_agent::{ use platform_llm::LlmMessage; use serde::Serialize; use serde_json::{Value, json}; +use module_editor_agent::agent::tool::Tool; use shared_contracts::editor_agent::{ CreateEditorAgentConversationRequest, EditorAgentConversationListResponse, EditorAgentConversationMessagesDocument, EditorAgentConversationResponse, - EditorAgentConversationSummary, EditorAgentMessage, EditorAgentMessageRole, - EditorAgentToolCall, EditorAgentToolCallStatus, StreamEditorAgentMessageRequest, + EditorAgentConversationSummary, EditorAgentGeneratedImage, EditorAgentMessage, + EditorAgentMessageRole, EditorAgentToolCall, EditorAgentToolCallStatus, + StreamEditorAgentMessageRequest, }; use spacetime_client::{ EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput, EditorAgentConversationRecord, EditorAgentConversationTouchRecordInput, + EditorProjectGetRecordInput, }; use crate::auth::AuthenticatedAccessToken; use crate::editor_agent::agent::LlmChatAgentBuilder; -use crate::editor_agent::editor_tools::edit_image::EditImageTool; +use crate::editor_agent::editor_tools::edit_image::{EditImageTool, EditImageToolArgs}; use crate::editor_agent::utils::{ - conversation_detail_from_record, conversation_summary_from_record, empty_messages_document, + build_editor_agent_canvas_completion, conversation_detail_from_record, + conversation_summary_from_record, editor_agent_bad_request, empty_messages_document, ensure_editor_project_access, EditorAgentMessageResponse, ImageId, ImageMetadata, normalize_editor_agent_attachments, now_rfc3339, read_messages_document, require_editor_agent_sidebar_enabled, write_messages_document, }; -use crate::editor_project::{current_utc_micros, map_editor_project_error}; +use crate::editor_project::{current_utc_micros, map_editor_project_error, EditorGenerationCaller}; use crate::http_error::AppError; use crate::request_context::RequestContext; use crate::api_response::json_success_body; @@ -359,3 +363,197 @@ pub async fn delete_editor_agent_conversation( }, )) } + +pub async fn cancel_editor_agent_tool_call( + State(state): State, + Path((conversation_id, message_id)): Path<(String, usize)>, + Extension(request_context): Extension, + Extension(authenticated): Extension, +) -> Result, AppError> { + let owner_user_id = authenticated.claims().user_id().to_string(); + require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?; + + let conversation = state + .spacetime_client() + .get_editor_agent_conversation(conversation_id, owner_user_id) + .await + .map_err(|e| { + AppError::from_status(axum::http::StatusCode::NOT_FOUND) + .with_details(json!({ "message": format!("conversation not found: {e}") })) + })?; + + let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock( + conversation.conversation_id.as_str(), + ); + let _conversation_lock_guard = conversation_lock.lock_owned().await; + + let mut document: EditorAgentConversationMessagesDocument = + read_messages_document(&state, &conversation).await?; + + // Validate message index + if message_id >= document.messages.len() { + return Err(AppError::from_status(axum::http::StatusCode::NOT_FOUND) + .with_details(json!({ "message": "message not found" }))); + } + + let msg = &mut document.messages[message_id]; + + // Validate role and tool_call + if msg.role != EditorAgentMessageRole::System { + return Err(editor_agent_bad_request("message is not a system message")); + } + let tc = msg.tool_call.as_mut().ok_or_else(|| { + editor_agent_bad_request("message has no tool call") + })?; + if tc.status != EditorAgentToolCallStatus::PendingConfirmation { + return Err(editor_agent_bad_request( + "tool call is not in PendingConfirmation status", + )); + } + + // Mark as cancelled + tc.status = EditorAgentToolCallStatus::Cancelled; + let arg_json = tc.args.to_string(); + msg.text = format!( + "[tool_call:{tool_name}] args: {arg_json} output: 用户已取消该操作", + tool_name = tc.tool_name, + arg_json = arg_json, + ); + + write_messages_document(&state, &conversation, &document).await?; + + Ok(json_success_body(Some(&request_context), &document.messages[message_id])) +} + +pub async fn confirm_editor_agent_tool_call( + State(state): State, + Path((conversation_id, message_id)): Path<(String, usize)>, + Extension(request_context): Extension, + Extension(authenticated): Extension, +) -> Result, AppError> { + let owner_user_id = authenticated.claims().user_id().to_string(); + require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?; + + let conversation = state + .spacetime_client() + .get_editor_agent_conversation(conversation_id, owner_user_id) + .await + .map_err(|e| { + AppError::from_status(axum::http::StatusCode::NOT_FOUND) + .with_details(json!({ "message": format!("conversation not found: {e}") })) + })?; + + let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock( + conversation.conversation_id.as_str(), + ); + let _conversation_lock_guard = conversation_lock.lock_owned().await; + + let mut document: EditorAgentConversationMessagesDocument = + read_messages_document(&state, &conversation).await?; + + // Validate message index + if message_id >= document.messages.len() { + return Err(AppError::from_status(axum::http::StatusCode::NOT_FOUND) + .with_details(json!({ "message": "message not found" }))); + } + + let msg = &document.messages[message_id]; + + // Validate role and tool_call + if msg.role != EditorAgentMessageRole::System { + return Err(editor_agent_bad_request("message is not a system message")); + } + let tc = msg.tool_call.as_ref().ok_or_else(|| { + editor_agent_bad_request("message has no tool call") + })?; + if tc.status != EditorAgentToolCallStatus::PendingConfirmation { + return Err(editor_agent_bad_request( + "tool call is not in PendingConfirmation status", + )); + } + + // TODO dont use match + match tc.tool_name.as_str() { + EditImageTool::NAME => { + let args: EditImageToolArgs = serde_json::from_value(tc.args.clone()) + .map_err(|e| { + editor_agent_bad_request(format!("invalid tool call args: {e}")) + })?; + + // Get project + let project = state + .spacetime_client() + .get_editor_project(EditorProjectGetRecordInput { + project_id: conversation.project_id.clone(), + owner_user_id: conversation.owner_user_id.clone(), + }) + .await + .map_err(|e| { + AppError::from_status(axum::http::StatusCode::NOT_FOUND) + .with_details(json!({ "message": format!("project not found: {e}") })) + })?; + + // Build caller + let caller = EditorGenerationCaller { + owner_user_id: conversation.owner_user_id.clone(), + audit_subject_user_id: Some(conversation.owner_user_id.clone()), + audit_project_id: Some(conversation.project_id.clone()), + }; + + // Build canvas completion + let title = args.prompt.clone(); + let canvas_completion = + build_editor_agent_canvas_completion(&project, "edit-image", &title); + + // Execute the real generation + let edit_tool = EditImageTool {}; + let result = edit_tool + .execute( + &state, + &request_context, + caller, + args.clone(), + None, // model + Some(conversation.project_id.clone()), // project_id + None, // generation_inputs + None, // asset_label + None, // source_resource_id + Some(canvas_completion), + ) + .await?; + + // Build generated images + let generated_images = vec![EditorAgentGeneratedImage { + resource_id: None, + object_key: result.object_key.clone(), + asset_object_id: result.asset_object_id.clone(), + image_src: result.image_src.clone(), + thumbnail_src: None, + width: Some(result.width), + height: Some(result.height), + }]; + + // Format message text — reuse parsed args for format + let formatted_text = edit_tool.format_execute_message(args, result); + + // Update message + let msg = &mut document.messages[message_id]; + msg.text = formatted_text; + if let Some(ref mut tool_call) = msg.tool_call { + tool_call.status = EditorAgentToolCallStatus::Completed; + tool_call.images = generated_images; + } + + write_messages_document(&state, &conversation, &document).await?; + + Ok(json_success_body( + Some(&request_context), + &document.messages[message_id], + )) + } + _ => Err(editor_agent_bad_request(format!( + "unsupported tool: {}", + tc.tool_name + ))), + } +} diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs index cf5fe40fa..689a70700 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs @@ -44,7 +44,7 @@ impl Display for EditImageError { impl Error for EditImageError {} -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct EditImageToolArgs { pub object_image_id: ImageId, #[serde(default)] @@ -182,9 +182,13 @@ impl EditImageTool { } // TODO should share with format in run.rs - fn format_execute_message(&self, args: Self::Args, result: EditorImageEditResult) -> String { + pub fn format_execute_message( + &self, + args: ::Args, + result: EditorImageEditResult, + ) -> String { let name = Self::NAME.to_string(); - let arg_json = args.to_string(); + let arg_json = serde_json::to_string(&args).unwrap_or_default(); let image_id: ImageId = ImageId { id: result diff --git a/server-rs/crates/api-server/src/editor_agent/mod.rs b/server-rs/crates/api-server/src/editor_agent/mod.rs index 847847d1c..16c3cb530 100644 --- a/server-rs/crates/api-server/src/editor_agent/mod.rs +++ b/server-rs/crates/api-server/src/editor_agent/mod.rs @@ -6,4 +6,6 @@ mod agent; pub use api::{ create_editor_agent_conversation, delete_editor_agent_conversation, get_editor_agent_conversation, list_editor_agent_conversations, + confirm_editor_agent_tool_call, cancel_editor_agent_tool_call, }; +pub use utils::build_editor_agent_canvas_completion; diff --git a/server-rs/crates/api-server/src/editor_agent/utils.rs b/server-rs/crates/api-server/src/editor_agent/utils.rs index 1af7f7e60..7e7a979e2 100644 --- a/server-rs/crates/api-server/src/editor_agent/utils.rs +++ b/server-rs/crates/api-server/src/editor_agent/utils.rs @@ -5,7 +5,10 @@ use crate::state::AppState; use axum::http::StatusCode; use platform_oss::{LegacyAssetPrefix, OssObjectAccess, OssPutObjectRequest, OssSignedGetObjectUrlRequest}; use serde::{Deserialize, Serialize}; -use serde_json::json; +use serde_json::{json, Value}; +use shared_contracts::assets::{ + EditorCanvasGenerationCompletionPayload, EditorCanvasGenerationPlaceholderPayload, +}; use shared_contracts::editor_agent::{ EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, EditorAgentAttachmentRef, EditorAgentAttachmentSource, EditorAgentConversationDetail, EditorAgentConversationMessagesDocument, @@ -551,6 +554,65 @@ pub fn conversation_detail_from_record( } } +pub fn build_editor_agent_canvas_completion( + project: &EditorProjectRecord, + tool_name: &str, + title: &str, +) -> EditorCanvasGenerationCompletionPayload { + let (width, height) = editor_agent_tool_display_size(tool_name); + let (x, y) = next_editor_agent_canvas_position(project.layers.clone(), width, height); + EditorCanvasGenerationCompletionPayload { + dialog_id: None, + title: title.to_string(), + placeholder: EditorCanvasGenerationPlaceholderPayload { + x, + y, + width, + height, + original_width: width, + original_height: height, + }, + } +} + +fn editor_agent_tool_display_size(tool_name: &str) -> (f64, f64) { + match tool_name { + "ui-design" => (640.0, 360.0), + "generate-character" => (512.0, 768.0), + _ => (512.0, 512.0), + } +} + +const EDITOR_AGENT_CANVAS_RESULT_GAP: f64 = 32.0; + +fn next_editor_agent_canvas_position(layers: serde_json::Value, _width: f64, _height: f64) -> (f64, f64) { + let mut max_right: Option = None; + let mut min_y: Option = None; + if let Value::Array(items) = layers { + for item in items { + if item.get("itemType").and_then(Value::as_str) == Some("generation-dialog") { + continue; + } + let Some(x) = item.get("x").and_then(Value::as_f64) else { + continue; + }; + let Some(y) = item.get("y").and_then(Value::as_f64) else { + continue; + }; + let width = item.get("width").and_then(Value::as_f64).unwrap_or(0.0); + if !x.is_finite() || !y.is_finite() || !width.is_finite() { + continue; + } + max_right = Some(max_right.map_or(x + width, |value| value.max(x + width))); + min_y = Some(min_y.map_or(y, |value| value.min(y))); + } + } + ( + max_right.map_or(0.0, |right| right + EDITOR_AGENT_CANVAS_RESULT_GAP), + min_y.unwrap_or(0.0), + ) +} + pub fn empty_messages_document(conversation_id: &str) -> EditorAgentConversationMessagesDocument { EditorAgentConversationMessagesDocument { version: EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, diff --git a/server-rs/crates/api-server/src/modules/editor_project.rs b/server-rs/crates/api-server/src/modules/editor_project.rs index 646228278..801fe0eb7 100644 --- a/server-rs/crates/api-server/src/modules/editor_project.rs +++ b/server-rs/crates/api-server/src/modules/editor_project.rs @@ -9,6 +9,7 @@ use crate::{ auth::require_bearer_auth, editor_agent::api::editor_agent_message, editor_agent::{ + cancel_editor_agent_tool_call, confirm_editor_agent_tool_call, create_editor_agent_conversation, delete_editor_agent_conversation, get_editor_agent_conversation, list_editor_agent_conversations, }, @@ -106,6 +107,20 @@ pub fn router(state: AppState) -> Router { require_bearer_auth, )), ) + .route( + "/api/editor/agent-conversations/{conversation_id}/messages/{message_id}/confirm", + post(confirm_editor_agent_tool_call).route_layer(middleware::from_fn_with_state( + state.clone(), + require_bearer_auth, + )), + ) + .route( + "/api/editor/agent-conversations/{conversation_id}/messages/{message_id}/cancel", + post(cancel_editor_agent_tool_call).route_layer(middleware::from_fn_with_state( + state.clone(), + require_bearer_auth, + )), + ) .route( "/api/editor/project-resources/{resource_id}/showcase", patch(update_editor_project_resource_showcase).route_layer( diff --git a/server-rs/crates/shared-contracts/src/editor_agent.rs b/server-rs/crates/shared-contracts/src/editor_agent.rs index 12e2eb461..295d41de5 100644 --- a/server-rs/crates/shared-contracts/src/editor_agent.rs +++ b/server-rs/crates/shared-contracts/src/editor_agent.rs @@ -69,6 +69,7 @@ pub enum EditorAgentToolCallStatus { Executing, Completed, Failed, + Cancelled, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] diff --git a/server-rs/crates/shared-contracts/src/lib.rs b/server-rs/crates/shared-contracts/src/lib.rs index 72b53b026..6298f0cc5 100644 --- a/server-rs/crates/shared-contracts/src/lib.rs +++ b/server-rs/crates/shared-contracts/src/lib.rs @@ -34,4 +34,3 @@ pub mod square_hole_works; pub mod story; pub mod visual_novel; pub mod wooden_fish; -pub mod editor_agent_new;