use std::{ collections::BTreeMap, convert::Infallible, sync::{Arc, Mutex, OnceLock}, }; use axum::{ Json, extract::{Extension, Path, State}, http::StatusCode, response::{ IntoResponse, Response, sse::{Event, Sse}, }, }; use module_editor_agent::{ EDITOR_AGENT_CONVERSATION_ID_PREFIX, EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE, EDITOR_AGENT_MESSAGE_ID_PREFIX, derive_conversation_title, editor_agent_messages_object_key, validate_user_message, }; use platform_llm::{LlmError, LlmErrorKind, LlmMessage, LlmMessageContentPart, LlmRunRequest}; use platform_oss::{ LegacyAssetPrefix, OssObjectAccess, OssPutObjectRequest, OssSignedGetObjectUrlRequest, }; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use shared_contracts::assets::{ EditorCanvasGenerationCompletionPayload, EditorCanvasGenerationPlaceholderPayload, }; use shared_contracts::editor_agent::{ CreateEditorAgentConversationRequest, EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, EditorAgentAttachmentRef, EditorAgentAttachmentSource, EditorAgentConversationDetail, EditorAgentConversationListResponse, EditorAgentConversationMessagesDocument, EditorAgentConversationResponse, EditorAgentConversationSummary, EditorAgentDoneEvent, EditorAgentErrorEvent, EditorAgentGeneratedImage, EditorAgentGenerationRecord, EditorAgentGenerationResultEvent, EditorAgentGenerationStatus, EditorAgentMessage, EditorAgentMessageDeltaEvent, EditorAgentMessageKind, EditorAgentMessageRole, EditorAgentMessageStatus, EditorAgentStage, EditorAgentStageEvent, EditorAgentToolEvent, EditorAgentToolName, StreamEditorAgentMessageRequest, }; use shared_kernel::{build_prefixed_uuid_id, normalize_optional_string, normalize_required_string}; use spacetime_client::{ EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput, EditorAgentConversationRecord, EditorAgentConversationTouchRecordInput, EditorAssetLibraryRecord, EditorAssetRecord, EditorProjectGetRecordInput, EditorProjectRecord, EditorProjectResourceRecord, }; use crate::{ api_response::json_success_body, auth::AuthenticatedAccessToken, editor_project::{ EditorGenerationCaller, EditorIconSpritesheetGenerationRequest, EditorImageEditRequest, EditorImageGenerationRequest, current_utc_micros, edit_editor_image_for_owner, generate_editor_icon_spritesheet_for_owner, generate_editor_image_for_owner, map_editor_project_error, }, http_error::AppError, platform_errors::map_oss_error, request_context::RequestContext, state::AppState, }; const EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES: usize = 2 * 1024 * 1024; const EDITOR_AGENT_MESSAGES_READ_EXPIRE_SECONDS: u64 = 60; const EDITOR_AGENT_LLM_ATTACHMENT_URL_EXPIRE_SECONDS: u64 = 300; const EDITOR_AGENT_LLM_MAX_HISTORY_MESSAGES: usize = 12; const EDITOR_AGENT_LLM_PLANNING_MODEL: &str = platform_agent::CREATIVE_AGENT_GPT5_MODEL; const EDITOR_AGENT_LLM_MAX_OUTPUT_TOKENS: u32 = 1024; const EDITOR_AGENT_LLM_REQUEST_TIMEOUT_MS: u64 = 60_000; const EDITOR_AGENT_TOOL_CALL_ID_PREFIX: &str = "editor-agent-tool"; const EDITOR_AGENT_CANVAS_RESULT_GAP: f64 = 32.0; type EditorAgentConversationLockMap = Mutex>>>; static EDITOR_AGENT_CONVERSATION_LOCKS: OnceLock = OnceLock::new(); #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct EditorAgentConversationDeleteResponse { deleted_conversation_id: String, conversation: EditorAgentConversationSummary, } pub async fn list_editor_agent_conversations( State(state): State, Path(project_id): Path, Extension(request_context): Extension, Extension(authenticated): Extension, ) -> Result, AppError> { let owner_user_id = authenticated.claims().user_id().to_string(); require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?; ensure_editor_project_access(&state, project_id.as_str(), owner_user_id.as_str()).await?; let conversations = state .spacetime_client() .list_editor_agent_conversations(project_id, owner_user_id) .await .map_err(map_editor_project_error)? .into_iter() .map(conversation_summary_from_record) .collect(); Ok(json_success_body( Some(&request_context), EditorAgentConversationListResponse { conversations }, )) } pub async fn create_editor_agent_conversation( State(state): State, Path(project_id): Path, Extension(request_context): Extension, Extension(authenticated): Extension, Json(payload): Json, ) -> Result, AppError> { let owner_user_id = authenticated.claims().user_id().to_string(); require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?; ensure_editor_project_access(&state, project_id.as_str(), owner_user_id.as_str()).await?; let conversation_id = build_prefixed_uuid_id(EDITOR_AGENT_CONVERSATION_ID_PREFIX); let messages_object_key = editor_agent_messages_object_key(conversation_id.as_str()); let title = normalize_optional_string(payload.title) .unwrap_or_else(|| EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE.to_string()); let now_micros = current_utc_micros(); let seed_record = EditorAgentConversationRecord { conversation_id: conversation_id.clone(), project_id: project_id.clone(), owner_user_id: owner_user_id.clone(), title: title.clone(), messages_object_key: messages_object_key.clone(), deleted: false, created_at: now_rfc3339(), updated_at: now_rfc3339(), updated_at_micros: now_micros, }; write_messages_document( &state, &seed_record, &empty_messages_document(conversation_id.as_str()), ) .await?; let conversation = state .spacetime_client() .create_editor_agent_conversation(EditorAgentConversationCreateRecordInput { conversation_id, project_id, owner_user_id, title, messages_object_key, created_at_micros: now_micros, }) .await .map_err(map_editor_project_error)?; let document = read_messages_document(&state, &conversation).await?; Ok(json_success_body( Some(&request_context), EditorAgentConversationResponse { conversation: conversation_detail_from_record(conversation, document.messages), }, )) } pub async fn get_editor_agent_conversation( State(state): State, Path(conversation_id): Path, Extension(request_context): Extension, Extension(authenticated): Extension, ) -> Result, AppError> { let owner_user_id = authenticated.claims().user_id().to_string(); require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?; let conversation = state .spacetime_client() .get_editor_agent_conversation(conversation_id, owner_user_id) .await .map_err(map_editor_project_error)?; let document = read_messages_document(&state, &conversation).await?; Ok(json_success_body( Some(&request_context), EditorAgentConversationResponse { conversation: conversation_detail_from_record(conversation, document.messages), }, )) } pub async fn delete_editor_agent_conversation( State(state): State, Path(conversation_id): Path, Extension(request_context): Extension, Extension(authenticated): Extension, ) -> Result, AppError> { let owner_user_id = authenticated.claims().user_id().to_string(); require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?; let conversation = state .spacetime_client() .delete_editor_agent_conversation(EditorAgentConversationDeleteRecordInput { conversation_id, owner_user_id, updated_at_micros: current_utc_micros(), }) .await .map_err(map_editor_project_error)?; Ok(json_success_body( Some(&request_context), EditorAgentConversationDeleteResponse { deleted_conversation_id: conversation.conversation_id.clone(), conversation: conversation_summary_from_record(conversation), }, )) } pub async fn stream_editor_agent_message( State(state): State, Path(conversation_id): Path, Extension(request_context): Extension, Extension(authenticated): Extension, Json(payload): Json, ) -> Result { let owner_user_id = authenticated.claims().user_id().to_string(); require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?; let client_message_id = normalize_required_string(payload.client_message_id.as_str()) .ok_or_else(|| editor_agent_bad_request("clientMessageId is required"))?; let attachment_reference_ids = payload .attachments .iter() .map(|attachment| attachment.reference_id.clone()) .collect::>(); validate_user_message(payload.text.as_str(), &attachment_reference_ids) .map_err(|error| editor_agent_bad_request(error.to_string()))?; let conversation = state .spacetime_client() .get_editor_agent_conversation(conversation_id, owner_user_id.clone()) .await .map_err(map_editor_project_error)?; let attachments = normalize_editor_agent_attachments(&state, &conversation, payload.attachments.as_slice()) .await?; let conversation_lock = editor_agent_conversation_lock(conversation.conversation_id.as_str()); let conversation_lock_guard = conversation_lock.lock_owned().await; let mut document = read_messages_document(&state, &conversation).await?; if document .messages .iter() .any(|message| message.id == client_message_id) { return Err( AppError::from_status(StatusCode::CONFLICT).with_details(json!({ "provider": "editor-agent", "field": "clientMessageId", "message": "clientMessageId already exists in this conversation", })), ); } let was_empty = document.messages.is_empty(); let user_message = EditorAgentMessage { id: client_message_id, role: EditorAgentMessageRole::User, kind: EditorAgentMessageKind::Chat, text: payload.text.trim().to_string(), attachments, generations: Vec::new(), status: EditorAgentMessageStatus::Completed, created_at: now_rfc3339(), }; document.messages.push(user_message.clone()); write_messages_document(&state, &conversation, &document).await?; let latest_generation_reference = latest_editor_agent_generated_image_reference(&document); let assistant_message_id = build_prefixed_uuid_id(EDITOR_AGENT_MESSAGE_ID_PREFIX); let stream_state = state.clone(); let stream_conversation = conversation.clone(); let stream_title = was_empty.then(|| derive_conversation_title(user_message.text.as_str())); let stream_assistant_message_id = assistant_message_id.clone(); let stream_user_message = user_message.clone(); let stream_latest_generation_reference = latest_generation_reference.clone(); let stream_request_context = request_context.clone(); let stream_conversation_lock_guard = conversation_lock_guard; let stream = async_stream::stream! { let _conversation_lock_guard = stream_conversation_lock_guard; yield Ok::(editor_agent_sse_json_event_or_error( "stage", EditorAgentStageEvent { conversation_id: stream_conversation.conversation_id.clone(), stage: EditorAgentStage::Thinking, }, )); let mut assistant_status = EditorAgentMessageStatus::Completed; let mut generation_records = Vec::new(); let mut planning_failed = false; let turn_plan = match plan_editor_agent_turn( &stream_state, &stream_conversation, &document, &stream_user_message, stream_latest_generation_reference.as_ref(), ) .await { Ok(turn_plan) => turn_plan, Err(error) => { let app_error = error.into_app_error(); let error_message = app_error.body_text(); assistant_status = EditorAgentMessageStatus::Failed; planning_failed = true; EditorAgentTurnPlan { reply_text: error_message, tool_call: None, } } }; let assistant_text = turn_plan.reply_text; let assistant_kind = if assistant_status == EditorAgentMessageStatus::Failed { EditorAgentMessageKind::Error } else { EditorAgentMessageKind::Chat }; yield Ok::(editor_agent_sse_json_event_or_error( "stage", EditorAgentStageEvent { conversation_id: stream_conversation.conversation_id.clone(), stage: EditorAgentStage::Responding, }, )); yield Ok::(editor_agent_sse_json_event_or_error( "message_delta", EditorAgentMessageDeltaEvent { conversation_id: stream_conversation.conversation_id.clone(), message_id: stream_assistant_message_id.clone(), role: EditorAgentMessageRole::Assistant, kind: assistant_kind, text_delta: assistant_text.clone(), }, )); if planning_failed { yield Ok::(editor_agent_sse_json_event_or_error( "stage", EditorAgentStageEvent { conversation_id: stream_conversation.conversation_id.clone(), stage: EditorAgentStage::Failed, }, )); } if let Some(tool_call) = turn_plan.tool_call { let tool_call_id = build_prefixed_uuid_id(EDITOR_AGENT_TOOL_CALL_ID_PREFIX); let tool_summary = tool_call .summary .clone() .or_else(|| Some(editor_agent_tool_default_summary(tool_call.tool_name))); yield Ok::(editor_agent_sse_json_event_or_error( "tool_started", EditorAgentToolEvent { conversation_id: stream_conversation.conversation_id.clone(), message_id: stream_assistant_message_id.clone(), tool_call_id: tool_call_id.clone(), tool_name: tool_call.tool_name, summary: tool_summary.clone(), task_id: None, model: tool_call.model.clone(), status: Some(EditorAgentGenerationStatus::Generating), error: None, }, )); yield Ok::(editor_agent_sse_json_event_or_error( "stage", EditorAgentStageEvent { conversation_id: stream_conversation.conversation_id.clone(), stage: EditorAgentStage::Generating, }, )); match execute_editor_agent_tool_call( &stream_state, &stream_request_context, &stream_conversation, &stream_user_message, &tool_call_id, stream_latest_generation_reference.as_ref(), tool_call, ) .await { Ok(execution) => { generation_records.push(execution.record.clone()); yield Ok::(editor_agent_sse_json_event_or_error( "generation_result", EditorAgentGenerationResultEvent { conversation_id: stream_conversation.conversation_id.clone(), message_id: stream_assistant_message_id.clone(), tool_call_id: tool_call_id.clone(), tool_name: execution.record.tool_name, model: execution.record.model.clone(), images: execution.record.images.clone(), }, )); yield Ok::(editor_agent_sse_json_event_or_error( "tool_completed", EditorAgentToolEvent { conversation_id: stream_conversation.conversation_id.clone(), message_id: stream_assistant_message_id.clone(), tool_call_id, tool_name: execution.record.tool_name, summary: Some(execution.summary), task_id: execution.record.task_id.clone(), model: execution.record.model.clone(), status: Some(EditorAgentGenerationStatus::Completed), error: None, }, )); } Err(error) => { let error_code = error.error.code().to_string(); let error_message = error.error.body_text(); let error_model = error.model.clone(); assistant_status = EditorAgentMessageStatus::Failed; generation_records.push(EditorAgentGenerationRecord { tool_call_id: tool_call_id.clone(), tool_name: error.tool_name, summary: tool_summary.clone(), task_id: None, status: EditorAgentGenerationStatus::Failed, model: error_model.clone(), images: Vec::new(), error: Some(error_message.clone()), }); yield Ok::(editor_agent_sse_json_event_or_error( "tool_completed", EditorAgentToolEvent { conversation_id: stream_conversation.conversation_id.clone(), message_id: stream_assistant_message_id.clone(), tool_call_id, tool_name: error.tool_name, summary: tool_summary, task_id: None, model: error_model, status: Some(EditorAgentGenerationStatus::Failed), error: Some(error_message.clone()), }, )); yield Ok::(editor_agent_sse_json_event_or_error( "stage", EditorAgentStageEvent { conversation_id: stream_conversation.conversation_id.clone(), stage: EditorAgentStage::Failed, }, )); yield Ok::(editor_agent_sse_json_event_or_error( "error", EditorAgentErrorEvent { conversation_id: Some(stream_conversation.conversation_id.clone()), code: error_code, message: error_message, recoverable: true, }, )); } } } let write_result = async { let mut next_document = read_messages_document(&stream_state, &stream_conversation).await?; if !next_document .messages .iter() .any(|message| message.id == stream_assistant_message_id) { next_document.messages.push(EditorAgentMessage { id: stream_assistant_message_id.clone(), role: EditorAgentMessageRole::Assistant, kind: if assistant_status == EditorAgentMessageStatus::Failed { EditorAgentMessageKind::Error } else { EditorAgentMessageKind::Chat }, text: assistant_text, attachments: Vec::new(), generations: generation_records, status: assistant_status, created_at: now_rfc3339(), }); } write_messages_document(&stream_state, &stream_conversation, &next_document).await } .await; match write_result { Ok(()) => { let mut done_title = None; if let Some(title) = stream_title { match stream_state.spacetime_client().touch_editor_agent_conversation( EditorAgentConversationTouchRecordInput { conversation_id: stream_conversation.conversation_id.clone(), owner_user_id: stream_conversation.owner_user_id.clone(), title: Some(title.clone()), updated_at_micros: current_utc_micros(), }, ).await { Ok(updated) => done_title = Some(updated.title), Err(error) => { yield Ok::(editor_agent_sse_json_event_or_error( "error", EditorAgentErrorEvent { conversation_id: Some(stream_conversation.conversation_id.clone()), code: "SPACETIME_UPDATE_FAILED".to_string(), message: error.to_string(), recoverable: true, }, )); } } } else { match stream_state.spacetime_client().touch_editor_agent_conversation( EditorAgentConversationTouchRecordInput { conversation_id: stream_conversation.conversation_id.clone(), owner_user_id: stream_conversation.owner_user_id.clone(), title: None, updated_at_micros: current_utc_micros(), }, ).await { Ok(_) => {} Err(error) => { yield Ok::(editor_agent_sse_json_event_or_error( "error", EditorAgentErrorEvent { conversation_id: Some(stream_conversation.conversation_id.clone()), code: "SPACETIME_UPDATE_FAILED".to_string(), message: error.to_string(), recoverable: true, }, )); } } } yield Ok::(editor_agent_sse_json_event_or_error( "stage", EditorAgentStageEvent { conversation_id: stream_conversation.conversation_id.clone(), stage: EditorAgentStage::Completed, }, )); yield Ok::(editor_agent_sse_json_event_or_error( "done", EditorAgentDoneEvent { conversation_id: stream_conversation.conversation_id.clone(), title: done_title, }, )); } Err(error) => { yield Ok::(editor_agent_sse_json_event_or_error( "stage", EditorAgentStageEvent { conversation_id: stream_conversation.conversation_id.clone(), stage: EditorAgentStage::Failed, }, )); yield Ok::(editor_agent_sse_json_event_or_error( "error", EditorAgentErrorEvent { conversation_id: Some(stream_conversation.conversation_id.clone()), code: error.code().to_string(), message: error.body_text(), recoverable: true, }, )); } } }; Ok(Sse::new(stream).into_response()) } fn editor_agent_conversation_lock(conversation_id: &str) -> Arc> { let locks = EDITOR_AGENT_CONVERSATION_LOCKS.get_or_init(|| Mutex::new(BTreeMap::new())); let mut locks = locks .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); locks .entry(conversation_id.to_string()) .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) .clone() } async fn ensure_editor_project_access( state: &AppState, project_id: &str, owner_user_id: &str, ) -> Result<(), AppError> { state .spacetime_client() .get_editor_project(EditorProjectGetRecordInput { project_id: project_id.to_string(), owner_user_id: owner_user_id.to_string(), }) .await .map(|_| ()) .map_err(map_editor_project_error) } async fn normalize_editor_agent_attachments( state: &AppState, conversation: &EditorAgentConversationRecord, attachments: &[EditorAgentAttachmentRef], ) -> Result, AppError> { if attachments.is_empty() { return Ok(Vec::new()); } let needs_canvas_resources = attachments .iter() .any(|attachment| attachment.source == EditorAgentAttachmentSource::CanvasResource); let needs_library_assets = attachments .iter() .any(|attachment| attachment.source == EditorAgentAttachmentSource::LibraryAsset); let project = if needs_canvas_resources { Some( state .spacetime_client() .get_editor_project(EditorProjectGetRecordInput { project_id: conversation.project_id.clone(), owner_user_id: conversation.owner_user_id.clone(), }) .await .map_err(map_editor_project_error)?, ) } else { None }; let library = if needs_library_assets { Some( state .spacetime_client() .get_editor_asset_library(conversation.owner_user_id.clone(), current_utc_micros()) .await .map_err(map_editor_project_error)?, ) } else { None }; attachments .iter() .map(|attachment| { normalize_editor_agent_attachment( conversation, project.as_ref(), library.as_ref(), attachment, ) }) .collect() } fn normalize_editor_agent_attachment( conversation: &EditorAgentConversationRecord, project: Option<&EditorProjectRecord>, library: Option<&EditorAssetLibraryRecord>, attachment: &EditorAgentAttachmentRef, ) -> Result { let reference_id = normalize_required_string(attachment.reference_id.as_str()) .ok_or_else(|| editor_agent_bad_request("attachment.referenceId is required"))?; match attachment.source { EditorAgentAttachmentSource::CanvasResource => { let project = project.ok_or_else(|| { editor_agent_bad_request("canvas resource attachment project context missing") })?; let resource = project .resources .iter() .find(|resource| resource.resource_id == reference_id) .ok_or_else(|| { editor_agent_bad_request(format!( "canvas resource attachment not found in current project: {reference_id}" )) })?; normalize_canvas_resource_attachment(conversation, attachment, resource) } EditorAgentAttachmentSource::LibraryAsset => { let library = library.ok_or_else(|| { editor_agent_bad_request("library asset attachment context missing") })?; let asset = library .assets .iter() .find(|asset| asset.asset_id == reference_id) .ok_or_else(|| { editor_agent_bad_request(format!( "library asset attachment not found for current user: {reference_id}" )) })?; normalize_library_asset_attachment(attachment, asset) } } } fn normalize_canvas_resource_attachment( conversation: &EditorAgentConversationRecord, attachment: &EditorAgentAttachmentRef, resource: &EditorProjectResourceRecord, ) -> Result { if resource.project_id != conversation.project_id || resource.owner_user_id != conversation.owner_user_id { return Err(editor_agent_bad_request( "canvas resource attachment does not belong to this conversation project", )); } validate_attachment_object_key( attachment.object_key.as_deref(), resource.object_key.as_deref(), resource.resource_id.as_str(), )?; Ok(EditorAgentAttachmentRef { source: EditorAgentAttachmentSource::CanvasResource, reference_id: resource.resource_id.clone(), object_key: resource.object_key.clone(), image_src: resource.image_src.clone(), thumbnail_src: None, label: normalize_optional_string(attachment.label.clone()), width: Some(resource.width), height: Some(resource.height), }) } fn normalize_library_asset_attachment( attachment: &EditorAgentAttachmentRef, asset: &EditorAssetRecord, ) -> Result { validate_attachment_object_key( attachment.object_key.as_deref(), asset.object_key.as_deref(), asset.asset_id.as_str(), )?; Ok(EditorAgentAttachmentRef { source: EditorAgentAttachmentSource::LibraryAsset, reference_id: asset.asset_id.clone(), object_key: asset.object_key.clone(), image_src: asset.image_src.clone(), thumbnail_src: asset.thumbnail_src.clone(), label: normalize_optional_string(attachment.label.clone()) .or_else(|| Some(asset.label.clone())), width: Some(asset.width), height: Some(asset.height), }) } fn validate_attachment_object_key( submitted_object_key: Option<&str>, stored_object_key: Option<&str>, reference_id: &str, ) -> Result<(), AppError> { let Some(submitted_object_key) = submitted_object_key.and_then(normalize_required_string) else { return Ok(()); }; let Some(stored_object_key) = stored_object_key.and_then(normalize_required_string) else { return Err(editor_agent_bad_request(format!( "attachment objectKey is not available for reference: {reference_id}" ))); }; if submitted_object_key != stored_object_key { return Err(editor_agent_bad_request(format!( "attachment objectKey does not match reference: {reference_id}" ))); } Ok(()) } fn conversation_summary_from_record( conversation: EditorAgentConversationRecord, ) -> EditorAgentConversationSummary { EditorAgentConversationSummary { conversation_id: conversation.conversation_id, project_id: conversation.project_id, title: conversation.title, created_at: conversation.created_at, updated_at: conversation.updated_at, } } fn conversation_detail_from_record( conversation: EditorAgentConversationRecord, messages: Vec, ) -> EditorAgentConversationDetail { EditorAgentConversationDetail { conversation_id: conversation.conversation_id, project_id: conversation.project_id, title: conversation.title, created_at: conversation.created_at, updated_at: conversation.updated_at, messages, } } fn empty_messages_document(conversation_id: &str) -> EditorAgentConversationMessagesDocument { EditorAgentConversationMessagesDocument { version: EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, conversation_id: conversation_id.to_string(), messages: Vec::new(), } } async fn read_messages_document( state: &AppState, conversation: &EditorAgentConversationRecord, ) -> Result { let oss_client = state .oss_client() .ok_or_else(editor_agent_oss_unavailable)?; let signed = oss_client .sign_internal_get_object_url(OssSignedGetObjectUrlRequest { object_key: conversation.messages_object_key.clone(), expire_seconds: Some(EDITOR_AGENT_MESSAGES_READ_EXPIRE_SECONDS), }) .map_err(|error| map_oss_error(error, "aliyun-oss"))?; let response = reqwest::Client::new() .get(signed.signed_url.as_str()) .send() .await .map_err(|error| editor_agent_oss_read_error(error.to_string()))?; if response.status() == reqwest::StatusCode::NOT_FOUND { return Ok(empty_messages_document( conversation.conversation_id.as_str(), )); } if !response.status().is_success() { return Err(editor_agent_oss_read_error(format!( "OSS returned non-success status {}", response.status().as_u16() ))); } if response .content_length() .is_some_and(|size| size > EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES as u64) { return Err(editor_agent_messages_document_too_large()); } let bytes = response .bytes() .await .map_err(|error| editor_agent_oss_read_error(error.to_string()))?; if bytes.is_empty() { return Ok(empty_messages_document( conversation.conversation_id.as_str(), )); } if bytes.len() > EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES { return Err(editor_agent_messages_document_too_large()); } let document: EditorAgentConversationMessagesDocument = serde_json::from_slice(&bytes) .map_err(|error| { editor_agent_oss_read_error(format!("message document JSON invalid: {error}")) })?; if document.conversation_id != conversation.conversation_id { return Err( AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ "provider": "editor-agent", "message": "message document conversationId does not match metadata", "conversationId": conversation.conversation_id, "documentConversationId": document.conversation_id, })), ); } Ok(document) } async fn write_messages_document( state: &AppState, conversation: &EditorAgentConversationRecord, document: &EditorAgentConversationMessagesDocument, ) -> Result<(), AppError> { if document.conversation_id != conversation.conversation_id { return Err( AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ "provider": "editor-agent", "message": "message document conversationId does not match metadata", })), ); } let body = serde_json::to_vec(document).map_err(|error| { AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ "provider": "editor-agent", "message": format!("failed to serialize message document: {error}"), })) })?; if body.len() > EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES { return Err(editor_agent_messages_document_too_large()); } let oss_client = state .oss_client() .ok_or_else(editor_agent_oss_unavailable)?; let put_result = oss_client .put_object( &reqwest::Client::new(), OssPutObjectRequest { prefix: LegacyAssetPrefix::EditorAgent, path_segments: Vec::new(), file_name: format!("{}.json", conversation.conversation_id), content_type: Some("application/json; charset=utf-8".to_string()), access: OssObjectAccess::Private, metadata: BTreeMap::from([ ( "conversation-id".to_string(), conversation.conversation_id.clone(), ), ("project-id".to_string(), conversation.project_id.clone()), ( "owner-user-id".to_string(), conversation.owner_user_id.clone(), ), ]), body, }, ) .await .map_err(|error| map_oss_error(error, "aliyun-oss"))?; if put_result.object_key != conversation.messages_object_key { return Err( AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ "provider": "editor-agent", "message": "OSS object key mismatch while writing message document", "expectedObjectKey": conversation.messages_object_key, "actualObjectKey": put_result.object_key, })), ); } Ok(()) } #[derive(Clone, Debug, PartialEq)] struct EditorAgentTurnPlan { reply_text: String, tool_call: Option, } #[derive(Clone, Debug, PartialEq)] struct EditorAgentToolCallPlan { tool_name: EditorAgentToolName, prompt: String, summary: Option, model: Option, aspect_ratio: Option, image_size: Option, icon_descriptions: Vec, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct EditorAgentRawTurnPlan { reply_text: Option, tool_call: Option, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct EditorAgentRawToolCallPlan { tool_name: EditorAgentToolName, prompt: Option, summary: Option, model: Option, aspect_ratio: Option, image_size: Option, icon_descriptions: Option>, } #[derive(Debug)] struct EditorAgentToolExecution { record: EditorAgentGenerationRecord, summary: String, } #[derive(Clone, Debug)] struct EditorAgentGeneratedImageReference { source_message_id: String, tool_call_id: String, tool_name: EditorAgentToolName, summary: Option, model: Option, resource_id: Option, object_key: Option, asset_object_id: Option, image_src: String, thumbnail_src: Option, width: Option, height: Option, } #[derive(Debug)] struct EditorAgentToolExecutionError { tool_name: EditorAgentToolName, model: Option, error: AppError, } #[derive(Debug)] enum EditorAgentPlanningError { LlmUnavailable, LlmRequestFailed(LlmError), InvalidLlmResponse, } impl EditorAgentPlanningError { fn into_app_error(self) -> AppError { match self { Self::LlmUnavailable => AppError::from_status(StatusCode::SERVICE_UNAVAILABLE) .with_details(json!({ "provider": "editor-agent-llm", "message": "画布 Agent 的 LLM 未配置，无法处理这句话。", })), Self::LlmRequestFailed(error) => { let status = match error.kind() { LlmErrorKind::Timeout => StatusCode::GATEWAY_TIMEOUT, LlmErrorKind::InvalidConfig | LlmErrorKind::InvalidRequest => { StatusCode::SERVICE_UNAVAILABLE } LlmErrorKind::Connectivity | LlmErrorKind::Upstream | LlmErrorKind::StreamUnavailable | LlmErrorKind::EmptyResponse | LlmErrorKind::Transport | LlmErrorKind::Deserialize => StatusCode::BAD_GATEWAY, }; AppError::from_status(status).with_details(json!({ "provider": "editor-agent-llm", "message": format!("画布 Agent 调用 LLM 失败：{error}"), })) } Self::InvalidLlmResponse => AppError::from_status(StatusCode::BAD_GATEWAY) .with_details(json!({ "provider": "editor-agent-llm", "message": "画布 Agent 的 LLM 返回格式错误，未能解析回复。", })), } } } async fn plan_editor_agent_turn( state: &AppState, conversation: &EditorAgentConversationRecord, document: &EditorAgentConversationMessagesDocument, user_message: &EditorAgentMessage, latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, ) -> Result { let Some(llm_client) = state.creative_agent_gpt5_client() else { return Err(EditorAgentPlanningError::LlmUnavailable); }; let request = build_editor_agent_llm_request( state, conversation, document, user_message, latest_generation_reference, ) .await; let response = llm_client .run(request) .await .map_err(EditorAgentPlanningError::LlmRequestFailed)?; parse_editor_agent_turn_plan(response.text.as_str()) .map(|plan| { apply_previous_generation_edit_default(plan, user_message, latest_generation_reference) }) .ok_or(EditorAgentPlanningError::InvalidLlmResponse) } async fn build_editor_agent_llm_request( state: &AppState, conversation: &EditorAgentConversationRecord, document: &EditorAgentConversationMessagesDocument, user_message: &EditorAgentMessage, latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, ) -> LlmRunRequest { let mut user_parts = vec![LlmMessageContentPart::InputText { text: build_editor_agent_llm_user_prompt( conversation, document, user_message, latest_generation_reference, ), }]; for attachment in &user_message.attachments { if let Some(image_url) = sign_editor_agent_attachment_image_url(state, attachment) { user_parts.push(LlmMessageContentPart::InputImage { image_url }); } } if user_message.attachments.is_empty() && editor_agent_text_mentions_previous_generation(user_message.text.as_str()) { if let Some(reference) = latest_generation_reference { if let Some(image_url) = sign_editor_agent_generated_image_url(state, reference) { user_parts.push(LlmMessageContentPart::InputImage { image_url }); } } } LlmRunRequest::new(vec![ LlmMessage::system(editor_agent_llm_system_prompt()), LlmMessage::user_multimodal(user_parts), ]) .with_model(EDITOR_AGENT_LLM_PLANNING_MODEL) .with_openai_chat() .with_max_output_tokens(EDITOR_AGENT_LLM_MAX_OUTPUT_TOKENS) .with_request_timeout_ms(EDITOR_AGENT_LLM_REQUEST_TIMEOUT_MS) } fn editor_agent_llm_system_prompt() -> String { [ "你是 Genarrative 图片画布 Agent，只负责帮助用户理解、规划和触发画布生成工具。", "必须只输出一个 JSON 对象，不要输出 Markdown、解释或额外文本。", "JSON 结构：{\"replyText\":\"给用户看的中文回复\",\"toolCall\":null 或 {\"toolName\":\"generate_image|edit_image|generate_character|generate_icon_spritesheet|generate_ui_design\",\"prompt\":\"生成或修改提示词\",\"summary\":\"短动作摘要\",\"model\":null,\"aspectRatio\":null,\"imageSize\":\"1K\",\"iconDescriptions\":[\"图标描述\"]}}。", "只有用户明确要求生成、重绘、修改、画 UI、出角色或图标时才给 toolCall；普通咨询、评价、解释时 toolCall 为 null。", "工具说明：", "generate_image：从文字生成一张全新图片，适合新场景、新物体、新插画、新背景、规范图/视觉规范图/风格规范图/素材规范展板；不要用于修改上一张图或附件图。prompt 必须写完整画面、主体、风格、构图和背景。", "edit_image：修改已有图片，适合换衣服、改颜色、替换背景、局部重绘、保持主体/构图/姿势不变的编辑请求；必须使用 latestAttachments 或 latestGeneratedImage 作为源图。", "generate_character：生成新的角色形象、人物立绘或角色设定图；如果用户是在改上一张角色图的服装、颜色、表情、姿势或背景，应改用 edit_image。", "generate_icon_spritesheet：生成一组图标素材或图标图集，适合用户明确要多个 icon / 图标 / spritesheet；必须有 latestAttachments 作为图标规范或风格参考，并填写 iconDescriptions。", "generate_ui_design：生成一张完整 UI 设计图或界面稿，适合 HUD、弹窗、面板、按钮组合和整页界面；不要用于提取图标、拆素材或修改上一张图。", "规范图要求：用户要求生成规范图/视觉规范图/风格规范图/素材规范展板时，当前使用 generate_image；prompt 必须明确这是规范展板，并写入统一视角、线条粗细、描边、填充风格、材质、阴影、圆角、状态层级、色卡/色号、尺寸标注和排版层级。", "角色规范图/角色美术视觉规范设定图属于规范展板时使用 generate_image，prompt 要包含头身比例、标准立绘、动作帧样例、服饰配饰分层和专属角色色卡；只有用户要生成单个全新角色形象/立绘/普通角色设定图时才使用 generate_character。", "图标规范图/图标视觉规范展板属于规范展板时使用 generate_image；只有用户明确要生成多个图标成品、图标素材图集或 spritesheet，并提供图标规范/风格参考附件时才使用 generate_icon_spritesheet。", "UI 规范图/组件规范展板如果是规范展板而非完整可用界面稿，也使用 generate_image；完整界面稿/HUD/弹窗/面板才使用 generate_ui_design。", "工具选择优先级：明确修改/指代已有图 => edit_image；规范图/视觉规范图/风格规范图/素材规范展板 => generate_image；明确新角色且不是规范展板或修改已有图 => generate_character；多个图标成品/图标图集 => generate_icon_spritesheet；完整界面稿 => generate_ui_design；其他全新图片 => generate_image。", "edit_image 必须依赖 latestAttachments 或 latestGeneratedImage；generate_icon_spritesheet 必须依赖 latestAttachments。", "当 latestGeneratedImage 存在，且用户说“这张/刚才那个/上一张/把衣服换成/改成/换成/修改上一张图”等指代或修改上一轮结果的话，必须选择 edit_image，prompt 保留用户修改要求，不要要求用户重新选择参考图，也不要降级成 generate_image。", "没有 latestAttachments 且没有 latestGeneratedImage 时，不要调用 edit_image 或 generate_icon_spritesheet，应在 replyText 中提示先选择参考图。", "generate_ui_design 表示生成一张 UI 设计图，不是提取 UI 素材。", "对话回复要简短，不能承诺免费生成；生成工具由后端按模型定价扣泥点。", ] .join("\n") } fn build_editor_agent_llm_user_prompt( conversation: &EditorAgentConversationRecord, document: &EditorAgentConversationMessagesDocument, user_message: &EditorAgentMessage, latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, ) -> String { let history = document .messages .iter() .rev() .take(EDITOR_AGENT_LLM_MAX_HISTORY_MESSAGES) .collect::>() .into_iter() .rev() .map(|message| { json!({ "role": message.role, "kind": message.kind, "text": message.text, "attachmentCount": message.attachments.len(), "generationCount": message.generations.len(), "generations": message .generations .iter() .map(editor_agent_generation_context_json) .collect::>(), }) }) .collect::>(); let attachments = user_message .attachments .iter() .map(|attachment| { json!({ "source": attachment.source, "referenceId": attachment.reference_id, "label": attachment.label, "width": attachment.width, "height": attachment.height, "hasObjectKey": attachment.object_key.as_ref().is_some_and(|value| !value.trim().is_empty()), }) }) .collect::>(); json!({ "conversationId": conversation.conversation_id, "projectId": conversation.project_id, "latestUserText": user_message.text, "latestAttachments": attachments, "latestGeneratedImage": latest_generation_reference .map(editor_agent_generated_image_reference_context_json), "recentMessages": history, }) .to_string() } fn editor_agent_generation_context_json(generation: &EditorAgentGenerationRecord) -> Value { json!({ "toolCallId": generation.tool_call_id, "toolName": generation.tool_name, "summary": generation.summary, "status": generation.status, "model": generation.model, "images": generation.images.iter().take(4).map(|image| json!({ "resourceId": image.resource_id, "objectKey": image.object_key, "assetObjectId": image.asset_object_id, "imageSrc": image.image_src, "thumbnailSrc": image.thumbnail_src, "width": image.width, "height": image.height, "hasObjectKey": image.object_key.as_ref().is_some_and(|value| !value.trim().is_empty()), })).collect::>(), }) } fn editor_agent_generated_image_reference_context_json( reference: &EditorAgentGeneratedImageReference, ) -> Value { json!({ "sourceMessageId": reference.source_message_id, "toolCallId": reference.tool_call_id, "toolName": reference.tool_name, "summary": reference.summary, "model": reference.model, "resourceId": reference.resource_id, "objectKey": reference.object_key, "assetObjectId": reference.asset_object_id, "imageSrc": reference.image_src, "thumbnailSrc": reference.thumbnail_src, "width": reference.width, "height": reference.height, "hasObjectKey": reference.object_key.as_ref().is_some_and(|value| !value.trim().is_empty()), }) } fn sign_editor_agent_attachment_image_url( state: &AppState, attachment: &EditorAgentAttachmentRef, ) -> Option { let object_key = attachment .object_key .as_deref() .map(str::trim) .filter(|value| !value.is_empty())?; state .oss_client()? .sign_get_object_url(OssSignedGetObjectUrlRequest { object_key: object_key.trim_start_matches('/').to_string(), expire_seconds: Some(EDITOR_AGENT_LLM_ATTACHMENT_URL_EXPIRE_SECONDS), }) .ok() .map(|signed| signed.signed_url) } fn sign_editor_agent_generated_image_url( state: &AppState, reference: &EditorAgentGeneratedImageReference, ) -> Option { let object_key = reference .object_key .as_deref() .map(str::trim) .filter(|value| !value.is_empty())?; state .oss_client()? .sign_get_object_url(OssSignedGetObjectUrlRequest { object_key: object_key.trim_start_matches('/').to_string(), expire_seconds: Some(EDITOR_AGENT_LLM_ATTACHMENT_URL_EXPIRE_SECONDS), }) .ok() .map(|signed| signed.signed_url) } fn latest_editor_agent_generated_image_reference( document: &EditorAgentConversationMessagesDocument, ) -> Option { document.messages.iter().rev().find_map(|message| { message .generations .iter() .rev() .filter(|generation| generation.status == EditorAgentGenerationStatus::Completed) .find_map(|generation| { generation.images.iter().rev().find_map(|image| { let has_reference_source = image .object_key .as_deref() .is_some_and(|value| !value.trim().is_empty()) || image .resource_id .as_deref() .is_some_and(|value| !value.trim().is_empty()) || !image.image_src.trim().is_empty(); has_reference_source.then(|| EditorAgentGeneratedImageReference { source_message_id: message.id.clone(), tool_call_id: generation.tool_call_id.clone(), tool_name: generation.tool_name, summary: generation.summary.clone(), model: generation.model.clone(), resource_id: image.resource_id.clone(), object_key: image.object_key.clone(), asset_object_id: image.asset_object_id.clone(), image_src: image.image_src.clone(), thumbnail_src: image.thumbnail_src.clone(), width: image.width, height: image.height, }) }) }) }) } fn editor_agent_generated_reference_source( reference: &EditorAgentGeneratedImageReference, ) -> Option { reference .object_key .as_deref() .map(str::trim) .filter(|value| !value.is_empty()) .map(|value| value.trim_start_matches('/').to_string()) .or_else(|| { reference .resource_id .as_deref() .map(str::trim) .filter(|value| !value.is_empty()) .map(ToOwned::to_owned) }) .or_else(|| { reference .image_src .trim() .strip_prefix('/') .map(str::to_string) .filter(|value| !value.is_empty()) }) } fn editor_agent_text_mentions_previous_generation(text: &str) -> bool { let normalized = text.trim().to_lowercase(); contains_any( normalized.as_str(), &[ "这张", "这个图", "这幅", "这张图", "刚才", "刚刚", "刚才那个", "刚才那张", "上一张", "上张", "上一幅", "上一个", "前一张", "上次生成", "刚生成", "latest image", "previous image", "last image", ], ) } fn editor_agent_text_requests_previous_generation_edit(text: &str) -> bool { let normalized = text.trim().to_lowercase(); if editor_agent_text_mentions_previous_generation(normalized.as_str()) && contains_any( normalized.as_str(), &[ "修改", "改成", "换成", "替换", "重绘", "编辑", "优化", "去掉", "加上", "增加", "变成", "调整", "换一", "换掉", "remove", "change", "edit", "redraw", ], ) { return true; } let has_visual_target = contains_any( normalized.as_str(), &[ "衣服", "服装", "发型", "头发", "背景", "颜色", "色调", "表情", "姿势", "主体", "风格", "脸", "眼睛", "帽子", "鞋", "裙子", "外套", ], ); let has_edit_verb = contains_any( normalized.as_str(), &[ "换", "改", "变", "加", "去", "删", "替换", "调整", "change", "edit", "remove", ], ); has_visual_target && has_edit_verb } fn apply_previous_generation_edit_default( mut plan: EditorAgentTurnPlan, user_message: &EditorAgentMessage, latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, ) -> EditorAgentTurnPlan { if latest_generation_reference.is_none() || !user_message.attachments.is_empty() || !editor_agent_text_requests_previous_generation_edit(user_message.text.as_str()) { return plan; } let should_force_edit = plan .tool_call .as_ref() .is_none_or(|tool_call| tool_call.tool_name != EditorAgentToolName::EditImage); if !should_force_edit { return plan; } let original_tool = plan.tool_call.take(); let prompt = original_tool .as_ref() .map(|tool_call| tool_call.prompt.clone()) .unwrap_or_else(|| user_message.text.trim().to_string()); let summary = original_tool .as_ref() .and_then(|tool_call| tool_call.summary.clone()) .or_else(|| Some("修改上一张图".to_string())); let model = original_tool .as_ref() .and_then(|tool_call| tool_call.model.clone()); let image_size = original_tool .as_ref() .and_then(|tool_call| tool_call.image_size.clone()) .or_else(|| Some("1K".to_string())); plan.reply_text = "我会基于上一张生成图按你的要求修改。".to_string(); plan.tool_call = Some(EditorAgentToolCallPlan { tool_name: EditorAgentToolName::EditImage, prompt, summary, model, aspect_ratio: None, image_size, icon_descriptions: Vec::new(), }); plan } fn parse_editor_agent_turn_plan(raw_text: &str) -> Option { let json_text = extract_editor_agent_json_object(raw_text)?; let raw: EditorAgentRawTurnPlan = serde_json::from_str(json_text).ok()?; let tool_call = match raw.tool_call { Some(tool) => { let prompt = normalize_optional_string(tool.prompt)?; Some(EditorAgentToolCallPlan { tool_name: tool.tool_name, prompt, summary: normalize_optional_string(tool.summary), model: normalize_optional_string(tool.model), aspect_ratio: normalize_optional_string(tool.aspect_ratio), image_size: normalize_optional_string(tool.image_size), icon_descriptions: tool .icon_descriptions .unwrap_or_default() .into_iter() .filter_map(|description| normalize_optional_string(Some(description))) .collect(), }) } None => None, }; let reply_text = normalize_editor_agent_reply_text(raw.reply_text, tool_call.as_ref())?; Some(EditorAgentTurnPlan { reply_text, tool_call, }) } fn extract_editor_agent_json_object(raw_text: &str) -> Option<&str> { let trimmed = raw_text.trim(); let without_fence = if let Some(rest) = trimmed.strip_prefix("```json") { rest.strip_suffix("```").unwrap_or(rest).trim() } else if let Some(rest) = trimmed.strip_prefix("```") { rest.strip_suffix("```").unwrap_or(rest).trim() } else { trimmed }; let start = without_fence.find('{')?; let end = without_fence.rfind('}')?; (start <= end).then_some(&without_fence[start..=end]) } fn normalize_editor_agent_reply_text( reply_text: Option, tool_call: Option<&EditorAgentToolCallPlan>, ) -> Option { let reply_text = normalize_optional_string(reply_text)?; if !editor_agent_reply_text_looks_structural(reply_text.as_str()) { return Some(reply_text); } tool_call.map(editor_agent_tool_call_reply_text) } fn editor_agent_reply_text_looks_structural(text: &str) -> bool { let trimmed = text.trim(); if trimmed.is_empty() { return true; } let lower = trimmed.to_ascii_lowercase(); if matches!(lower.as_str(), "null" | "undefined") { return true; } trimmed.len() <= 2 && trimmed .chars() .all(|character| matches!(character, '{' | '}' | '[' | ']' | '"' | ':' | ',')) } fn editor_agent_tool_call_reply_text(tool_call: &EditorAgentToolCallPlan) -> String { match tool_call.tool_name { EditorAgentToolName::GenerateImage => "我来生成图片。".to_string(), EditorAgentToolName::EditImage => "我来修改图片。".to_string(), EditorAgentToolName::GenerateCharacter => "我来生成角色形象。".to_string(), EditorAgentToolName::GenerateIconSpritesheet => "我来生成图标素材。".to_string(), EditorAgentToolName::GenerateUiDesign => "我来生成 UI 设计图。".to_string(), } } #[cfg(test)] fn heuristic_editor_agent_turn_plan( text: &str, attachments: &[EditorAgentAttachmentRef], ) -> EditorAgentTurnPlan { let trimmed = text.trim(); let normalized = trimmed.to_lowercase(); let wants_generation = contains_any( normalized.as_str(), &[ "生成", "出图", "绘制", "设计", "画一", "画个", "画张", "generate", "create", ], ); let wants_edit = contains_any( normalized.as_str(), &[ "修改", "改成", "重绘", "换成", "编辑", "优化", "edit", "redraw", ], ); let wants_character = contains_any( normalized.as_str(), &["角色", "人物", "立绘", "形象", "character"], ); let wants_icons = contains_any( normalized.as_str(), &["图标", "icon", "spritesheet", "素材图集"], ); let wants_ui = contains_any( normalized.as_str(), &["ui", "界面", "hud", "按钮", "面板", "弹窗"], ); let tool_name = if wants_icons && !attachments.is_empty() { Some(EditorAgentToolName::GenerateIconSpritesheet) } else if wants_edit && !attachments.is_empty() { Some(EditorAgentToolName::EditImage) } else if wants_character && wants_generation { Some(EditorAgentToolName::GenerateCharacter) } else if wants_ui && wants_generation { Some(EditorAgentToolName::GenerateUiDesign) } else if wants_generation { Some(EditorAgentToolName::GenerateImage) } else { None }; let reply_text = match tool_name { Some(EditorAgentToolName::EditImage) => "我来按你的要求修改这张图。".to_string(), Some(EditorAgentToolName::GenerateCharacter) => "我来生成角色形象。".to_string(), Some(EditorAgentToolName::GenerateIconSpritesheet) => "我来生成图标素材图集。".to_string(), Some(EditorAgentToolName::GenerateUiDesign) => "我来生成 UI 设计图。".to_string(), Some(EditorAgentToolName::GenerateImage) => "我来生成图片。".to_string(), None => build_echo_assistant_text(trimmed, attachments.len()), }; let tool_call = tool_name.map(|tool_name| EditorAgentToolCallPlan { tool_name, prompt: if trimmed.is_empty() { "根据附件生成适合当前画布的图片".to_string() } else { trimmed.to_string() }, summary: Some(editor_agent_tool_default_summary(tool_name)), model: None, aspect_ratio: None, image_size: Some("1K".to_string()), icon_descriptions: derive_icon_descriptions(trimmed), }); EditorAgentTurnPlan { reply_text, tool_call, } } fn contains_any(text: &str, needles: &[&str]) -> bool { needles.iter().any(|needle| text.contains(needle)) } fn derive_icon_descriptions(prompt: &str) -> Vec { let normalized = prompt.trim(); if normalized.is_empty() { return vec![ "主按钮图标".to_string(), "奖励图标".to_string(), "关闭图标".to_string(), ]; } normalized .split(['、', ',', '，', '\n', ';', '；']) .map(str::trim) .filter(|item| !item.is_empty()) .take(8) .map(ToOwned::to_owned) .collect::>() .into_iter() .filter(|item| item.chars().count() <= 80) .collect::>() .pipe_non_empty_or_else(|| vec![normalized.to_string()]) } trait VecPipeNonEmpty { fn pipe_non_empty_or_else(self, fallback: F) -> Self where F: FnOnce() -> Self; } impl VecPipeNonEmpty for Vec { fn pipe_non_empty_or_else(self, fallback: F) -> Self where F: FnOnce() -> Self, { if self.is_empty() { fallback() } else { self } } } async fn execute_editor_agent_tool_call( state: &AppState, request_context: &RequestContext, conversation: &EditorAgentConversationRecord, user_message: &EditorAgentMessage, tool_call_id: &str, latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, tool_call: EditorAgentToolCallPlan, ) -> Result { let project = state .spacetime_client() .get_editor_project(EditorProjectGetRecordInput { project_id: conversation.project_id.clone(), owner_user_id: conversation.owner_user_id.clone(), }) .await .map_err(map_editor_project_error) .map_err(|error| EditorAgentToolExecutionError { tool_name: tool_call.tool_name, model: tool_call.model.clone(), error, })?; let completion = build_editor_agent_canvas_completion(&project, tool_call.tool_name, &tool_call.prompt); let caller = EditorGenerationCaller { owner_user_id: conversation.owner_user_id.clone(), audit_subject_user_id: Some(conversation.owner_user_id.clone()), audit_project_id: Some(conversation.project_id.clone()), }; let tool_request_context = editor_agent_tool_request_context(request_context, tool_call_id); let attachment_sources = editor_agent_attachment_sources(user_message.attachments.as_slice()); let previous_generation_source = (tool_call.tool_name == EditorAgentToolName::EditImage && user_message.attachments.is_empty()) .then(|| latest_generation_reference.and_then(editor_agent_generated_reference_source)) .flatten(); let tool_reference_sources = previous_generation_source .clone() .map(|source| vec![source]) .unwrap_or_else(|| attachment_sources.clone()); let generation_reference_context = editor_agent_tool_generation_reference_context( user_message.attachments.as_slice(), latest_generation_reference, previous_generation_source.as_deref(), ); let generation_inputs = Some(json!({ "source": "editor-agent", "conversationId": conversation.conversation_id, "messageId": user_message.id, "toolCallId": tool_call_id, "fields": [{ "title": "用户指令", "value": tool_call.prompt }], "references": generation_reference_context, })); let tool_name = tool_call.tool_name; let tool_model = tool_call.model.clone(); let response = match tool_name { EditorAgentToolName::GenerateImage | EditorAgentToolName::GenerateCharacter | EditorAgentToolName::GenerateUiDesign => { let kind = match tool_name { EditorAgentToolName::GenerateCharacter => Some("character".to_string()), EditorAgentToolName::GenerateUiDesign => Some("ui-design".to_string()), _ => None, }; generate_editor_image_for_owner( state, &tool_request_context, caller, EditorImageGenerationRequest { prompt: tool_call.prompt.clone(), size: None, kind, model: tool_call.model.clone(), screen_color: Some("auto".to_string()), seg_model: Some("birefnet".to_string()), aspect_ratio: tool_call.aspect_ratio.clone(), image_size: tool_call.image_size.clone(), reference_image_srcs: Some(tool_reference_sources.clone()), project_id: Some(conversation.project_id.clone()), asset_kind: Some(editor_agent_tool_asset_kind(tool_name).to_string()), generation_inputs, asset_folder_id: Some(editor_agent_default_asset_folder_id()), asset_label: tool_call.summary.clone(), source_resource_id: editor_agent_tool_source_resource_id( user_message.attachments.as_slice(), latest_generation_reference, previous_generation_source.as_deref(), ), canvas_completion: Some(completion), }, ) .await } EditorAgentToolName::EditImage => { let (source_image_src, reference_image_srcs) = split_editor_agent_edit_sources( tool_reference_sources.clone(), ) .ok_or_else(|| EditorAgentToolExecutionError { tool_name, model: tool_model.clone(), error: editor_agent_bad_request("修改图片需要先选择一张画布或素材库图片附件"), })?; edit_editor_image_for_owner( state, &tool_request_context, caller, EditorImageEditRequest { prompt: tool_call.prompt.clone(), source_image_src, size: None, model: tool_call.model.clone(), reference_image_srcs: Some(reference_image_srcs), project_id: Some(conversation.project_id.clone()), asset_kind: Some(editor_agent_tool_asset_kind(tool_name).to_string()), generation_inputs, asset_folder_id: Some(editor_agent_default_asset_folder_id()), asset_label: tool_call.summary.clone(), source_resource_id: editor_agent_tool_source_resource_id( user_message.attachments.as_slice(), latest_generation_reference, previous_generation_source.as_deref(), ), target_layer_id: None, canvas_completion: Some(completion), }, ) .await } EditorAgentToolName::GenerateIconSpritesheet => { let (reference_image_src, extra_references) = split_editor_agent_edit_sources(attachment_sources).ok_or_else(|| { EditorAgentToolExecutionError { tool_name, model: tool_model.clone(), error: editor_agent_bad_request("生成图标素材需要先选择一张图标规范参考图"), } })?; let icon_descriptions = if tool_call.icon_descriptions.is_empty() { derive_icon_descriptions(tool_call.prompt.as_str()) } else { tool_call.icon_descriptions.clone() }; generate_editor_icon_spritesheet_for_owner( state, &tool_request_context, caller, EditorIconSpritesheetGenerationRequest { reference_image_src, reference_image_srcs: Some(extra_references), icon_descriptions, model: tool_call.model.clone(), screen_color: Some("auto".to_string()), seg_model: Some("birefnet".to_string()), aspect_ratio: tool_call.aspect_ratio.clone(), image_size: tool_call.image_size.clone(), project_id: Some(conversation.project_id.clone()), generation_inputs, asset_folder_id: Some(editor_agent_default_asset_folder_id()), asset_label: None, canvas_completion: Some(completion), }, ) .await } } .map_err(|error| EditorAgentToolExecutionError { tool_name, model: tool_model.clone(), error, })?; Ok(build_editor_agent_tool_execution( tool_call_id, tool_name, tool_call.summary, response.0, )) } fn editor_agent_tool_request_context(base: &RequestContext, tool_call_id: &str) -> RequestContext { RequestContext::new( format!("{}:{tool_call_id}", base.request_id()), format!("{} editor-agent-tool", base.operation()), std::time::Duration::ZERO, false, ) } fn build_editor_agent_tool_execution( tool_call_id: &str, tool_name: EditorAgentToolName, summary: Option, response_body: Value, ) -> EditorAgentToolExecution { let data = response_body .get("data") .filter(|_| response_body.get("ok").and_then(Value::as_bool) == Some(true)) .unwrap_or(&response_body); let images = editor_agent_generated_images_from_response(tool_name, data); let task_id = data .get("taskId") .and_then(Value::as_str) .map(str::to_string) .or_else(|| { images .iter() .find_map(|image| image.resource_id.as_ref().map(|_| ())) .and_then(|_| { data.get("taskId") .and_then(Value::as_str) .map(str::to_string) }) }); let model = data .get("model") .and_then(Value::as_str) .map(str::to_string); let resolved_summary = summary.unwrap_or_else(|| editor_agent_tool_default_summary(tool_name)); EditorAgentToolExecution { record: EditorAgentGenerationRecord { tool_call_id: tool_call_id.to_string(), tool_name, summary: Some(resolved_summary.clone()), task_id, status: EditorAgentGenerationStatus::Completed, model, images, error: None, }, summary: resolved_summary, } } fn editor_agent_generated_images_from_response( tool_name: EditorAgentToolName, data: &Value, ) -> Vec { if tool_name == EditorAgentToolName::GenerateIconSpritesheet { let mut images = Vec::new(); if let Some(resource) = data.get("spritesheetResource") { images.push(editor_agent_generated_image_from_resource( resource, data.get("spritesheetImageSrc").and_then(Value::as_str), data.get("spritesheetWidth").and_then(Value::as_u64), data.get("spritesheetHeight").and_then(Value::as_u64), )); } if let Some(icon_images) = data.get("iconImageSrcs").and_then(Value::as_array) { for icon in icon_images { if let Some(resource) = icon.get("resource") { images.push(editor_agent_generated_image_from_resource( resource, icon.get("imageSrc").and_then(Value::as_str), icon.get("width").and_then(Value::as_u64), icon.get("height").and_then(Value::as_u64), )); } } } return images; } data.get("resource") .map(|resource| { vec![editor_agent_generated_image_from_resource( resource, data.get("imageSrc").and_then(Value::as_str), data.get("width").and_then(Value::as_u64), data.get("height").and_then(Value::as_u64), )] }) .unwrap_or_else(|| { data.get("imageSrc") .and_then(Value::as_str) .map(|image_src| { vec![EditorAgentGeneratedImage { resource_id: None, object_key: None, asset_object_id: None, image_src: image_src.to_string(), thumbnail_src: None, width: data .get("width") .and_then(Value::as_u64) .and_then(|value| u32::try_from(value).ok()), height: data .get("height") .and_then(Value::as_u64) .and_then(|value| u32::try_from(value).ok()), }] }) .unwrap_or_default() }) } fn editor_agent_generated_image_from_resource( resource: &Value, fallback_image_src: Option<&str>, fallback_width: Option, fallback_height: Option, ) -> EditorAgentGeneratedImage { EditorAgentGeneratedImage { resource_id: resource .get("resourceId") .and_then(Value::as_str) .map(str::to_string), object_key: resource .get("objectKey") .and_then(Value::as_str) .map(str::to_string), asset_object_id: resource .get("assetObjectId") .and_then(Value::as_str) .map(str::to_string), image_src: resource .get("imageSrc") .and_then(Value::as_str) .or(fallback_image_src) .unwrap_or("") .to_string(), thumbnail_src: resource .get("thumbnailSrc") .and_then(Value::as_str) .map(str::to_string), width: resource .get("width") .and_then(Value::as_u64) .or(fallback_width) .and_then(|value| u32::try_from(value).ok()), height: resource .get("height") .and_then(Value::as_u64) .or(fallback_height) .and_then(|value| u32::try_from(value).ok()), } } fn editor_agent_attachment_sources(attachments: &[EditorAgentAttachmentRef]) -> Vec { attachments .iter() .filter_map(|attachment| { attachment .object_key .as_deref() .map(str::trim) .filter(|value| !value.is_empty()) .map(|value| value.trim_start_matches('/').to_string()) .or_else(|| normalize_required_string(attachment.reference_id.as_str())) }) .collect() } fn editor_agent_tool_generation_reference_context( attachments: &[EditorAgentAttachmentRef], latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, previous_generation_source: Option<&str>, ) -> Vec { if previous_generation_source.is_some() { return latest_generation_reference .map(|reference| { vec![json!({ "title": reference .summary .clone() .unwrap_or_else(|| "上一张生成图".to_string()), "label": reference.summary, "refType": "previous-generation", "refId": reference.resource_id, "resourceId": reference.resource_id, "objectKey": reference.object_key, "assetObjectId": reference.asset_object_id, "toolCallId": reference.tool_call_id, "toolName": reference.tool_name, "sourceMessageId": reference.source_message_id, "source": previous_generation_source, "implicit": true, })] }) .unwrap_or_default(); } attachments .iter() .map(|attachment| { json!({ "title": attachment.label.clone().unwrap_or_else(|| "对话附件".to_string()), "label": attachment.label, "refType": match attachment.source { EditorAgentAttachmentSource::CanvasResource => "project-resource", EditorAgentAttachmentSource::LibraryAsset => "asset", }, "refId": attachment.reference_id, "resourceId": (attachment.source == EditorAgentAttachmentSource::CanvasResource) .then(|| attachment.reference_id.clone()), "objectKey": attachment.object_key, }) }) .collect() } fn editor_agent_tool_source_resource_id( attachments: &[EditorAgentAttachmentRef], latest_generation_reference: Option<&EditorAgentGeneratedImageReference>, previous_generation_source: Option<&str>, ) -> Option { if previous_generation_source.is_some() { return latest_generation_reference.and_then(|reference| reference.resource_id.clone()); } attachments .first() .map(|attachment| attachment.reference_id.clone()) } fn split_editor_agent_edit_sources(sources: Vec) -> Option<(String, Vec)> { let mut iter = sources.into_iter(); let first = iter.next()?; Some((first, iter.collect())) } fn editor_agent_tool_asset_kind(tool_name: EditorAgentToolName) -> &'static str { match tool_name { EditorAgentToolName::GenerateCharacter => "character", EditorAgentToolName::GenerateUiDesign => "ui-design", EditorAgentToolName::EditImage => "editor_agent_edit_image", EditorAgentToolName::GenerateIconSpritesheet => "icon-spritesheet", EditorAgentToolName::GenerateImage => "editor_agent_generated_image", } } fn editor_agent_default_asset_folder_id() -> String { "project".to_string() } fn editor_agent_tool_default_summary(tool_name: EditorAgentToolName) -> String { match tool_name { EditorAgentToolName::GenerateImage => "生成图片", EditorAgentToolName::EditImage => "修改图片", EditorAgentToolName::GenerateCharacter => "生成角色形象", EditorAgentToolName::GenerateIconSpritesheet => "生成图标素材", EditorAgentToolName::GenerateUiDesign => "生成 UI 设计图", } .to_string() } fn build_editor_agent_canvas_completion( project: &EditorProjectRecord, tool_name: EditorAgentToolName, title: &str, ) -> EditorCanvasGenerationCompletionPayload { let (width, height) = editor_agent_tool_display_size(tool_name); let (x, y) = next_editor_agent_canvas_position(project.layers.clone(), width, height); EditorCanvasGenerationCompletionPayload { dialog_id: None, title: normalize_optional_string(Some(title.to_string())) .unwrap_or_else(|| editor_agent_tool_default_summary(tool_name)), placeholder: EditorCanvasGenerationPlaceholderPayload { x, y, width, height, original_width: width, original_height: height, }, } } fn editor_agent_tool_display_size(tool_name: EditorAgentToolName) -> (f64, f64) { match tool_name { EditorAgentToolName::GenerateUiDesign => (640.0, 360.0), EditorAgentToolName::GenerateCharacter => (512.0, 768.0), _ => (512.0, 512.0), } } fn next_editor_agent_canvas_position(layers: Value, _width: f64, _height: f64) -> (f64, f64) { let mut max_right: Option = None; let mut min_y: Option = None; if let Value::Array(items) = layers { for item in items { if item.get("itemType").and_then(Value::as_str) == Some("generation-dialog") { continue; } let Some(x) = item.get("x").and_then(Value::as_f64) else { continue; }; let Some(y) = item.get("y").and_then(Value::as_f64) else { continue; }; let width = item.get("width").and_then(Value::as_f64).unwrap_or(0.0); if !x.is_finite() || !y.is_finite() || !width.is_finite() { continue; } max_right = Some(max_right.map_or(x + width, |value| value.max(x + width))); min_y = Some(min_y.map_or(y, |value| value.min(y))); } } ( max_right.map_or(0.0, |right| right + EDITOR_AGENT_CANVAS_RESULT_GAP), min_y.unwrap_or(0.0), ) } #[cfg(test)] fn build_echo_assistant_text(text: &str, attachment_count: usize) -> String { let trimmed = text.trim(); if trimmed.is_empty() { return format!("已收到 {attachment_count} 个附件。"); } format!("收到：{trimmed}") } fn now_rfc3339() -> String { shared_kernel::format_rfc3339(time::OffsetDateTime::now_utc()) .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_string()) } async fn require_editor_agent_sidebar_enabled( state: &AppState, owner_user_id: &str, ) -> Result<(), AppError> { match state .is_image_editor_agent_sidebar_enabled_for_user(Some(owner_user_id)) .await { Ok(true) => Ok(()), Ok(false) => Err(editor_agent_sidebar_unavailable()), Err(error) => Err(AppError::from_status(StatusCode::BAD_GATEWAY) .with_message("读取画布 Agent 灰度配置失败") .with_details(json!({ "provider": "spacetimedb", "message": error.to_string(), }))), } } fn editor_agent_sidebar_unavailable() -> AppError { AppError::from_status(StatusCode::SERVICE_UNAVAILABLE) .with_message("画布 Agent 暂不可用") .with_details(json!({ "provider": "editor-agent", "reason": "image_editor_agent_sidebar_disabled", "gateKey": module_runtime::IMAGE_EDITOR_AGENT_SIDEBAR_GATE_KEY, })) } fn editor_agent_bad_request(message: impl Into) -> AppError { AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ "provider": "editor-agent", "message": message.into(), })) } fn editor_agent_oss_unavailable() -> AppError { AppError::from_status(StatusCode::SERVICE_UNAVAILABLE).with_details(json!({ "provider": "aliyun-oss", "reason": "OSS is not configured for editor agent conversations", })) } fn editor_agent_oss_read_error(message: impl Into) -> AppError { AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({ "provider": "aliyun-oss", "message": message.into(), })) } fn editor_agent_messages_document_too_large() -> AppError { AppError::from_status(StatusCode::PAYLOAD_TOO_LARGE).with_details(json!({ "provider": "editor-agent", "message": "message document is too large", "maxBytes": EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES, })) } fn editor_agent_sse_json_event_or_error(event_name: &str, payload: T) -> Event where T: Serialize, { match serde_json::to_string(&payload) { Ok(data) => Event::default().event(event_name).data(data), Err(error) => Event::default().event("error").data( serde_json::to_string(&EditorAgentErrorEvent { conversation_id: None, code: "SERIALIZE_EVENT_FAILED".to_string(), message: error.to_string(), recoverable: false, }) .unwrap_or_else(|_| "{}".to_string()), ), } } #[cfg(test)] mod tests { use super::*; use crate::AppConfig; use platform_llm::LlmApiKind; #[test] fn echo_assistant_text_uses_attachment_fallback() { assert_eq!( build_echo_assistant_text(" 画一棵树 ", 0), "收到：画一棵树" ); assert_eq!(build_echo_assistant_text(" ", 2), "已收到 2 个附件。"); } #[test] fn empty_document_uses_contract_version() { let document = empty_messages_document("editor-agent-conv-1"); assert_eq!(document.version, EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION); assert_eq!(document.conversation_id, "editor-agent-conv-1"); assert!(document.messages.is_empty()); } #[test] fn canvas_attachment_normalization_uses_project_resource_truth() { let conversation = test_conversation_record(); let attachment = EditorAgentAttachmentRef { source: EditorAgentAttachmentSource::CanvasResource, reference_id: " resource-1 ".to_string(), object_key: Some("generated-editor-assets/resource-1.png".to_string()), image_src: "https://client.example/stale.png".to_string(), thumbnail_src: Some("https://client.example/thumb.png".to_string()), label: Some(" 参考图 ".to_string()), width: Some(1), height: Some(1), }; let resource = test_project_resource_record(); let normalized = normalize_canvas_resource_attachment(&conversation, &attachment, &resource) .expect("attachment should normalize"); assert_eq!(normalized.reference_id, "resource-1"); assert_eq!( normalized.object_key.as_deref(), Some("generated-editor-assets/resource-1.png") ); assert_eq!(normalized.image_src, "/api/assets/read/resource-1.png"); assert_eq!(normalized.thumbnail_src, None); assert_eq!(normalized.label.as_deref(), Some("参考图")); assert_eq!(normalized.width, Some(512)); assert_eq!(normalized.height, Some(256)); } #[test] fn attachment_normalization_rejects_mismatched_object_key() { let conversation = test_conversation_record(); let attachment = EditorAgentAttachmentRef { source: EditorAgentAttachmentSource::CanvasResource, reference_id: "resource-1".to_string(), object_key: Some("generated-editor-assets/other.png".to_string()), image_src: "/api/assets/read/resource-1.png".to_string(), thumbnail_src: None, label: None, width: None, height: None, }; let resource = test_project_resource_record(); assert!( normalize_canvas_resource_attachment(&conversation, &attachment, &resource).is_err() ); } #[test] fn library_attachment_normalization_uses_asset_label_fallback() { let attachment = EditorAgentAttachmentRef { source: EditorAgentAttachmentSource::LibraryAsset, reference_id: "asset-1".to_string(), object_key: None, image_src: "https://client.example/stale.png".to_string(), thumbnail_src: None, label: None, width: None, height: None, }; let asset = EditorAssetRecord { asset_id: "asset-1".to_string(), folder_id: "project".to_string(), label: "素材库图".to_string(), asset_object_id: Some("asset-object-1".to_string()), image_src: "/api/assets/read/asset-1.png".to_string(), object_key: Some("generated-editor-assets/asset-1.png".to_string()), width: 128, height: 128, source_type: "uploaded".to_string(), prompt: None, actual_prompt: None, model: None, provider: None, task_id: None, asset_kind: None, generation_inputs: None, source_resource_id: None, public_showcase_enabled: None, thumbnail_src: Some("/api/assets/read/asset-1-thumb.png".to_string()), generation_cost_mud_points: 0, showcase_id: None, showcase_review_status: None, showcase_display_enabled: None, showcase_like_count: None, created_at: "2026-07-03T00:00:00Z".to_string(), updated_at: "2026-07-03T00:00:00Z".to_string(), }; let normalized = normalize_library_asset_attachment(&attachment, &asset) .expect("library attachment should normalize"); assert_eq!(normalized.label.as_deref(), Some("素材库图")); assert_eq!( normalized.thumbnail_src.as_deref(), Some("/api/assets/read/asset-1-thumb.png") ); assert_eq!(normalized.width, Some(128)); assert_eq!(normalized.height, Some(128)); } #[test] fn generated_image_from_resource_keeps_private_asset_keys() { let image = editor_agent_generated_image_from_resource( &json!({ "resourceId": "resource-1", "imageSrc": "/generated-editor-assets/result.png", "thumbnailSrc": "/generated-editor-assets/result-thumb.png", "objectKey": "generated-editor-assets/result.png", "assetObjectId": "asset-object-result", "width": 1024, "height": 768 }), None, None, None, ); assert_eq!(image.resource_id.as_deref(), Some("resource-1")); assert_eq!( image.object_key.as_deref(), Some("generated-editor-assets/result.png") ); assert_eq!( image.asset_object_id.as_deref(), Some("asset-object-result") ); assert_eq!(image.width, Some(1024)); assert_eq!(image.height, Some(768)); } #[test] fn editor_agent_generations_target_default_asset_library_folder() { assert_eq!(editor_agent_default_asset_folder_id(), "project"); } #[test] fn editor_agent_turn_plan_parses_fenced_json_tool_call() { let plan = parse_editor_agent_turn_plan( r#"```json { "replyText": "我来生成一张像素风角色图。", "toolCall": { "toolName": "generate_character", "prompt": "像素风银发游侠", "summary": "生成像素角色", "imageSize": "1K" } } ```"#, ) .expect("fenced JSON should parse"); assert_eq!(plan.reply_text, "我来生成一张像素风角色图。"); let tool_call = plan.tool_call.expect("tool call should exist"); assert_eq!(tool_call.tool_name, EditorAgentToolName::GenerateCharacter); assert_eq!(tool_call.prompt, "像素风银发游侠"); assert_eq!(tool_call.summary.as_deref(), Some("生成像素角色")); assert_eq!(tool_call.image_size.as_deref(), Some("1K")); } #[test] fn editor_agent_turn_plan_rejects_invalid_llm_response() { assert!(parse_editor_agent_turn_plan("收到：画一棵树").is_none()); assert!( parse_editor_agent_turn_plan( r#"{"toolCall":{"toolName":"generate_image","prompt":"画一棵树"}}"#, ) .is_none() ); assert!( parse_editor_agent_turn_plan( r#"{"replyText":"我来生成图片。","toolCall":{"toolName":"generate_image"}}"#, ) .is_none() ); } #[test] fn editor_agent_turn_plan_replaces_structural_reply_fragment() { let plan = parse_editor_agent_turn_plan( r#"{"replyText":"{","toolCall":{"toolName":"generate_image","prompt":"生成一张角色规范图","summary":"生成角色规范图"}}"#, ) .expect("valid tool call should parse"); assert_eq!(plan.reply_text, "我来生成图片。"); assert_eq!( plan.tool_call.as_ref().map(|tool_call| tool_call.tool_name), Some(EditorAgentToolName::GenerateImage) ); } #[test] fn editor_agent_llm_system_prompt_describes_each_tool() { let prompt = editor_agent_llm_system_prompt(); for expected in [ "generate_image：从文字生成一张全新图片", "edit_image：修改已有图片", "generate_character：生成新的角色形象", "generate_icon_spritesheet：生成一组图标素材或图标图集", "generate_ui_design：生成一张完整 UI 设计图或界面稿", "规范图/视觉规范图/风格规范图/素材规范展板", "当前使用 generate_image", "角色规范图/角色美术视觉规范设定图", "图标规范图/图标视觉规范展板", "统一视角、线条粗细", "色卡/色号", "工具选择优先级", "明确修改/指代已有图 => edit_image", "规范图/视觉规范图/风格规范图/素材规范展板 => generate_image", ] { assert!( prompt.contains(expected), "system prompt should contain {expected}" ); } } #[test] fn latest_generation_reference_enters_llm_prompt() { let conversation = test_conversation_record(); let user_message = test_user_message("把刚才那个衣服换成蓝色"); let mut document = empty_messages_document(conversation.conversation_id.as_str()); document.messages.push(EditorAgentMessage { id: "editor-agent-message-assistant-1".to_string(), role: EditorAgentMessageRole::Assistant, kind: EditorAgentMessageKind::Chat, text: "已生成角色图。".to_string(), attachments: Vec::new(), generations: vec![test_completed_generation_record()], status: EditorAgentMessageStatus::Completed, created_at: "2026-07-05T00:00:00Z".to_string(), }); document.messages.push(user_message.clone()); let reference = latest_editor_agent_generated_image_reference(&document) .expect("latest generated image should be available"); let prompt = build_editor_agent_llm_user_prompt( &conversation, &document, &user_message, Some(&reference), ); let prompt_json: Value = serde_json::from_str(prompt.as_str()).expect("prompt is JSON"); assert_eq!( prompt_json["latestGeneratedImage"]["toolName"], json!("generate_image") ); assert_eq!( prompt_json["latestGeneratedImage"]["summary"], json!("生成红衣角色") ); assert_eq!( prompt_json["latestGeneratedImage"]["resourceId"], json!("resource-generated-1") ); assert_eq!( prompt_json["latestGeneratedImage"]["objectKey"], json!("generated-editor-assets/result.png") ); assert_eq!( prompt_json["recentMessages"][0]["generations"][0]["images"][0]["objectKey"], json!("generated-editor-assets/result.png") ); } #[test] fn previous_generation_edit_default_coerces_generate_image_to_edit_image() { let user_message = test_user_message("把衣服换成蓝色"); let reference = test_generated_image_reference(); let plan = EditorAgentTurnPlan { reply_text: "我来生成图片。".to_string(), tool_call: Some(EditorAgentToolCallPlan { tool_name: EditorAgentToolName::GenerateImage, prompt: "把衣服换成蓝色".to_string(), summary: Some("生成蓝色衣服".to_string()), model: None, aspect_ratio: Some("1:1".to_string()), image_size: Some("1K".to_string()), icon_descriptions: Vec::new(), }), }; let adjusted = apply_previous_generation_edit_default(plan, &user_message, Some(&reference)); let tool_call = adjusted.tool_call.expect("tool should be forced"); assert_eq!(tool_call.tool_name, EditorAgentToolName::EditImage); assert_eq!(tool_call.prompt, "把衣服换成蓝色"); assert_eq!(adjusted.reply_text, "我会基于上一张生成图按你的要求修改。"); } #[test] fn previous_generation_reference_becomes_implicit_edit_source() { let reference = test_generated_image_reference(); let sources = editor_agent_generated_reference_source(&reference); let context = editor_agent_tool_generation_reference_context( &[], Some(&reference), sources.as_deref(), ); assert_eq!( sources.as_deref(), Some("generated-editor-assets/result.png") ); assert_eq!(context[0]["refType"], json!("previous-generation")); assert_eq!(context[0]["resourceId"], json!("resource-generated-1")); assert_eq!( context[0]["objectKey"], json!("generated-editor-assets/result.png") ); assert_eq!(context[0]["implicit"], json!(true)); } #[test] fn editor_agent_tool_execution_persists_summary() { let execution = build_editor_agent_tool_execution( "editor-agent-tool-1", EditorAgentToolName::EditImage, Some("换蓝色衣服".to_string()), json!({ "ok": true, "data": { "model": "gpt-image-2", "resource": { "resourceId": "resource-generated-2", "imageSrc": "/generated-editor-assets/blue.png", "objectKey": "generated-editor-assets/blue.png", "assetObjectId": "asset-object-blue", "width": 1024, "height": 1024 } } }), ); assert_eq!(execution.summary, "换蓝色衣服"); assert_eq!(execution.record.summary.as_deref(), Some("换蓝色衣服")); } #[tokio::test] async fn editor_agent_llm_request_uses_vector_engine_chat_model() { let conversation = test_conversation_record(); let document = empty_messages_document(conversation.conversation_id.as_str()); let user_message = test_user_message("帮我看看这张图还能怎么改"); let state = AppState::new(AppConfig::default()).expect("state should build"); let request = build_editor_agent_llm_request(&state, &conversation, &document, &user_message, None) .await; assert_eq!(request.api_kind, LlmApiKind::OpenAiChat); assert_eq!( request.model.as_deref(), Some(platform_agent::CREATIVE_AGENT_GPT5_MODEL) ); assert_eq!( request.max_output_tokens, Some(EDITOR_AGENT_LLM_MAX_OUTPUT_TOKENS) ); assert_eq!( request.request_timeout_ms, Some(EDITOR_AGENT_LLM_REQUEST_TIMEOUT_MS) ); } #[test] fn editor_agent_heuristic_routes_tools_from_user_text() { let character_plan = heuristic_editor_agent_turn_plan("帮我生成一个角色形象：猫耳骑士", &[]); assert_eq!( character_plan .tool_call .as_ref() .expect("character tool expected") .tool_name, EditorAgentToolName::GenerateCharacter ); let edit_plan = heuristic_editor_agent_turn_plan( "把附件这张图改成赛博朋克风格", &[EditorAgentAttachmentRef { source: EditorAgentAttachmentSource::CanvasResource, reference_id: "resource-1".to_string(), object_key: Some("generated-editor-assets/resource-1.png".to_string()), image_src: "/generated-editor-assets/resource-1.png".to_string(), thumbnail_src: None, label: None, width: Some(512), height: Some(512), }], ); assert_eq!( edit_plan .tool_call .as_ref() .expect("edit tool expected") .tool_name, EditorAgentToolName::EditImage ); let chat_plan = heuristic_editor_agent_turn_plan("这张图适合做头像吗？", &[]); assert!(chat_plan.tool_call.is_none()); } #[test] fn editor_agent_canvas_completion_places_result_after_existing_layers() { let project = EditorProjectRecord { project_id: "project-1".to_string(), owner_user_id: "user-1".to_string(), title: "画布".to_string(), canvas: spacetime_client::EditorCanvasRecord { canvas_id: "canvas-1".to_string(), project_id: "project-1".to_string(), title: "画布".to_string(), viewport: spacetime_client::EditorCanvasViewportRecord { x: 0.0, y: 0.0, scale: 1.0, }, layers: json!([]), created_at: "2026-07-03T00:00:00Z".to_string(), updated_at: "2026-07-03T00:00:00Z".to_string(), }, viewport: spacetime_client::EditorCanvasViewportRecord { x: 0.0, y: 0.0, scale: 1.0, }, layers: json!([ { "layerId": "layer-1", "resourceId": "resource-1", "x": 10.0, "y": 20.0, "width": 180.0, "height": 120.0 } ]), resources: Vec::new(), created_at: "2026-07-03T00:00:00Z".to_string(), updated_at: "2026-07-03T00:00:00Z".to_string(), }; let completion = build_editor_agent_canvas_completion( &project, EditorAgentToolName::GenerateUiDesign, "UI 设计图", ); assert_eq!(completion.dialog_id, None); assert_eq!(completion.title, "UI 设计图"); assert_eq!(completion.placeholder.x, 222.0); assert_eq!(completion.placeholder.y, 20.0); assert_eq!(completion.placeholder.width, 640.0); assert_eq!(completion.placeholder.height, 360.0); } fn test_conversation_record() -> EditorAgentConversationRecord { EditorAgentConversationRecord { conversation_id: "editor-agent-conv-1".to_string(), project_id: "project-1".to_string(), owner_user_id: "user-1".to_string(), title: "新对话".to_string(), messages_object_key: "editor-agent/editor-agent-conv-1.json".to_string(), deleted: false, created_at: "2026-07-03T00:00:00Z".to_string(), updated_at: "2026-07-03T00:00:00Z".to_string(), updated_at_micros: 1, } } fn test_project_resource_record() -> EditorProjectResourceRecord { EditorProjectResourceRecord { resource_id: "resource-1".to_string(), project_id: "project-1".to_string(), owner_user_id: "user-1".to_string(), asset_object_id: Some("asset-object-1".to_string()), image_src: "/api/assets/read/resource-1.png".to_string(), object_key: Some("generated-editor-assets/resource-1.png".to_string()), width: 512, height: 256, source_type: "generated".to_string(), prompt: Some("森林".to_string()), actual_prompt: None, model: Some("gpt-image-2".to_string()), provider: Some("VectorEngine".to_string()), task_id: Some("task-1".to_string()), source_resource_id: None, asset_kind: Some("editor_generated_image".to_string()), generation_inputs: None, public_showcase_enabled: false, created_at: "2026-07-03T00:00:00Z".to_string(), updated_at: "2026-07-03T00:00:00Z".to_string(), } } fn test_user_message(text: &str) -> EditorAgentMessage { EditorAgentMessage { id: "editor-agent-message-user-1".to_string(), role: EditorAgentMessageRole::User, kind: EditorAgentMessageKind::Chat, text: text.to_string(), attachments: Vec::new(), generations: Vec::new(), status: EditorAgentMessageStatus::Completed, created_at: "2026-07-05T00:00:00Z".to_string(), } } fn test_completed_generation_record() -> EditorAgentGenerationRecord { EditorAgentGenerationRecord { tool_call_id: "editor-agent-tool-generated-1".to_string(), tool_name: EditorAgentToolName::GenerateImage, summary: Some("生成红衣角色".to_string()), task_id: Some("task-generated-1".to_string()), status: EditorAgentGenerationStatus::Completed, model: Some("gpt-image-2".to_string()), images: vec![EditorAgentGeneratedImage { resource_id: Some("resource-generated-1".to_string()), object_key: Some("generated-editor-assets/result.png".to_string()), asset_object_id: Some("asset-object-result".to_string()), image_src: "/generated-editor-assets/result.png".to_string(), thumbnail_src: Some("/generated-editor-assets/result-thumb.png".to_string()), width: Some(1024), height: Some(1024), }], error: None, } } fn test_generated_image_reference() -> EditorAgentGeneratedImageReference { let mut document = empty_messages_document("editor-agent-conv-1"); document.messages.push(EditorAgentMessage { id: "editor-agent-message-assistant-1".to_string(), role: EditorAgentMessageRole::Assistant, kind: EditorAgentMessageKind::Chat, text: "已生成角色图。".to_string(), attachments: Vec::new(), generations: vec![test_completed_generation_record()], status: EditorAgentMessageStatus::Completed, created_at: "2026-07-05T00:00:00Z".to_string(), }); latest_editor_agent_generated_image_reference(&document) .expect("latest generated image should be available") } }