From f774a791e832b54c3b9352c1e6888e3425e00285 Mon Sep 17 00:00:00 2001 From: kvtodev Date: Mon, 13 Jul 2026 14:24:24 +0800 Subject: [PATCH] =?UTF-8?q?=E7=94=BB=E5=B8=83Agent=E5=85=B3=E8=81=94?= =?UTF-8?q?=E5=A4=96=E9=83=A8=E7=94=9F=E6=88=90=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 工具消息改为保存外部任务 ID 与取消事实，不再复制执行状态。 确认请求以稳定身份提交外部任务，并由前端查询任务状态和结果。 同步更新画布 Agent 测试与执行契约文档。 --- .../shared-memory/decision-log.md | 6 + .../【编辑器】画布Agent对话面板-2026-07-03.md | 6 +- packages/shared/src/contracts/editorAgent.ts | 10 +- .../crates/api-server/src/editor_agent/api.rs | 1015 ++++++----------- .../shared-contracts/src/editor_agent.rs | 42 +- .../EditorAgentConversationPanelView.test.tsx | 5 +- .../EditorAgentConversationPanelView.tsx | 1 + .../EditorAgentConversation/MessageBubble.tsx | 143 ++- .../useEditorAgentConversation.test.tsx | 53 +- .../useEditorAgentConversation.ts | 33 +- .../ImageCanvasEditorView.test.tsx | 8 +- 11 files changed, 587 insertions(+), 735 deletions(-) diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 7e262446d..0b571ecd3 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -3104,6 +3104,12 @@ - 影响范围：`server-rs/crates/api-server/src/editor_generation_queue.rs`、`server-rs/crates/api-server/src/external_generation_worker.rs`、`server-rs/crates/api-server/src/editor_project.rs`、`server-rs/crates/api-server/src/character_animation_assets.rs`、`server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs`、`src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.ts`、`src/services/image-editor/editorProjectClient.ts`。 - 验证方式：`cargo test -p api-server external_generation_worker --manifest-path server-rs/Cargo.toml`、`cargo test -p api-server editor_canvas_generation --manifest-path server-rs/Cargo.toml`、`cargo test -p shared-contracts --manifest-path server-rs/Cargo.toml`、`npm run test -- src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx src/services/image-editor/editorProjectClient.test.ts`、`npm run typecheck`、`npm run check:encoding`、`git diff --check`。 +## 2026-07-13 画布 Agent 工具执行状态复用外部生成任务 + +- 决策：画布 Agent 的 OSS 工具消息删除独立 `status`，只保存可选 `externalJobId`、确认前取消事实 `cancelledAt` 和消息展示结果；`external_generation_job` 是排队、执行、完成、失败、lease 与计费结算的唯一真相。确认接口按 `conversationId + messageId + toolName` 稳定去重并复用既有编辑器 worker job kind，不新增 `editor_agent_tool_execution` 或其它关联表。 +- 影响范围：画布 Agent 共享契约、确认/取消接口、编辑器生成入队 helper、对话状态展示与恢复。 +- 验证方式：`cargo check -p api-server -p shared-contracts --manifest-path server-rs/Cargo.toml`、画布 Agent 定向前端测试、`npm run typecheck`、`npm run check:encoding`、`git diff --check`。 + ## 2026-06-18 原生壳 WebView 刷新能力只保留受控当前页刷新 - 背景：Expo 移动壳和 Tauri 桌面壳都需要一个真实的宿主级刷新入口，供 H5 在检测到资源、登录态或运行态需要重新载入时请求宿主刷新当前容器；该能力不能演变成任意 URL 导航或原生 WebView ref 透传。 diff --git a/docs/【编辑器】画布Agent对话面板-2026-07-03.md b/docs/【编辑器】画布Agent对话面板-2026-07-03.md index 0d7822ab5..8e31e7744 100644 --- a/docs/【编辑器】画布Agent对话面板-2026-07-03.md +++ b/docs/【编辑器】画布Agent对话面板-2026-07-03.md @@ -73,7 +73,11 @@ ## 工具调用确认展示契约 -- Agent 规划出生成或编辑工具后，先把工具消息持久化为 `pending_confirmation`；确认卡必须在真正调用生成 provider 前展示本次提示词、规格参数、目标图和参考图缩略图，用户确认后才执行，取消后保留同一条已取消记录。内部 system 文本和图片哈希 ID 不直接展示给用户。 +- Agent 规划出生成或编辑工具后，先把没有 `externalJobId` / `cancelledAt` 的工具消息持久化为待确认记录；确认卡必须在真正调用生成 provider 前展示本次提示词、规格参数、目标图和参考图缩略图，用户确认后才执行，取消后保留同一条已取消记录。内部 system 文本和图片哈希 ID 不直接展示给用户。 +- 工具消息不再保存独立 `status`。OSS 文档只保存可选 `externalJobId` 与确认前取消时间 `cancelledAt`：两者都为空表示待确认，只有 `cancelledAt` 表示已取消；存在 `externalJobId` 时，排队、执行、完成和失败状态统一读取 SpacetimeDB `external_generation_job`，不得在 OSS 中复制第二套执行状态。 +- 确认接口必须先把工具参数转换为既有编辑器 worker payload，再使用 `editor-agent:{conversationId}:{messageId}:{toolName}` 稳定 dedupe key 入队；同一确认的请求重试只能得到同一个 external job。入队成功后把返回的 job id 写回同一条 OSS 工具消息，不新增 Agent 工具执行关联表。 +- 前端根据 `externalJobId` 查询通用 external-generation job 状态；worker 继续通过 `canvasCompletion` 把生成结果写回工程与素材库。浏览器断线、刷新或 api-server 重启不得导致确认接口重新扣费或重新提交 provider。 +- worker 成功时把既有编辑器生成回包写入该 job 的 `result_payload_json`；通用 job 状态接口只读返回其中的 `result` 投影，Agent 卡据此恢复图片、视频或音频预览。该投影属于 SpacetimeDB job，不回写 OSS 对话文档。 - `EditorAgentToolCall.args` 保留为工具返回的原始 JSON，是确认接口重新反序列化并执行工具的唯一参数真相。图片参数继续只保存由真实 data key 计算出的 opaque SHA-256 `imageId`；不得为了前端预览把 `args` 中的图片 ID 改写成 `objectKey`、URL 或展示对象，也不得由前端重组或回传一份新的执行参数。 - `EditorAgentToolCall.displayArgs` 是必填、只读的用户确认展示投影，与 `args` 分离： - `stringArgs` 保存提示词、比例、清晰度、模型、时长等可展示参数的稳定名称、用户可见标题和值； diff --git a/packages/shared/src/contracts/editorAgent.ts b/packages/shared/src/contracts/editorAgent.ts index 3bec78949..b91c3a39a 100644 --- a/packages/shared/src/contracts/editorAgent.ts +++ b/packages/shared/src/contracts/editorAgent.ts @@ -7,13 +7,6 @@ export const EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION = 1; export type EditorAgentMessageRole = 'user' | 'assistant' | 'system'; -export type EditorAgentToolCallStatus = - | 'pending_confirmation' - | 'executing' - | 'completed' - | 'cancelled' - | 'failed'; - export type EditorAgentAttachmentSource = 'canvas_resource' | 'library_asset'; export interface EditorAgentAttachmentRef { @@ -88,9 +81,10 @@ export interface EditorAgentToolCallDisplayArgs { export interface EditorAgentToolCall { toolName: string; - status: EditorAgentToolCallStatus; args: unknown; displayArgs: EditorAgentToolCallDisplayArgs; + externalJobId?: string | null; + cancelledAt?: string | null; images: EditorAgentGeneratedImage[]; // Older persisted conversation documents do not contain these media fields. videos?: EditorAgentGeneratedVideo[]; diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index 4ed3319dd..fd322e744 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -14,14 +14,18 @@ use module_editor_agent::{ use platform_llm::LlmMessage; use serde::Serialize; use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use shared_contracts::assets::{ + EditorBackgroundMusicGenerateRequest, EditorSoundEffectGenerateRequest, + EditorVideoGenerateRequest, +}; use shared_contracts::editor_agent::{ CreateEditorAgentConversationRequest, EditorAgentConversationListResponse, EditorAgentConversationMessagesDocument, EditorAgentConversationResponse, - EditorAgentConversationSummary, EditorAgentGeneratedAudio, EditorAgentGeneratedImage, - EditorAgentGeneratedVideo, EditorAgentMessage, EditorAgentMessageRequest, + EditorAgentConversationSummary, EditorAgentMessage, EditorAgentMessageRequest, EditorAgentMessageResponse, EditorAgentMessageRole, EditorAgentToolCall, EditorAgentToolCallDisplayArgs, EditorAgentToolCallDisplayExtras, EditorAgentToolCallImageArg, - EditorAgentToolCallImageRef, EditorAgentToolCallStatus, EditorAgentToolCallStringArg, + EditorAgentToolCallImageRef, EditorAgentToolCallStringArg, }; use spacetime_client::{ EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput, @@ -41,9 +45,7 @@ use crate::editor_agent::editor_tools::generate_character::GenerateCharacterTool use crate::editor_agent::editor_tools::generate_icon_spritesheet::{ GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, }; -use crate::editor_agent::editor_tools::generate_image::{ - EditorImageGenerationResult, GenerateImageTool, GenerateImageToolArgs, -}; +use crate::editor_agent::editor_tools::generate_image::{GenerateImageTool, GenerateImageToolArgs}; use crate::editor_agent::editor_tools::generate_sound_effect::{ GenerateSoundEffectTool, GenerateSoundEffectToolArgs, }; @@ -57,7 +59,16 @@ use crate::editor_agent::utils::{ write_messages_document, }; use crate::editor_generation_config::EditorGenerationPricingConfig; -use crate::editor_project::{EditorGenerationCaller, current_utc_micros, map_editor_project_error}; +use crate::editor_generation_queue::{ + EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND, + EDITOR_IMAGE_EDIT_JOB_KIND, EDITOR_IMAGE_GENERATION_JOB_KIND, + EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND, EDITOR_VIDEO_GENERATION_JOB_KIND, + enqueue_editor_generation_job_with_identity, +}; +use crate::editor_project::{ + EditorIconSpritesheetGenerationRequest, EditorImageEditRequest, EditorImageGenerationRequest, +}; +use crate::editor_project::{current_utc_micros, map_editor_project_error}; use crate::http_error::AppError; use crate::request_context::RequestContext; use crate::state::AppState; @@ -272,9 +283,10 @@ fn build_delta_messages( attachments: Vec::new(), tool_call: Some(EditorAgentToolCall { tool_name: tco.tool_call.name, - status: EditorAgentToolCallStatus::PendingConfirmation, args: tco.tool_call.args, display_args, + external_job_id: None, + cancelled_at: None, images: Vec::new(), videos: Vec::new(), audios: Vec::new(), @@ -624,7 +636,6 @@ mod tool_context_tests { use shared_contracts::editor_agent::{ EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, EditorAgentAttachmentRef, EditorAgentAttachmentSource, EditorAgentGeneratedImage, EditorAgentMessageRole, - EditorAgentToolCallStatus, }; #[test] @@ -640,9 +651,10 @@ mod tool_context_tests { attachments: Vec::new(), tool_call: Some(EditorAgentToolCall { tool_name: "generate-image".to_string(), - status: EditorAgentToolCallStatus::Completed, args: json!({}), display_args: EditorAgentToolCallDisplayArgs::default(), + external_job_id: Some("task-existing".to_string()), + cancelled_at: None, images: vec![EditorAgentGeneratedImage { resource_id: None, object_key: Some(data_key.to_string()), @@ -929,14 +941,13 @@ pub async fn cancel_editor_agent_tool_call( .tool_call .as_mut() .ok_or_else(|| editor_agent_bad_request("message has no tool call"))?; - if tc.status != EditorAgentToolCallStatus::PendingConfirmation { + if tc.external_job_id.is_some() || tc.cancelled_at.is_some() { return Err(editor_agent_bad_request( - "tool call is not in PendingConfirmation status", + "tool call is no longer pending confirmation", )); } - // Mark as cancelled - tc.status = EditorAgentToolCallStatus::Cancelled; + tc.cancelled_at = Some(now_rfc3339()); let arg_json = tc.args.to_string(); msg.text = format!( "[tool_call:{tool_name}] args: {arg_json} output: 用户已取消该操作", @@ -960,528 +971,384 @@ pub async fn confirm_editor_agent_tool_call( ) -> Result, AppError> { let owner_user_id = authenticated.claims().user_id().to_string(); require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?; - let conversation = state .spacetime_client() .get_editor_agent_conversation(conversation_id, owner_user_id) .await - .map_err(|e| { - AppError::from_status(axum::http::StatusCode::NOT_FOUND) - .with_details(json!({ "message": format!("conversation not found: {e}") })) + .map_err(|error| { + AppError::from_status(axum::http::StatusCode::NOT_FOUND).with_details(json!({ + "message": format!("conversation not found: {error}"), + })) })?; - let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock( conversation.conversation_id.as_str(), ); let _conversation_lock_guard = conversation_lock.lock_owned().await; - - let mut document: EditorAgentConversationMessagesDocument = - read_messages_document(&state, &conversation).await?; - - // Validate message index - if message_id >= document.messages.len() { - return Err(AppError::from_status(axum::http::StatusCode::NOT_FOUND) - .with_details(json!({ "message": "message not found" }))); - } - - let msg = &mut document.messages[message_id]; - - // A persisted executing state prevents a repeated confirmation from starting a second job. - if msg.role != EditorAgentMessageRole::System { + let mut document = read_messages_document(&state, &conversation).await?; + let message = document + .messages + .get(message_id) + .ok_or_else(|| AppError::from_status(axum::http::StatusCode::NOT_FOUND))?; + if message.role != EditorAgentMessageRole::System { return Err(editor_agent_bad_request("message is not a system message")); } - let tc = msg + let tool_call = message .tool_call - .as_mut() + .as_ref() .ok_or_else(|| editor_agent_bad_request("message has no tool call"))?; - if tc.status != EditorAgentToolCallStatus::PendingConfirmation { - return Err(editor_agent_bad_request( - "tool call is not in PendingConfirmation status", - )); + if tool_call.cancelled_at.is_some() { + return Err(editor_agent_bad_request("tool call was cancelled")); + } + if tool_call.external_job_id.is_some() { + return Ok(json_success_body(Some(&request_context), message)); } - let tool_name = tc.tool_name.clone(); - let tool_args = tc.args.clone(); + let tool_name = tool_call.tool_name.clone(); + let tool_args = tool_call.args.clone(); + let pricing = state.editor_generation_pricing().await.map_err(|error| { + AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ + "provider": "editor-generation-pricing", + "message": error.to_string(), + })) + })?; + let project = load_editor_agent_project(&state, &conversation).await?; + let context = build_tool_context(&document); + let generation_inputs = |title: &str, value: &str| { + Some(editor_agent_media_generation_inputs( + &conversation, + message_id, + title, + value, + )) + }; - // Each confirmed tool reuses its existing editor-generation entry point. - match tool_name.as_str() { - GenerateSoundEffectTool::NAME => { - let args: GenerateSoundEffectToolArgs = serde_json::from_value(tool_args.clone()) - .map_err(|error| { - editor_agent_bad_request(format!("invalid tool call args: {error}")) - })?; - let project = load_editor_agent_project(&state, &conversation).await?; - let title = args.prompt.clone(); - let tool = GenerateSoundEffectTool; - persist_editor_agent_tool_call_executing( - &state, - &conversation, - &mut document, - message_id, - ) - .await?; - let result = tool - .execute( - &state, - &request_context, - conversation.owner_user_id.clone(), - args.clone(), - conversation.project_id.clone(), - Some(editor_agent_media_generation_inputs( - &conversation, - message_id, - "用户指令", - &args.prompt, - )), - Some(title.clone()), - build_editor_agent_canvas_completion( - &project, - GenerateSoundEffectTool::NAME, - &title, - ), - ) - .await?; - complete_editor_agent_media_tool_call( - &state, - &request_context, - &conversation, - &mut document, - message_id, - tool.format_execute_message(&args, &result), - Vec::new(), - Vec::new(), - vec![editor_agent_generated_audio( - result.resource.as_ref(), - result.object_key, - result.asset_object_id, - result.audio_src, - )], - ) - .await - } - GenerateBackgroundMusicTool::NAME => { - let args: GenerateBackgroundMusicToolArgs = serde_json::from_value(tool_args.clone()) - .map_err(|error| { + let (job_kind, request_label, price_mud_points, payload) = match tool_name.as_str() { + GenerateImageTool::NAME | GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => { + let args: GenerateImageToolArgs = parse_confirm_tool_args(&tool_args)?; + let tool = GenerateImageTool { + context: context.clone(), + }; + tool.validate_args(&args).map_err(|error| { editor_agent_bad_request(format!("invalid tool call args: {error}")) })?; - let project = load_editor_agent_project(&state, &conversation).await?; - let title = "生成背景音乐".to_string(); - let tool = GenerateBackgroundMusicTool; - persist_editor_agent_tool_call_executing( - &state, - &conversation, - &mut document, - message_id, - ) - .await?; - let result = tool - .execute( - &state, - &request_context, - conversation.owner_user_id.clone(), - args.clone(), - conversation.project_id.clone(), - Some(editor_agent_media_generation_inputs( - &conversation, - message_id, - "用户指令", - &args.prompt, - )), - Some(title.clone()), - build_editor_agent_canvas_completion( - &project, - GenerateBackgroundMusicTool::NAME, - &title, - ), - ) - .await?; - complete_editor_agent_media_tool_call( - &state, - &request_context, - &conversation, - &mut document, - message_id, - tool.format_execute_message(&args, &result), - Vec::new(), - Vec::new(), - vec![editor_agent_generated_audio( - result.resource.as_ref(), - result.object_key, - result.asset_object_id, - result.audio_src, - )], - ) - .await - } - GenerateVideoTool::NAME => { - let args: GenerateVideoToolArgs = - serde_json::from_value(tool_args.clone()).map_err(|error| { - editor_agent_bad_request(format!("invalid tool call args: {error}")) - })?; - let project = load_editor_agent_project(&state, &conversation).await?; + let kind = match tool_name.as_str() { + GenerateCharacterTool::NAME => Some("character"), + GenerateUiDesignTool::NAME => Some("ui-design"), + _ => None, + }; + let price = match kind { + Some("character") => GenerateCharacterTool { + context: context.clone(), + } + .pricing(&pricing, &args), + Some("ui-design") => GenerateUiDesignTool { + context: context.clone(), + } + .pricing(&pricing, &args), + _ => tool.pricing(&pricing, &args), + }; let title = args.prompt.clone(); - let tool = GenerateVideoTool { - context: build_tool_context(&document), + let reference_image_srcs = args + .reference_image_ids + .iter() + .map(|image_id| { + context + .image_data_key(image_id) + .map(ToOwned::to_owned) + .ok_or_else(|| { + editor_agent_bad_request(format!("image {image_id} not found")) + }) + }) + .collect::, _>>()?; + let payload = EditorImageGenerationRequest { + prompt: args.prompt, + size: None, + kind: kind.map(ToOwned::to_owned), + model: None, + screen_color: (kind == Some("character")).then(|| "auto".to_string()), + seg_model: (kind == Some("character")).then(|| "birefnet".to_string()), + aspect_ratio: args.aspect_ratio, + image_size: args.image_size, + reference_image_srcs: Some(reference_image_srcs), + project_id: Some(conversation.project_id.clone()), + asset_kind: Some(kind.unwrap_or("editor_agent_generated_image").to_string()), + generation_inputs: editor_agent_generation_inputs( + &conversation, + message_id, + &GenerateImageToolArgs { + prompt: title.clone(), + reference_image_ids: Vec::new(), + aspect_ratio: None, + image_size: None, + }, + ), + asset_folder_id: Some("project".to_string()), + asset_label: Some(title.clone()), + source_resource_id: None, + canvas_completion: Some(build_editor_agent_canvas_completion( + &project, &tool_name, &title, + )), }; - persist_editor_agent_tool_call_executing( - &state, - &conversation, - &mut document, - message_id, + ( + EDITOR_IMAGE_GENERATION_JOB_KIND, + "画布 Agent 生成图片", + price, + serde_json::to_value(payload), ) - .await?; - let result = tool - .execute( - &state, - &request_context, - conversation.owner_user_id.clone(), - args.clone(), - conversation.project_id.clone(), - Some(editor_agent_media_generation_inputs( - &conversation, - message_id, - "用户指令", - &args.prompt, - )), - Some(title.clone()), - build_editor_agent_canvas_completion(&project, GenerateVideoTool::NAME, &title), - ) - .await?; - complete_editor_agent_media_tool_call( - &state, - &request_context, - &conversation, - &mut document, - message_id, - tool.format_execute_message(&args, &result), - Vec::new(), - vec![editor_agent_generated_video( - result.resource.as_ref(), - result.object_key, - result.asset_object_id, - result.video_src, - result.thumbnail_src, - result.width, - result.height, - )], - Vec::new(), - ) - .await - } - GenerateIconSpritesheetTool::NAME => { - let args = serde_json::from_value::(tool_args.clone()) - .map_err(|e| editor_agent_bad_request(format!("invalid tool call args: {e}")))?; - let project = load_editor_agent_project(&state, &conversation).await?; - let title = "图标素材图集".to_string(); - let canvas_completion = build_editor_agent_canvas_completion( - &project, - GenerateIconSpritesheetTool::NAME, - &title, - ); - let tool = GenerateIconSpritesheetTool { - context: build_tool_context(&document), - }; - persist_editor_agent_tool_call_executing( - &state, - &conversation, - &mut document, - message_id, - ) - .await?; - let result = tool - .execute( - &state, - &request_context, - editor_agent_generation_caller(&conversation), - args.clone(), - conversation.project_id.clone(), - Some(json!({ - "source": "editor-agent", - "conversationId": conversation.conversation_id, - "toolCallMessageId": message_id, - "fields": [{ "title": "图标描述", "value": args.icon_descriptions }], - })), - canvas_completion, - ) - .await?; - let image = EditorAgentGeneratedImage { - resource_id: result - .spritesheet_resource - .as_ref() - .and_then(|resource| resource.get("resourceId")) - .and_then(Value::as_str) - .map(ToOwned::to_owned), - object_key: result - .spritesheet_resource - .as_ref() - .and_then(|resource| resource.get("objectKey")) - .and_then(Value::as_str) - .map(ToOwned::to_owned), - asset_object_id: result - .spritesheet_resource - .as_ref() - .and_then(|resource| resource.get("assetObjectId")) - .and_then(Value::as_str) - .map(ToOwned::to_owned), - image_src: result.spritesheet_image_src.clone(), - thumbnail_src: None, - width: Some(result.spritesheet_width), - height: Some(result.spritesheet_height), - }; - complete_editor_agent_tool_call( - &state, - &request_context, - &conversation, - &mut document, - message_id, - tool.format_execute_message(&args, &result), - vec![image], - ) - .await - } - GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => { - let args: GenerateImageToolArgs = serde_json::from_value(tool_args.clone()) - .map_err(|e| editor_agent_bad_request(format!("invalid tool call args: {e}")))?; - let project = load_editor_agent_project(&state, &conversation).await?; - let caller = editor_agent_generation_caller(&conversation); - let title = args.prompt.clone(); - let canvas_completion = - build_editor_agent_canvas_completion(&project, tool_name.as_str(), &title); - let generation_inputs = - editor_agent_generation_inputs(&conversation, message_id, &args); - persist_editor_agent_tool_call_executing( - &state, - &conversation, - &mut document, - message_id, - ) - .await?; - let result = if tool_name == GenerateCharacterTool::NAME { - GenerateCharacterTool { - context: build_tool_context(&document), - } - .execute( - &state, - &request_context, - caller, - args.clone(), - conversation.project_id.clone(), - generation_inputs, - Some(title), - canvas_completion, - ) - .await? - } else { - GenerateUiDesignTool { - context: build_tool_context(&document), - } - .execute( - &state, - &request_context, - caller, - args.clone(), - conversation.project_id.clone(), - generation_inputs, - Some(title), - canvas_completion, - ) - .await? - }; - let formatted_text = if tool_name == GenerateCharacterTool::NAME { - GenerateCharacterTool { - context: build_tool_context(&document), - } - .format_execute_message(&args, &result) - } else { - GenerateUiDesignTool { - context: build_tool_context(&document), - } - .format_execute_message(&args, &result) - }; - complete_editor_agent_image_tool_call( - &state, - &request_context, - &conversation, - &mut document, - message_id, - formatted_text, - result, - ) - .await - } - GenerateImageTool::NAME => { - let args: GenerateImageToolArgs = serde_json::from_value(tool_args.clone()) - .map_err(|e| editor_agent_bad_request(format!("invalid tool call args: {e}")))?; - let project = load_editor_agent_project(&state, &conversation).await?; - let caller = editor_agent_generation_caller(&conversation); - let title = args.prompt.clone(); - let canvas_completion = - build_editor_agent_canvas_completion(&project, GenerateImageTool::NAME, &title); - let tool_context = build_tool_context(&document); - let generate_tool = GenerateImageTool { - context: tool_context, - }; - persist_editor_agent_tool_call_executing( - &state, - &conversation, - &mut document, - message_id, - ) - .await?; - let result = generate_tool - .execute( - &state, - &request_context, - caller, - args.clone(), - conversation.project_id.clone(), - editor_agent_generation_inputs(&conversation, message_id, &args), - Some(title), - canvas_completion, - ) - .await?; - let formatted_text = generate_tool.format_execute_message(&args, &result); - complete_editor_agent_image_tool_call( - &state, - &request_context, - &conversation, - &mut document, - message_id, - formatted_text, - result, - ) - .await } EditImageTool::NAME => { - let args: EditImageToolArgs = serde_json::from_value(tool_args) - .map_err(|e| editor_agent_bad_request(format!("invalid tool call args: {e}")))?; - - // Get project - let project = state - .spacetime_client() - .get_editor_project(EditorProjectGetRecordInput { - project_id: conversation.project_id.clone(), - owner_user_id: conversation.owner_user_id.clone(), - }) - .await - .map_err(|e| { - AppError::from_status(axum::http::StatusCode::NOT_FOUND) - .with_details(json!({ "message": format!("project not found: {e}") })) - })?; - - // Build caller - let caller = EditorGenerationCaller { - owner_user_id: conversation.owner_user_id.clone(), - audit_subject_user_id: Some(conversation.owner_user_id.clone()), - audit_project_id: Some(conversation.project_id.clone()), - }; - - // Build canvas completion - let title = args.prompt.clone(); - let canvas_completion = - build_editor_agent_canvas_completion(&project, "edit-image", &title); - - // Execute the real generation - let tool_context = build_tool_context(&document); - let edit_tool = EditImageTool { - context: tool_context, - }; - persist_editor_agent_tool_call_executing( - &state, - &conversation, - &mut document, - message_id, - ) - .await?; - - let result = edit_tool - .execute( - &state, - &request_context, - caller, - args.clone(), - None, // model - Some(conversation.project_id.clone()), // project_id - None, // generation_inputs - None, // asset_label - None, // source_resource_id - Some(canvas_completion), - ) - .await; - - let result = match result { - Ok(result) => result, - Err(error) => { - let msg = &mut document.messages[message_id]; - msg.text = format!("[tool_call:{tool_name}] output: {error}"); - if let Some(tool_call) = &mut msg.tool_call { - tool_call.status = EditorAgentToolCallStatus::Failed; - tool_call.error = Some(error.to_string()); - } - write_messages_document(&state, &conversation, &document).await?; - return Err(error); - } - }; - - // Build generated images - let generated_images = vec![EditorAgentGeneratedImage { - resource_id: None, - object_key: result.object_key.clone(), - asset_object_id: result.asset_object_id.clone(), - image_src: result.image_src.clone(), - thumbnail_src: None, - width: Some(result.width), - height: Some(result.height), - }]; - - // Format message text — reuse parsed args for format - let formatted_text = edit_tool.format_execute_message(args, result); - - // Update message - let msg = &mut document.messages[message_id]; - msg.text = formatted_text; - if let Some(ref mut tool_call) = msg.tool_call { - tool_call.status = EditorAgentToolCallStatus::Completed; - tool_call.images = generated_images; + let args: EditImageToolArgs = parse_confirm_tool_args(&tool_args)?; + if args.prompt.trim().is_empty() { + return Err(editor_agent_bad_request("prompt not provided")); } - - write_messages_document(&state, &conversation, &document).await?; - - Ok(json_success_body( - Some(&request_context), - &document.messages[message_id], - )) + let source_image_src = context + .image_data_key(&args.object_image_id) + .ok_or_else(|| editor_agent_bad_request("object image not found"))? + .to_string(); + let reference_image_srcs = args + .reference_image_ids + .iter() + .map(|image_id| { + context + .image_data_key(image_id) + .map(ToOwned::to_owned) + .ok_or_else(|| { + editor_agent_bad_request(format!("image {image_id} not found")) + }) + }) + .collect::, _>>()?; + let title = args.prompt.clone(); + let price = EditImageTool { + context: context.clone(), + } + .pricing(&pricing, &args); + let payload = EditorImageEditRequest { + prompt: args.prompt, + source_image_src, + size: None, + model: None, + reference_image_srcs: Some(reference_image_srcs), + project_id: Some(conversation.project_id.clone()), + asset_kind: Some("editor_agent_edit_image".to_string()), + generation_inputs: generation_inputs("用户指令", &title), + asset_folder_id: Some("project".to_string()), + asset_label: Some(title.clone()), + source_resource_id: None, + target_layer_id: None, + canvas_completion: Some(build_editor_agent_canvas_completion( + &project, + EditImageTool::NAME, + &title, + )), + }; + ( + EDITOR_IMAGE_EDIT_JOB_KIND, + "画布 Agent 修改图片", + price, + serde_json::to_value(payload), + ) } - _ => Err(editor_agent_bad_request(format!( - "unsupported tool: {}", - tool_name - ))), - } -} - -async fn persist_editor_agent_tool_call_executing( - state: &AppState, - conversation: &EditorAgentConversationRecord, - document: &mut EditorAgentConversationMessagesDocument, - message_id: usize, -) -> Result<(), AppError> { - fn mark_editor_agent_tool_call_executing( - tool_call: &mut EditorAgentToolCall, - ) -> Result<(), AppError> { - if tool_call.status != EditorAgentToolCallStatus::PendingConfirmation { - return Err(editor_agent_bad_request( - "tool call is not in PendingConfirmation status", - )); + GenerateIconSpritesheetTool::NAME => { + let args: GenerateIconSpritesheetToolArgs = parse_confirm_tool_args(&tool_args)?; + let tool = GenerateIconSpritesheetTool { + context: context.clone(), + }; + let price = tool.pricing(&pricing, &args); + let reference_image_src = context + .image_data_key(&args.reference_image_id) + .ok_or_else(|| editor_agent_bad_request("reference image not found"))? + .to_string(); + let reference_image_srcs = args + .reference_image_ids + .iter() + .map(|image_id| { + context + .image_data_key(image_id) + .map(ToOwned::to_owned) + .ok_or_else(|| { + editor_agent_bad_request(format!("image {image_id} not found")) + }) + }) + .collect::, _>>()?; + let payload = EditorIconSpritesheetGenerationRequest { + reference_image_src, + reference_image_srcs: Some(reference_image_srcs), + icon_descriptions: args.icon_descriptions.clone(), + model: None, + screen_color: Some("auto".to_string()), + seg_model: Some("birefnet".to_string()), + aspect_ratio: args.aspect_ratio, + image_size: args.image_size, + project_id: Some(conversation.project_id.clone()), + generation_inputs: Some( + json!({"source":"editor-agent","conversationId":conversation.conversation_id,"toolCallMessageId":message_id}), + ), + asset_folder_id: Some("project".to_string()), + canvas_completion: Some(build_editor_agent_canvas_completion( + &project, + GenerateIconSpritesheetTool::NAME, + "图标素材图集", + )), + }; + ( + EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND, + "画布 Agent 生成图标素材", + price, + serde_json::to_value(payload), + ) } - tool_call.status = EditorAgentToolCallStatus::Executing; - Ok(()) - } - let tool_call = document.messages[message_id] + GenerateVideoTool::NAME => { + let args: GenerateVideoToolArgs = parse_confirm_tool_args(&tool_args)?; + let reference_image_srcs = args + .reference_image_ids + .iter() + .map(|image_id| { + context + .image_data_key(image_id) + .map(ToOwned::to_owned) + .ok_or_else(|| { + editor_agent_bad_request(format!("image {image_id} not found")) + }) + }) + .collect::, _>>()?; + let title = args.prompt.clone(); + let price = GenerateVideoTool { + context: context.clone(), + } + .pricing(&pricing, &args); + let payload = EditorVideoGenerateRequest { + prompt: args.prompt, + model: args.model.unwrap_or_else(|| "seedance2.0-fast".to_string()), + aspect_ratio: args.aspect_ratio.unwrap_or_else(|| "16:9".to_string()), + duration_seconds: args.duration_seconds.unwrap_or(4), + resolution: args.resolution.unwrap_or_else(|| "720p".to_string()), + mode: "std".to_string(), + sound: args.sound.unwrap_or_else(|| "off".to_string()), + web_search_enabled: false, + reference_image_srcs, + reference_video_srcs: Vec::new(), + reference_audio_srcs: Vec::new(), + project_id: Some(conversation.project_id.clone()), + canvas_completion: Some(build_editor_agent_canvas_completion( + &project, + GenerateVideoTool::NAME, + &title, + )), + generation_inputs: generation_inputs("用户指令", &title), + source_resource_id: None, + asset_kind: Some("video".to_string()), + asset_folder_id: Some("project".to_string()), + asset_label: Some(title), + }; + ( + EDITOR_VIDEO_GENERATION_JOB_KIND, + "画布 Agent 生成视频", + price, + serde_json::to_value(payload), + ) + } + GenerateSoundEffectTool::NAME => { + let args: GenerateSoundEffectToolArgs = parse_confirm_tool_args(&tool_args)?; + let price = GenerateSoundEffectTool.pricing(&pricing, &args); + let title = args.prompt.clone(); + let payload = EditorSoundEffectGenerateRequest { + prompt: args.prompt, + model: args.model, + duration: args.duration.unwrap_or(3), + project_id: Some(conversation.project_id.clone()), + canvas_completion: Some(build_editor_agent_canvas_completion( + &project, + GenerateSoundEffectTool::NAME, + &title, + )), + generation_inputs: generation_inputs("用户指令", &title), + asset_folder_id: Some("project".to_string()), + asset_label: Some(title), + }; + ( + EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND, + "画布 Agent 生成音效", + price, + serde_json::to_value(payload), + ) + } + GenerateBackgroundMusicTool::NAME => { + let args: GenerateBackgroundMusicToolArgs = parse_confirm_tool_args(&tool_args)?; + let price = GenerateBackgroundMusicTool.pricing(&pricing, &args); + let title = args.prompt.clone(); + let payload = EditorBackgroundMusicGenerateRequest { + gpt_description_prompt: args.prompt, + make_instrumental: args.make_instrumental, + project_id: Some(conversation.project_id.clone()), + canvas_completion: Some(build_editor_agent_canvas_completion( + &project, + GenerateBackgroundMusicTool::NAME, + "生成背景音乐", + )), + generation_inputs: generation_inputs("用户指令", &title), + asset_folder_id: Some("project".to_string()), + asset_label: Some("生成背景音乐".to_string()), + }; + ( + EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, + "画布 Agent 生成背景音乐", + price, + serde_json::to_value(payload), + ) + } + _ => { + return Err(editor_agent_bad_request(format!( + "unsupported tool: {tool_name}" + ))); + } + }; + let payload = payload.map_err(|error| { + AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_details(json!({"message": format!("failed to serialize tool payload: {error}")})) + })?; + let (job_id, dedupe_key) = editor_agent_tool_job_identity( + conversation.conversation_id.as_str(), + message_id, + tool_name.as_str(), + ); + let job = enqueue_editor_generation_job_with_identity( + &state, + conversation.owner_user_id.as_str(), + job_kind, + conversation.project_id.clone(), + request_label, + u64::from(price_mud_points), + &payload, + job_id, + dedupe_key, + ) + .await?; + let message = &mut document.messages[message_id]; + let tool_call = message .tool_call .as_mut() .ok_or_else(|| editor_agent_bad_request("message has no tool call"))?; - mark_editor_agent_tool_call_executing(tool_call)?; - write_messages_document(state, conversation, document).await + tool_call.external_job_id = Some(job.job_id); + write_messages_document(&state, &conversation, &document).await?; + Ok(json_success_body( + Some(&request_context), + &document.messages[message_id], + )) +} + +fn parse_confirm_tool_args(value: &Value) -> Result { + serde_json::from_value(value.clone()) + .map_err(|error| editor_agent_bad_request(format!("invalid tool call args: {error}"))) +} + +fn editor_agent_tool_job_identity( + conversation_id: &str, + message_id: usize, + tool_name: &str, +) -> (String, String) { + let dedupe_key = format!("editor-agent:{conversation_id}:{message_id}:{tool_name}"); + let digest = Sha256::digest(dedupe_key.as_bytes()); + (format!("task-editor-agent-{digest:x}"), dedupe_key) } async fn load_editor_agent_project( @@ -1500,17 +1367,6 @@ async fn load_editor_agent_project( .with_details(json!({ "message": format!("project not found: {error}") })) }) } - -fn editor_agent_generation_caller( - conversation: &EditorAgentConversationRecord, -) -> EditorGenerationCaller { - EditorGenerationCaller { - owner_user_id: conversation.owner_user_id.clone(), - audit_subject_user_id: Some(conversation.owner_user_id.clone()), - audit_project_id: Some(conversation.project_id.clone()), - } -} - fn editor_agent_generation_inputs( conversation: &EditorAgentConversationRecord, message_id: usize, @@ -1537,126 +1393,3 @@ fn editor_agent_media_generation_inputs( "fields": [{ "title": title, "value": value }], }) } - -fn editor_agent_generated_video( - resource: Option<&Value>, - object_key: Option, - asset_object_id: Option, - video_src: String, - thumbnail_src: Option, - width: u32, - height: u32, -) -> EditorAgentGeneratedVideo { - EditorAgentGeneratedVideo { - resource_id: resource - .and_then(|value| value.get("resourceId")) - .and_then(Value::as_str) - .map(ToOwned::to_owned), - object_key, - asset_object_id, - video_src, - thumbnail_src, - width: Some(width), - height: Some(height), - } -} - -fn editor_agent_generated_audio( - resource: Option<&Value>, - object_key: Option, - asset_object_id: Option, - audio_src: String, -) -> EditorAgentGeneratedAudio { - EditorAgentGeneratedAudio { - resource_id: resource - .and_then(|value| value.get("resourceId")) - .and_then(Value::as_str) - .map(ToOwned::to_owned), - object_key, - asset_object_id, - audio_src, - } -} - -async fn complete_editor_agent_image_tool_call( - state: &AppState, - request_context: &RequestContext, - conversation: &EditorAgentConversationRecord, - document: &mut EditorAgentConversationMessagesDocument, - message_id: usize, - formatted_text: String, - result: EditorImageGenerationResult, -) -> Result, AppError> { - let generated_images = vec![EditorAgentGeneratedImage { - resource_id: result - .resource - .as_ref() - .and_then(|resource| resource.get("resourceId")) - .and_then(Value::as_str) - .map(ToOwned::to_owned), - object_key: result.object_key, - asset_object_id: result.asset_object_id, - image_src: result.image_src, - thumbnail_src: None, - width: Some(result.width), - height: Some(result.height), - }]; - complete_editor_agent_tool_call( - state, - request_context, - conversation, - document, - message_id, - formatted_text, - generated_images, - ) - .await -} - -async fn complete_editor_agent_tool_call( - state: &AppState, - request_context: &RequestContext, - conversation: &EditorAgentConversationRecord, - document: &mut EditorAgentConversationMessagesDocument, - message_id: usize, - formatted_text: String, - images: Vec, -) -> Result, AppError> { - let message = &mut document.messages[message_id]; - message.text = formatted_text; - if let Some(tool_call) = &mut message.tool_call { - tool_call.status = EditorAgentToolCallStatus::Completed; - tool_call.images = images; - } - write_messages_document(state, conversation, document).await?; - Ok(json_success_body( - Some(request_context), - &document.messages[message_id], - )) -} - -async fn complete_editor_agent_media_tool_call( - state: &AppState, - request_context: &RequestContext, - conversation: &EditorAgentConversationRecord, - document: &mut EditorAgentConversationMessagesDocument, - message_id: usize, - formatted_text: String, - images: Vec, - videos: Vec, - audios: Vec, -) -> Result, AppError> { - let message = &mut document.messages[message_id]; - message.text = formatted_text; - if let Some(tool_call) = &mut message.tool_call { - tool_call.status = EditorAgentToolCallStatus::Completed; - tool_call.images = images; - tool_call.videos = videos; - tool_call.audios = audios; - } - write_messages_document(state, conversation, document).await?; - Ok(json_success_body( - Some(request_context), - &document.messages[message_id], - )) -} diff --git a/server-rs/crates/shared-contracts/src/editor_agent.rs b/server-rs/crates/shared-contracts/src/editor_agent.rs index c39ddd449..e93d1bfdb 100644 --- a/server-rs/crates/shared-contracts/src/editor_agent.rs +++ b/server-rs/crates/shared-contracts/src/editor_agent.rs @@ -23,16 +23,6 @@ pub enum EditorAgentAttachmentSource { LibraryAsset, } -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum EditorAgentToolCallStatus { - PendingConfirmation, - Executing, - Completed, - Failed, - Cancelled, -} - #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct EditorAgentAttachmentRef { @@ -147,13 +137,16 @@ pub struct EditorAgentToolCallImageRef { } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[serde(rename_all = "camelCase")] pub struct EditorAgentToolCall { pub tool_name: String, - pub status: EditorAgentToolCallStatus, pub args: serde_json::Value, pub display_args: EditorAgentToolCallDisplayArgs, #[serde(default)] + pub external_job_id: Option, + #[serde(default)] + pub cancelled_at: Option, + #[serde(default)] pub images: Vec, #[serde(default)] pub videos: Vec, @@ -248,7 +241,6 @@ mod tests { fn tool_call_display_args_and_media_use_camel_case() { let missing_display_args = serde_json::from_value::(json!({ "toolName": "generate-video", - "status": "completed", "args": {}, "images": [] })); @@ -256,7 +248,6 @@ mod tests { let tool_call = EditorAgentToolCall { tool_name: "generate-video".to_string(), - status: EditorAgentToolCallStatus::Completed, args: json!({}), display_args: EditorAgentToolCallDisplayArgs { string_args: vec![EditorAgentToolCallStringArg { @@ -281,6 +272,8 @@ mod tests { price_mud_points: 5, }, }, + external_job_id: Some("task-editor-agent-1".to_string()), + cancelled_at: None, images: Vec::new(), videos: vec![EditorAgentGeneratedVideo { resource_id: Some("resource-video-1".to_string()), @@ -301,6 +294,8 @@ mod tests { }; let payload = serde_json::to_value(tool_call).expect("tool call should serialize"); assert_eq!(payload["videos"][0]["videoSrc"], "/generated-video.mp4"); + assert_eq!(payload["externalJobId"], "task-editor-agent-1"); + assert!(payload.get("status").is_none()); assert_eq!(payload["audios"][0]["audioSrc"], "/generated-audio.mp3"); assert_eq!( payload["displayArgs"]["stringArgs"][0]["value"], @@ -316,4 +311,23 @@ mod tests { ); assert_eq!(payload["displayArgs"]["extras"]["priceMudPoints"], 5); } + + #[test] + fn legacy_tool_call_status_is_ignored_on_read() { + let tool_call = serde_json::from_value::(json!({ + "toolName": "generate-image", + "status": "executing", + "args": {}, + "displayArgs": { + "stringArgs": [], + "imageArgs": [], + "extras": { "priceMudPoints": 3 } + }, + "images": [] + })) + .expect("legacy status should not break persisted OSS documents"); + + let payload = serde_json::to_value(tool_call).expect("tool call should serialize"); + assert!(payload.get("status").is_none()); + } } diff --git a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx index 541274859..d1f00a1d6 100644 --- a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx +++ b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx @@ -122,7 +122,6 @@ function createPendingToolCallMessage(): EditorAgentMessage { attachments: [], toolCall: { toolName: 'edit-image', - status: 'pending_confirmation', args: { object_image_id: 'source-image-1', reference_image_ids: ['reference-image-1', 'reference-image-2'], @@ -570,7 +569,7 @@ describe('EditorAgentConversationPanelView', () => { text: 'internal completed tool output that must stay hidden', toolCall: { ...pendingMessage.toolCall!, - status: 'completed', + externalJobId: 'task-edit-panel-1', images: [ { resourceId: null, @@ -610,7 +609,7 @@ describe('EditorAgentConversationPanelView', () => { text: 'internal cancelled tool output that must stay hidden', toolCall: { ...pendingMessage.toolCall!, - status: 'cancelled', + cancelledAt: '2026-07-03T00:00:11.000Z', }, }); diff --git a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx index e07cce48e..c64f4f79d 100644 --- a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx +++ b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx @@ -604,6 +604,7 @@ export function EditorAgentConversationPanelView({ onCancelToolCall={(messageId) => { void cancelToolCall(messageId); }} + onJobCompleted={onCanvasRefreshRequested} /> ))} {isWaiting ? : null} diff --git a/src/components/image-editor/EditorAgentConversation/MessageBubble.tsx b/src/components/image-editor/EditorAgentConversation/MessageBubble.tsx index e1c35701f..c74198dcd 100644 --- a/src/components/image-editor/EditorAgentConversation/MessageBubble.tsx +++ b/src/components/image-editor/EditorAgentConversation/MessageBubble.tsx @@ -1,16 +1,20 @@ import { Check, Image as ImageIcon, Loader2, Volume2, X } from 'lucide-react'; +import { useEffect, useRef, useState } from 'react'; import type { EditorAgentAttachmentRef, EditorAgentMessage, EditorAgentToolCall, } from '@/packages/shared/src/contracts'; +import type { ExternalGenerationJobStatus } from '@/packages/shared/src/contracts/externalGeneration.ts'; +import type { ExternalGenerationJobStatusRecord } from '@/packages/shared/src/contracts/externalGeneration.ts'; import { attachmentKey } from '@/src/components/image-editor/EditorAgentConversation/common.ts'; import { PendingToolCall } from '@/src/components/image-editor/EditorAgentConversation/PendingToolCall.tsx'; import { editorAgentToolLabel } from '@/src/components/image-editor/EditorAgentConversation/toolCallPresentation.ts'; import { ResolvedAssetAudio } from '@/src/components/ResolvedAssetAudio.tsx'; import { ResolvedAssetImage } from '@/src/components/ResolvedAssetImage.tsx'; import { ResolvedAssetVideo } from '@/src/components/ResolvedAssetVideo.tsx'; +import { getExternalGenerationJobStatus } from '@/src/services/external-generation'; function messageRoleLabel(role: EditorAgentMessage['role']) { if (role === 'user') { @@ -56,18 +60,88 @@ export function AttachmentChip({ ); } -function ToolCallView({ toolCall }: { toolCall: EditorAgentToolCall }) { +function ToolCallView({ + toolCall, + jobStatus, + onJobCompleted, +}: { + toolCall: EditorAgentToolCall; + jobStatus?: ExternalGenerationJobStatus | null; + onJobCompleted?: () => void; +}) { const videos = toolCall.videos ?? []; const audios = toolCall.audios ?? []; - const isExecuting = toolCall.status === 'executing'; + const [resolvedJob, setResolvedJob] = + useState(null); + const completionNotifiedRef = useRef(false); + useEffect(() => { + const jobId = toolCall.externalJobId?.trim(); + if (!jobId) { + setResolvedJob(null); + return; + } + let disposed = false; + let timeoutId: ReturnType | undefined; + const refresh = async () => { + try { + const response = await getExternalGenerationJobStatus(jobId); + if (disposed) return; + setResolvedJob(response.job); + if ( + response.job.status === 'completed' && + !completionNotifiedRef.current + ) { + completionNotifiedRef.current = true; + onJobCompleted?.(); + } + if ( + response.job.status === 'queued' || + response.job.status === 'running' + ) { + timeoutId = setTimeout(refresh, 1500); + } + } catch { + if (!disposed) timeoutId = setTimeout(refresh, 3000); + } + }; + void refresh(); + return () => { + disposed = true; + if (timeoutId) clearTimeout(timeoutId); + }; + }, [onJobCompleted, toolCall.externalJobId]); + const effectiveJobStatus = jobStatus ?? resolvedJob?.status ?? null; + const generationResult = + resolvedJob?.result && typeof resolvedJob.result === 'object' + ? (resolvedJob.result as Record) + : null; + const resultImageSrc = + typeof generationResult?.imageSrc === 'string' + ? generationResult.imageSrc + : null; + const resultVideoSrc = + typeof generationResult?.videoSrc === 'string' + ? generationResult.videoSrc + : null; + const resultAudioSrc = + typeof generationResult?.audioSrc === 'string' + ? generationResult.audioSrc + : null; + const resultObjectKey = + typeof generationResult?.objectKey === 'string' + ? generationResult.objectKey + : null; + const isCancelled = Boolean(toolCall.cancelledAt); + const isExecuting = + effectiveJobStatus === 'queued' || effectiveJobStatus === 'running'; const statusLabel = - toolCall.status === 'completed' + effectiveJobStatus === 'completed' ? '已完成' - : toolCall.status === 'cancelled' + : isCancelled ? '已取消' - : toolCall.status === 'failed' + : effectiveJobStatus === 'failed' ? '失败' - : toolCall.status === 'pending_confirmation' + : !toolCall.externalJobId ? '待确认' : '执行中'; return ( @@ -75,9 +149,9 @@ function ToolCallView({ toolCall }: { toolCall: EditorAgentToolCall }) {
{isExecuting ? (
- {toolCall.error ? ( -
{toolCall.error}
+ {toolCall.error || resolvedJob?.error ? ( +
+ {toolCall.error ?? resolvedJob?.error} +
) : null} {toolCall.images.length ? (
@@ -105,6 +181,16 @@ function ToolCallView({ toolCall }: { toolCall: EditorAgentToolCall }) {
))} + ) : resultImageSrc ? ( +
+ +
) : null} {videos.length ? (
@@ -128,6 +214,16 @@ function ToolCallView({ toolCall }: { toolCall: EditorAgentToolCall }) {
))} + ) : resultVideoSrc ? ( + ) : null} {audios.length ? (
@@ -153,6 +249,15 @@ function ToolCallView({ toolCall }: { toolCall: EditorAgentToolCall }) {
))} + ) : resultAudioSrc ? ( + ) : null} ); @@ -185,23 +290,29 @@ export function ThinkingBubble() { type MessageBubbleProps = { message: EditorAgentMessage; + jobStatus?: ExternalGenerationJobStatus | null; busyAction: 'confirm' | 'cancel' | null; onConfirmToolCall: (messageId: number) => void; onCancelToolCall: (messageId: number) => void; + onJobCompleted?: () => void; }; export function MessageBubble({ message, + jobStatus, busyAction, onConfirmToolCall, onCancelToolCall, + onJobCompleted, }: MessageBubbleProps) { if (message.role === 'system' && !message.toolCall) { return null; } if ( message.role === 'system' && - message.toolCall?.status === 'pending_confirmation' + message.toolCall && + !message.toolCall.externalJobId && + !message.toolCall.cancelledAt ) { return ( ) : null} - {message.toolCall ? : null} + {message.toolCall ? ( + + ) : null} ); diff --git a/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx b/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx index cda8fd741..f2c36ebe9 100644 --- a/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx +++ b/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx @@ -81,7 +81,7 @@ function createClient(): EditorAgentConversationClient { attachments: [], toolCall: { toolName: 'generate_image', - status: 'completed', + externalJobId: 'task-generated-1', args: {}, displayArgs: { stringArgs: [], @@ -157,7 +157,7 @@ describe('useEditorAgentConversation', () => { expect(result.current.messages[1]?.toolCall).toEqual( expect.objectContaining({ toolName: 'generate_image', - status: 'completed', + externalJobId: 'task-generated-1', images: [ expect.objectContaining({ resourceId: 'resource-result-1', @@ -276,7 +276,6 @@ describe('useEditorAgentConversation', () => { attachments: [], toolCall: { toolName: 'edit-image', - status: 'pending_confirmation', args: { object_image_id: 'source-image', prompt: '换成像素风', @@ -297,7 +296,7 @@ describe('useEditorAgentConversation', () => { attachments: [], toolCall: { toolName: 'edit-image', - status: 'completed', + externalJobId: 'task-edit-1', args: { object_image_id: 'source-image', prompt: '换成像素风', @@ -338,7 +337,7 @@ describe('useEditorAgentConversation', () => { const systemMsg = result.current.messages[1]; expect(systemMsg?.role).toBe('system'); - expect(systemMsg?.toolCall?.status).toBe('pending_confirmation'); + expect(systemMsg?.toolCall?.externalJobId).toBeUndefined(); const messageCount = result.current.messages.length; await act(async () => { @@ -347,14 +346,16 @@ describe('useEditorAgentConversation', () => { expect(client.confirmToolCall).toHaveBeenCalledWith('conversation-1', 1); expect(result.current.messages).toHaveLength(messageCount); - expect(result.current.messages[1]?.toolCall?.status).toBe('completed'); + expect(result.current.messages[1]?.toolCall?.externalJobId).toBe( + 'task-edit-1', + ); expect(result.current.messages[1]?.toolCall?.images[0]?.objectKey).toBe( 'generated/result.png', ); expect(onCanvasRefreshRequested).toHaveBeenCalledTimes(1); }); - it('marks a confirmation as executing before the request finishes', async () => { + it('keeps the action busy until confirmation returns an external job id', async () => { const client = createClient(); const pendingMessage: EditorAgentMessage = { id: 0, @@ -363,7 +364,6 @@ describe('useEditorAgentConversation', () => { attachments: [], toolCall: { toolName: 'edit-image', - status: 'pending_confirmation', args: { object_image_id: 'source-image', prompt: '换成像素风' }, displayArgs: createEditImageDisplayArgs('换成像素风'), images: [], @@ -375,11 +375,12 @@ describe('useEditorAgentConversation', () => { ...pendingMessage, toolCall: { ...pendingMessage.toolCall!, - status: 'completed', + externalJobId: 'task-edit-2', images: [], }, }; - let resolveConfirmation: ((message: EditorAgentMessage) => void) | undefined; + let resolveConfirmation: + ((message: EditorAgentMessage) => void) | undefined; vi.mocked(client.getConversation).mockResolvedValue({ conversationId: 'conversation-1', projectId: 'project-1', @@ -399,26 +400,27 @@ describe('useEditorAgentConversation', () => { ); await waitFor(() => { - expect(result.current.messages[0]?.toolCall?.status).toBe( - 'pending_confirmation', - ); + expect(result.current.messages[0]?.role).toBe('system'); + expect( + result.current.messages[0]?.toolCall?.externalJobId, + ).toBeUndefined(); }); + let confirmationPromise: Promise | undefined; act(() => { - void result.current.confirmToolCall(0); + confirmationPromise = result.current.confirmToolCall(0); }); - - await waitFor(() => { - expect(result.current.messages[0]?.toolCall?.status).toBe('executing'); - }); - await result.current.confirmToolCall(0); + await waitFor(() => expect(resolveConfirmation).toBeTypeOf('function')); expect(client.confirmToolCall).toHaveBeenCalledTimes(1); await act(async () => { resolveConfirmation?.(completedMessage); + await confirmationPromise; }); await waitFor(() => { - expect(result.current.messages[0]?.toolCall?.status).toBe('completed'); + expect(result.current.messages[0]?.toolCall?.externalJobId).toBe( + 'task-edit-2', + ); }); }); @@ -431,7 +433,6 @@ describe('useEditorAgentConversation', () => { attachments: [], toolCall: { toolName: 'edit-image', - status: 'pending_confirmation' as const, args: { object_image_id: 'source-image', prompt: '换成像素风' }, displayArgs: createEditImageDisplayArgs('换成像素风'), images: [], @@ -452,7 +453,7 @@ describe('useEditorAgentConversation', () => { text: 'internal cancelled tool output', toolCall: { ...pendingMessage.toolCall, - status: 'cancelled', + cancelledAt: '2026-07-03T00:00:01.000Z', }, }); const onCanvasRefreshRequested = vi.fn(); @@ -465,9 +466,7 @@ describe('useEditorAgentConversation', () => { ); await waitFor(() => { - expect(result.current.messages[0]?.toolCall?.status).toBe( - 'pending_confirmation', - ); + expect(result.current.messages[0]?.toolCall?.cancelledAt).toBeUndefined(); }); await act(async () => { await result.current.cancelToolCall(0); @@ -475,7 +474,9 @@ describe('useEditorAgentConversation', () => { expect(client.cancelToolCall).toHaveBeenCalledWith('conversation-1', 0); expect(result.current.messages).toHaveLength(1); - expect(result.current.messages[0]?.toolCall?.status).toBe('cancelled'); + expect(result.current.messages[0]?.toolCall?.cancelledAt).toBe( + '2026-07-03T00:00:01.000Z', + ); expect(onCanvasRefreshRequested).not.toHaveBeenCalled(); }); diff --git a/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts b/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts index c202a1e94..f7119fd24 100644 --- a/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts +++ b/src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts @@ -295,13 +295,15 @@ export function useEditorAgentConversation({ const requestCanvasRefreshForMessages = useCallback( (nextMessages: EditorAgentMessage[]) => { if ( - nextMessages.some( - (message) => - message.toolCall?.status === 'completed' && - (message.toolCall.images.length > 0 || - (message.toolCall.videos?.length ?? 0) > 0 || - (message.toolCall.audios?.length ?? 0) > 0), - ) + nextMessages.some((message) => { + const toolCall = message.toolCall; + return Boolean( + toolCall?.externalJobId && + (toolCall.images.length > 0 || + (toolCall.videos?.length ?? 0) > 0 || + (toolCall.audios?.length ?? 0) > 0), + ); + }) ) { onCanvasRefreshRequested?.(); } @@ -418,23 +420,6 @@ export function useEditorAgentConversation({ setToolCallAction(nextAction); setErrorMessage(null); - if (action === 'confirm') { - setMessages((currentMessages) => - currentMessages.map((message) => - message.id === messageId && - message.toolCall?.status === 'pending_confirmation' - ? { - ...message, - toolCall: { - ...message.toolCall, - status: 'executing', - }, - } - : message, - ), - ); - } - try { const updatedMessage = await (action === 'confirm' ? client.confirmToolCall(conversationId, messageId) diff --git a/src/components/image-editor/ImageCanvasEditorView.test.tsx b/src/components/image-editor/ImageCanvasEditorView.test.tsx index af183f10a..9a9226395 100644 --- a/src/components/image-editor/ImageCanvasEditorView.test.tsx +++ b/src/components/image-editor/ImageCanvasEditorView.test.tsx @@ -213,7 +213,7 @@ function createEditorAgentDetailWithGeneration( attachments: [], toolCall: { toolName: 'generate_image', - status: 'completed', + externalJobId: 'task-generation-1', args: {}, displayArgs: { stringArgs: [], @@ -2069,7 +2069,6 @@ describe('ImageCanvasEditorView', () => { attachments: [], toolCall: { toolName: 'edit-image', - status: 'pending_confirmation', args: { object_image_id: 'source-image', prompt: '把图片换成像素风', @@ -2092,10 +2091,9 @@ describe('ImageCanvasEditorView', () => { attachments: [], toolCall: { toolName: 'edit-image', - status: 'completed', + externalJobId: 'task-edit-canvas-1', args: {}, - displayArgs: - createEditImageToolCallDisplayArgs('把图片换成像素风'), + displayArgs: createEditImageToolCallDisplayArgs('把图片换成像素风'), images: [ { resourceId: null,