diff --git a/packages/shared/src/contracts/editorAgent.ts b/packages/shared/src/contracts/editorAgent.ts index e7412842d..3bec78949 100644 --- a/packages/shared/src/contracts/editorAgent.ts +++ b/packages/shared/src/contracts/editorAgent.ts @@ -54,11 +54,43 @@ export interface EditorAgentGeneratedAudio { audioSrc: string; } +export interface EditorAgentToolCallStringArg { + name: string; + label: string; + value: string; +} + +export interface EditorAgentToolCallImageRef { + imageId: string; + imageSrc: string; + objectKey?: string | null; + thumbnailSrc?: string | null; + label?: string | null; + width?: number | null; + height?: number | null; +} + +export interface EditorAgentToolCallImageArg { + name: string; + label: string; + refs: EditorAgentToolCallImageRef[]; +} + +export interface EditorAgentToolCallDisplayExtras { + priceMudPoints: number; +} + +export interface EditorAgentToolCallDisplayArgs { + stringArgs: EditorAgentToolCallStringArg[]; + imageArgs: EditorAgentToolCallImageArg[]; + extras: EditorAgentToolCallDisplayExtras; +} + export interface EditorAgentToolCall { toolName: string; - summary: string; status: EditorAgentToolCallStatus; args: unknown; + displayArgs: EditorAgentToolCallDisplayArgs; images: EditorAgentGeneratedImage[]; // Older persisted conversation documents do not contain these media fields. videos?: EditorAgentGeneratedVideo[]; diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index fd06c99c6..62806b488 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -18,8 +18,10 @@ use shared_contracts::editor_agent::{ CreateEditorAgentConversationRequest, EditorAgentConversationListResponse, EditorAgentConversationMessagesDocument, EditorAgentConversationResponse, EditorAgentConversationSummary, EditorAgentGeneratedAudio, EditorAgentGeneratedImage, - EditorAgentGeneratedVideo, EditorAgentMessage,EditorAgentMessageRequest, EditorAgentMessageResponse, EditorAgentMessageRole, - EditorAgentToolCall, EditorAgentToolCallStatus, + EditorAgentGeneratedVideo, EditorAgentMessage, EditorAgentMessageRequest, + EditorAgentMessageResponse, EditorAgentMessageRole, EditorAgentToolCall, + EditorAgentToolCallDisplayArgs, EditorAgentToolCallDisplayExtras, EditorAgentToolCallImageArg, + EditorAgentToolCallImageRef, EditorAgentToolCallStatus, EditorAgentToolCallStringArg, }; use spacetime_client::{ EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput, @@ -30,7 +32,7 @@ use spacetime_client::{ use crate::api_response::json_success_body; use crate::auth::AuthenticatedAccessToken; use crate::editor_agent::agent::LlmChatAgentBuilder; -use crate::editor_agent::editor_tools::common::EditorToolContext; +use crate::editor_agent::editor_tools::common::{EditorAgentPricedTool, EditorToolContext}; use crate::editor_agent::editor_tools::edit_image::{EditImageTool, EditImageToolArgs}; use crate::editor_agent::editor_tools::generate_background_music::{ GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, @@ -54,6 +56,7 @@ use crate::editor_agent::utils::{ now_rfc3339, read_messages_document, require_editor_agent_sidebar_enabled, write_messages_document, }; +use crate::editor_generation_config::EditorGenerationPricingConfig; use crate::editor_project::{EditorGenerationCaller, current_utc_micros, map_editor_project_error}; use crate::http_error::AppError; use crate::request_context::RequestContext; @@ -156,6 +159,12 @@ pub async fn editor_agent_message( .with_details(json!({ "message": "LLM client not configured" })) })?; let llm_client = llm_client.clone(); + let pricing = state.editor_generation_pricing().map_err(|error| { + AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({ + "provider": "editor-generation-pricing", + "message": error.to_string(), + })) + })?; let memory = VecMemory::new(previous_messages); @@ -166,7 +175,7 @@ pub async fn editor_agent_message( context: tool_context.clone(), }) .tool(GenerateImageTool { - context: tool_context.clone(), + context: tool_context.clone(), }) .tool(GenerateCharacterTool { context: tool_context.clone(), @@ -180,7 +189,7 @@ pub async fn editor_agent_message( context: tool_context.clone(), }) .tool(GenerateUiDesignTool { - context: tool_context, + context: tool_context.clone(), }) .max_turns(3) .memory(memory) @@ -190,7 +199,14 @@ pub async fn editor_agent_message( let assistant_now = now_rfc3339(); - match build_delta_messages(agent_result, &assistant_now, document.messages.len()) { + match build_delta_messages( + agent_result, + &assistant_now, + document.messages.len(), + &document, + &tool_context, + &pricing, + ) { Err(err) => Ok(Json(EditorAgentMessageResponse { conversation: conversation_summary, delta_messages: vec![], @@ -221,6 +237,9 @@ fn build_delta_messages( result: Result, PromptError>, created_at: &str, messages_offset: usize, + document: &EditorAgentConversationMessagesDocument, + tool_context: &EditorToolContext, + pricing: &EditorGenerationPricingConfig, ) -> Result, PromptError> { let outputs = result?; let mut messages = Vec::with_capacity(outputs.len()); @@ -239,7 +258,13 @@ fn build_delta_messages( }); } PromptOutput::Tool(tco) => { - let summary = tco.tool_call.args.to_string(); + let display_args = build_tool_call_display_args( + tco.tool_call.name.as_str(), + &tco.tool_call.args, + document, + tool_context, + pricing, + )?; messages.push(EditorAgentMessage { id: absolute_idx, role: EditorAgentMessageRole::System, @@ -247,9 +272,9 @@ fn build_delta_messages( attachments: Vec::new(), tool_call: Some(EditorAgentToolCall { tool_name: tco.tool_call.name, - summary, status: EditorAgentToolCallStatus::PendingConfirmation, args: tco.tool_call.args, + display_args, images: Vec::new(), videos: Vec::new(), audios: Vec::new(), @@ -264,6 +289,305 @@ fn build_delta_messages( Ok(messages) } +fn build_tool_call_display_args( + tool_name: &str, + args: &Value, + document: &EditorAgentConversationMessagesDocument, + tool_context: &EditorToolContext, + pricing: &EditorGenerationPricingConfig, +) -> Result { + let mut display_args = EditorAgentToolCallDisplayArgs::default(); + + let price_mud_points = match tool_name { + EditImageTool::NAME => { + let args: EditImageToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = EditImageTool { + context: tool_context.clone(), + } + .pricing(pricing, &args); + push_string_display_arg(&mut display_args, "prompt", "修改要求", args.prompt); + push_image_display_arg( + &mut display_args, + document, + "object_image_id", + "目标图片", + std::slice::from_ref(&args.object_image_id), + )?; + push_image_display_arg( + &mut display_args, + document, + "reference_image_ids", + "参考图片", + args.reference_image_ids.as_slice(), + )?; + price_mud_points + } + GenerateImageTool::NAME | GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => { + let args: GenerateImageToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = match tool_name { + GenerateImageTool::NAME => GenerateImageTool { + context: tool_context.clone(), + } + .pricing(pricing, &args), + GenerateCharacterTool::NAME => GenerateCharacterTool { + context: tool_context.clone(), + } + .pricing(pricing, &args), + GenerateUiDesignTool::NAME => GenerateUiDesignTool { + context: tool_context.clone(), + } + .pricing(pricing, &args), + _ => unreachable!("tool name was matched above"), + }; + push_string_display_arg(&mut display_args, "prompt", "提示词", args.prompt); + push_optional_string_display_arg( + &mut display_args, + "aspect_ratio", + "画面比例", + args.aspect_ratio, + ); + push_optional_string_display_arg( + &mut display_args, + "image_size", + "图片尺寸", + args.image_size, + ); + push_image_display_arg( + &mut display_args, + document, + "reference_image_ids", + "参考图片", + args.reference_image_ids.as_slice(), + )?; + price_mud_points + } + GenerateIconSpritesheetTool::NAME => { + let args: GenerateIconSpritesheetToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = GenerateIconSpritesheetTool { + context: tool_context.clone(), + } + .pricing(pricing, &args); + push_string_display_arg( + &mut display_args, + "icon_descriptions", + "图标描述", + args.icon_descriptions.join("\n"), + ); + push_optional_string_display_arg( + &mut display_args, + "aspect_ratio", + "图集比例", + args.aspect_ratio, + ); + push_optional_string_display_arg( + &mut display_args, + "image_size", + "图片尺寸", + args.image_size, + ); + push_image_display_arg( + &mut display_args, + document, + "reference_image_id", + "规范参考图", + std::slice::from_ref(&args.reference_image_id), + )?; + push_image_display_arg( + &mut display_args, + document, + "reference_image_ids", + "额外参考图", + args.reference_image_ids.as_slice(), + )?; + price_mud_points + } + GenerateVideoTool::NAME => { + let args: GenerateVideoToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = GenerateVideoTool { + context: tool_context.clone(), + } + .pricing(pricing, &args); + push_string_display_arg(&mut display_args, "prompt", "视频描述", args.prompt); + push_optional_string_display_arg( + &mut display_args, + "aspect_ratio", + "画面比例", + args.aspect_ratio, + ); + if let Some(duration_seconds) = args.duration_seconds { + push_string_display_arg( + &mut display_args, + "duration_seconds", + "时长（秒）", + duration_seconds.to_string(), + ); + } + push_optional_string_display_arg(&mut display_args, "model", "模型", args.model); + push_optional_string_display_arg( + &mut display_args, + "resolution", + "分辨率", + args.resolution, + ); + push_optional_string_display_arg(&mut display_args, "sound", "声音", args.sound); + push_image_display_arg( + &mut display_args, + document, + "reference_image_ids", + "参考图片", + args.reference_image_ids.as_slice(), + )?; + price_mud_points + } + GenerateSoundEffectTool::NAME => { + let args: GenerateSoundEffectToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = GenerateSoundEffectTool.pricing(pricing, &args); + push_string_display_arg(&mut display_args, "prompt", "音效描述", args.prompt); + if let Some(duration) = args.duration { + push_string_display_arg( + &mut display_args, + "duration", + "时长（秒）", + duration.to_string(), + ); + } + push_optional_string_display_arg(&mut display_args, "model", "模型", args.model); + price_mud_points + } + GenerateBackgroundMusicTool::NAME => { + let args: GenerateBackgroundMusicToolArgs = parse_display_tool_args(tool_name, args)?; + let price_mud_points = GenerateBackgroundMusicTool.pricing(pricing, &args); + push_string_display_arg(&mut display_args, "prompt", "音乐描述", args.prompt); + push_string_display_arg( + &mut display_args, + "make_instrumental", + "纯音乐", + if args.make_instrumental { "是" } else { "否" }, + ); + price_mud_points + } + _ => { + return Err(PromptError::InternalError(format!( + "missing display argument mapping for tool {tool_name}" + ))); + } + }; + + display_args.extras = EditorAgentToolCallDisplayExtras { price_mud_points }; + + Ok(display_args) +} + +fn parse_display_tool_args(tool_name: &str, args: &Value) -> Result +where + T: serde::de::DeserializeOwned, +{ + serde_json::from_value(args.clone()).map_err(|error| { + PromptError::InternalError(format!( + "failed to build display arguments for {tool_name}: {error}" + )) + }) +} + +fn push_string_display_arg( + display_args: &mut EditorAgentToolCallDisplayArgs, + name: &str, + label: &str, + value: impl Into, +) { + let value = value.into(); + if value.trim().is_empty() { + return; + } + display_args.string_args.push(EditorAgentToolCallStringArg { + name: name.to_string(), + label: label.to_string(), + value, + }); +} + +fn push_optional_string_display_arg( + display_args: &mut EditorAgentToolCallDisplayArgs, + name: &str, + label: &str, + value: Option, +) { + if let Some(value) = value { + push_string_display_arg(display_args, name, label, value); + } +} + +fn push_image_display_arg( + display_args: &mut EditorAgentToolCallDisplayArgs, + document: &EditorAgentConversationMessagesDocument, + name: &str, + label: &str, + image_ids: &[ImageId], +) -> Result<(), PromptError> { + if image_ids.is_empty() { + return Ok(()); + } + + let refs = image_ids + .iter() + .map(|image_id| { + resolve_tool_call_image_ref(document, image_id).ok_or_else(|| { + PromptError::InternalError(format!( + "display image reference {} is missing from conversation context", + image_id.id + )) + }) + }) + .collect::, _>>()?; + + display_args.image_args.push(EditorAgentToolCallImageArg { + name: name.to_string(), + label: label.to_string(), + refs, + }); + Ok(()) +} + +fn resolve_tool_call_image_ref( + document: &EditorAgentConversationMessagesDocument, + image_id: &ImageId, +) -> Option { + for message in document.messages.iter().rev() { + for attachment in message.attachments.iter().rev() { + if attachment.clone().into_image_id() == *image_id { + return Some(EditorAgentToolCallImageRef { + image_id: image_id.id.clone(), + image_src: attachment.image_src.clone(), + object_key: attachment.object_key.clone(), + thumbnail_src: attachment.thumbnail_src.clone(), + label: attachment.label.clone(), + width: attachment.width, + height: attachment.height, + }); + } + } + + if let Some(tool_call) = &message.tool_call { + for image in tool_call.images.iter().rev() { + let data_key = image.clone().into_data_key(); + if ImageId::from_data_key(data_key) == *image_id { + return Some(EditorAgentToolCallImageRef { + image_id: image_id.id.clone(), + image_src: image.image_src.clone(), + object_key: image.object_key.clone(), + thumbnail_src: image.thumbnail_src.clone(), + label: None, + width: image.width, + height: image.height, + }); + } + } + } + } + + None +} + fn build_tool_context(document: &EditorAgentConversationMessagesDocument) -> EditorToolContext { let mut images: HashMap = HashMap::new(); @@ -296,9 +620,11 @@ fn build_tool_context(document: &EditorAgentConversationMessagesDocument) -> Edi #[cfg(test)] mod tool_context_tests { use super::*; + use crate::editor_generation_config::load_editor_generation_pricing_from_paths; use shared_contracts::editor_agent::{ - EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, EditorAgentGeneratedImage, - EditorAgentMessageRole, EditorAgentToolCallStatus, + EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, EditorAgentAttachmentRef, + EditorAgentAttachmentSource, EditorAgentGeneratedImage, EditorAgentMessageRole, + EditorAgentToolCallStatus, }; #[test] @@ -314,9 +640,9 @@ mod tool_context_tests { attachments: Vec::new(), tool_call: Some(EditorAgentToolCall { tool_name: "generate-image".to_string(), - summary: String::new(), status: EditorAgentToolCallStatus::Completed, args: json!({}), + display_args: EditorAgentToolCallDisplayArgs::default(), images: vec![EditorAgentGeneratedImage { resource_id: None, object_key: Some(data_key.to_string()), @@ -339,6 +665,88 @@ mod tool_context_tests { assert_eq!(context.image_data_key(&image_id), Some(data_key)); assert!(!image_id.id.contains(data_key)); + + let display_ref = resolve_tool_call_image_ref(&document, &image_id) + .expect("generated image should have a display reference"); + assert_eq!(display_ref.image_src, "/generated-images/proj-1/output.png"); + assert_eq!(display_ref.object_key.as_deref(), Some(data_key)); + } + + #[test] + fn tool_call_display_args_resolve_target_and_reference_image_previews() { + let target = EditorAgentAttachmentRef { + source: EditorAgentAttachmentSource::CanvasResource, + reference_id: "resource-target".to_string(), + object_key: Some("editor-projects/proj-1/target.png".to_string()), + image_src: "/target.png".to_string(), + thumbnail_src: Some("/target-thumbnail.png".to_string()), + label: Some("目标角色".to_string()), + width: Some(640), + height: Some(960), + }; + let reference = EditorAgentAttachmentRef { + source: EditorAgentAttachmentSource::LibraryAsset, + reference_id: "asset-reference".to_string(), + object_key: Some("editor-assets/reference.png".to_string()), + image_src: "/reference.png".to_string(), + thumbnail_src: None, + label: Some("像素风参考".to_string()), + width: Some(512), + height: Some(512), + }; + let target_id = target.clone().into_image_id(); + let reference_id = reference.clone().into_image_id(); + let document = EditorAgentConversationMessagesDocument { + version: EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, + conversation_id: "editor-agent-conversation-1".to_string(), + messages: vec![EditorAgentMessage { + id: 0, + role: EditorAgentMessageRole::User, + text: "改成像素风".to_string(), + attachments: vec![target, reference], + tool_call: None, + created_at: String::new(), + }], + }; + + let tool_context = build_tool_context(&document); + let pricing = load_editor_generation_pricing_from_paths(None) + .expect("default editor pricing should load"); + let display_args = build_tool_call_display_args( + EditImageTool::NAME, + &json!({ + "object_image_id": target_id.id, + "reference_image_ids": [reference_id.id], + "prompt": "把角色换成像素风" + }), + &document, + &tool_context, + &pricing, + ) + .expect("validated tool args should build a display projection"); + + assert_eq!(display_args.string_args.len(), 1); + assert_eq!(display_args.string_args[0].label, "修改要求"); + assert_eq!(display_args.string_args[0].value, "把角色换成像素风"); + assert_eq!(display_args.image_args.len(), 2); + assert_eq!(display_args.image_args[0].label, "目标图片"); + assert_eq!( + display_args.image_args[0].refs[0].label.as_deref(), + Some("目标角色") + ); + assert_eq!(display_args.image_args[1].label, "参考图片"); + assert_eq!( + display_args.image_args[1].refs[0].label.as_deref(), + Some("像素风参考") + ); + assert!( + display_args + .image_args + .iter() + .flat_map(|arg| &arg.refs) + .all(|image| image.image_id.starts_with("sha256:")) + ); + assert_eq!(display_args.extras.price_mud_points, 3); } } @@ -740,7 +1148,7 @@ pub async fn confirm_editor_agent_tool_call( .await } GenerateIconSpritesheetTool::NAME => { - let args: GenerateIconSpritesheetToolArgs = serde_json::from_value(tool_args.clone()) + let args = serde_json::from_value::(tool_args.clone()) .map_err(|e| editor_agent_bad_request(format!("invalid tool call args: {e}")))?; let project = load_editor_agent_project(&state, &conversation).await?; let title = "图标素材图集".to_string(); diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs index 06803a14b..9a2939460 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs @@ -27,12 +27,10 @@ impl EditorToolContext { /// /// 通用 `Tool` 仍只负责参数校验；价格依赖 api-server 的运行时配置，不能下沉到 /// `module-editor-agent`。实际执行和扣费仍由既有生成 BFF 负责。 -#[allow(dead_code)] pub(crate) trait EditorAgentPricedTool: Tool { fn pricing(&self, pricing: &EditorGenerationPricingConfig, args: &::Args) -> u32; } -#[allow(dead_code)] pub(crate) fn editor_agent_image_mud_points( pricing: &EditorGenerationPricingConfig, kind: Option<&str>, diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs index 0f8505a9e..42a539eca 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs +++ b/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs @@ -24,11 +24,8 @@ pub struct GenerateVideoTool { pub context: crate::editor_agent::editor_tools::common::EditorToolContext, } -#[allow(dead_code)] const DEFAULT_VIDEO_MODEL: &str = "seedance2.0-fast"; -#[allow(dead_code)] const DEFAULT_VIDEO_RESOLUTION: &str = "720p"; -#[allow(dead_code)] const DEFAULT_VIDEO_DURATION_SECONDS: u32 = 4; #[derive(Debug, Clone)] diff --git a/server-rs/crates/shared-contracts/src/editor_agent.rs b/server-rs/crates/shared-contracts/src/editor_agent.rs index bda696606..c39ddd449 100644 --- a/server-rs/crates/shared-contracts/src/editor_agent.rs +++ b/server-rs/crates/shared-contracts/src/editor_agent.rs @@ -99,14 +99,60 @@ pub struct EditorAgentGeneratedAudio { pub audio_src: String, } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct EditorAgentToolCallDisplayArgs { + pub string_args: Vec, + pub image_args: Vec, + pub extras: EditorAgentToolCallDisplayExtras, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct EditorAgentToolCallDisplayExtras { + pub price_mud_points: u32, +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] +pub struct EditorAgentToolCallStringArg { + pub name: String, + pub label: String, + pub value: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct EditorAgentToolCallImageArg { + pub name: String, + pub label: String, + pub refs: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct EditorAgentToolCallImageRef { + pub image_id: String, + pub image_src: String, + #[serde(default)] + pub object_key: Option, + #[serde(default)] + pub thumbnail_src: Option, + #[serde(default)] + pub label: Option, + #[serde(default)] + pub width: Option, + #[serde(default)] + pub height: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct EditorAgentToolCall { pub tool_name: String, - #[serde(default)] - pub summary: String, pub status: EditorAgentToolCallStatus, pub args: serde_json::Value, + pub display_args: EditorAgentToolCallDisplayArgs, #[serde(default)] pub images: Vec, #[serde(default)] @@ -195,46 +241,79 @@ pub struct EditorAgentMessageResponse { #[cfg(test)] mod tests { - use super::*; - use serde_json::json; + use super::*; + use serde_json::json; - #[test] - fn tool_call_media_results_use_camel_case_and_default_for_existing_documents() { - let legacy: EditorAgentToolCall = serde_json::from_value(json!({ + #[test] + fn tool_call_display_args_and_media_use_camel_case() { + let missing_display_args = serde_json::from_value::(json!({ "toolName": "generate-video", "status": "completed", "args": {}, "images": [] - })) - .expect("existing message documents should remain readable"); - assert!(legacy.videos.is_empty()); - assert!(legacy.audios.is_empty()); + })); + assert!(missing_display_args.is_err()); - let tool_call = EditorAgentToolCall { - tool_name: "generate-video".to_string(), - summary: String::new(), - status: EditorAgentToolCallStatus::Completed, - args: json!({}), - images: Vec::new(), - videos: vec![EditorAgentGeneratedVideo { - resource_id: Some("resource-video-1".to_string()), - object_key: Some("generated/video.mp4".to_string()), - asset_object_id: Some("asset-video-1".to_string()), - video_src: "/generated-video.mp4".to_string(), - thumbnail_src: Some("/generated-video.png".to_string()), - width: Some(1280), - height: Some(720), - }], - audios: vec![EditorAgentGeneratedAudio { - resource_id: Some("resource-audio-1".to_string()), - object_key: Some("generated/audio.mp3".to_string()), - asset_object_id: Some("asset-audio-1".to_string()), - audio_src: "/generated-audio.mp3".to_string(), - }], - error: None, - }; - let payload = serde_json::to_value(tool_call).expect("tool call should serialize"); - assert_eq!(payload["videos"][0]["videoSrc"], "/generated-video.mp4"); - assert_eq!(payload["audios"][0]["audioSrc"], "/generated-audio.mp3"); - } + let tool_call = EditorAgentToolCall { + tool_name: "generate-video".to_string(), + status: EditorAgentToolCallStatus::Completed, + args: json!({}), + display_args: EditorAgentToolCallDisplayArgs { + string_args: vec![EditorAgentToolCallStringArg { + name: "prompt".to_string(), + label: "提示词".to_string(), + value: "生成像素风角色".to_string(), + }], + image_args: vec![EditorAgentToolCallImageArg { + name: "reference_image_ids".to_string(), + label: "参考图片".to_string(), + refs: vec![EditorAgentToolCallImageRef { + image_id: "sha256:reference-1".to_string(), + image_src: "/reference.png".to_string(), + object_key: Some("generated/reference.png".to_string()), + thumbnail_src: Some("/reference-thumbnail.png".to_string()), + label: Some("角色参考".to_string()), + width: Some(512), + height: Some(512), + }], + }], + extras: EditorAgentToolCallDisplayExtras { + price_mud_points: 5, + }, + }, + images: Vec::new(), + videos: vec![EditorAgentGeneratedVideo { + resource_id: Some("resource-video-1".to_string()), + object_key: Some("generated/video.mp4".to_string()), + asset_object_id: Some("asset-video-1".to_string()), + video_src: "/generated-video.mp4".to_string(), + thumbnail_src: Some("/generated-video.png".to_string()), + width: Some(1280), + height: Some(720), + }], + audios: vec![EditorAgentGeneratedAudio { + resource_id: Some("resource-audio-1".to_string()), + object_key: Some("generated/audio.mp3".to_string()), + asset_object_id: Some("asset-audio-1".to_string()), + audio_src: "/generated-audio.mp3".to_string(), + }], + error: None, + }; + let payload = serde_json::to_value(tool_call).expect("tool call should serialize"); + assert_eq!(payload["videos"][0]["videoSrc"], "/generated-video.mp4"); + assert_eq!(payload["audios"][0]["audioSrc"], "/generated-audio.mp3"); + assert_eq!( + payload["displayArgs"]["stringArgs"][0]["value"], + "生成像素风角色" + ); + assert_eq!( + payload["displayArgs"]["imageArgs"][0]["refs"][0]["imageId"], + "sha256:reference-1" + ); + assert_eq!( + payload["displayArgs"]["imageArgs"][0]["refs"][0]["objectKey"], + "generated/reference.png" + ); + assert_eq!(payload["displayArgs"]["extras"]["priceMudPoints"], 5); + } } diff --git a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx index 1104adac2..541274859 100644 --- a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx +++ b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx @@ -122,13 +122,53 @@ function createPendingToolCallMessage(): EditorAgentMessage { attachments: [], toolCall: { toolName: 'edit-image', - summary: '', status: 'pending_confirmation', args: { object_image_id: 'source-image-1', reference_image_ids: ['reference-image-1', 'reference-image-2'], prompt: '把角色换成像素风', }, + displayArgs: { + stringArgs: [ + { + name: 'prompt', + label: '提示词', + value: '把角色换成像素风', + }, + ], + imageArgs: [ + { + name: 'object_image_id', + label: '目标图片', + refs: [ + { + imageId: 'source-image-1', + imageSrc: 'data:image/png;base64,c291cmNl', + label: '原角色', + width: 512, + height: 512, + }, + ], + }, + { + name: 'reference_image_ids', + label: '参考图片', + refs: [ + { + imageId: 'reference-image-1', + imageSrc: 'data:image/png;base64,cmVmZXJlbmNlLTE=', + label: '像素风参考', + }, + { + imageId: 'reference-image-2', + imageSrc: 'data:image/png;base64,cmVmZXJlbmNlLTI=', + label: '配色参考', + }, + ], + }, + ], + extras: { priceMudPoints: 3 }, + }, images: [], error: null, }, @@ -500,8 +540,14 @@ describe('EditorAgentConversationPanelView', () => { ); expect(await screen.findByText('把角色换成像素风')).toBeTruthy(); - expect(screen.getByText('source-image-1')).toBeTruthy(); + expect(screen.getByText('提示词')).toBeTruthy(); expect(screen.getByText('2 张')).toBeTruthy(); + expect(screen.getByAltText('目标图片：原角色')).toBeTruthy(); + expect(screen.getByAltText('参考图片：像素风参考')).toBeTruthy(); + expect(screen.getByAltText('参考图片：配色参考')).toBeTruthy(); + expect(screen.getByText('预计消耗 3泥点')).toBeTruthy(); + expect(screen.queryByText('source-image-1')).toBeNull(); + expect(screen.queryByText('reference-image-1')).toBeNull(); expect( screen.queryByText('internal system prompt that must stay hidden'), ).toBeNull(); @@ -511,13 +557,12 @@ describe('EditorAgentConversationPanelView', () => { expect(client.confirmToolCall).toHaveBeenCalledWith('conversation-1', 2); }); expect( - (screen.getByRole('button', { name: '执行中' }) as HTMLButtonElement) - .disabled, - ).toBe(true); - expect( - (screen.getByRole('button', { name: '取消' }) as HTMLButtonElement) - .disabled, - ).toBe(true); + within(screen.getByRole('article', { name: 'Agent操作' })).getByText( + '执行中', + ), + ).toBeTruthy(); + expect(screen.queryByRole('button', { name: '确认' })).toBeNull(); + expect(screen.queryByRole('button', { name: '取消' })).toBeNull(); await act(async () => { resolveConfirmation({ diff --git a/src/components/image-editor/EditorAgentConversation/PendingToolCall.tsx b/src/components/image-editor/EditorAgentConversation/PendingToolCall.tsx index 753270118..7d30cdab6 100644 --- a/src/components/image-editor/EditorAgentConversation/PendingToolCall.tsx +++ b/src/components/image-editor/EditorAgentConversation/PendingToolCall.tsx @@ -1,7 +1,8 @@ -import { Check, Loader2, Pencil, X } from 'lucide-react'; +import { Check, Coins, Loader2, Pencil, X } from 'lucide-react'; import type { EditorAgentToolCall } from '@/packages/shared/src/contracts'; import { editorAgentToolLabel } from '@/src/components/image-editor/EditorAgentConversation/toolCallPresentation.ts'; +import { ResolvedAssetImage } from '@/src/components/ResolvedAssetImage.tsx'; type PendingToolCallAction = 'confirm' | 'cancel' | null; @@ -13,37 +14,10 @@ type PendingToolCallProps = { onCancel: (messageId: number) => void; }; -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null; -} - function readString(value: unknown) { return typeof value === 'string' && value.trim() ? value.trim() : null; } -function readStringArray(value: unknown) { - return Array.isArray(value) - ? value.map(readString).filter((item): item is string => item !== null) - : []; -} - -function readPendingToolDetails(toolCall: EditorAgentToolCall) { - const args = isRecord(toolCall.args) ? toolCall.args : {}; - const prompt = readString(args.prompt); - const targetImage = - readString(args.object_image_id) ?? readString(args.objectImageId); - const referenceImages = readStringArray( - args.reference_image_ids ?? args.referenceImageIds, - ); - const summary = readString(toolCall.summary); - - return { - prompt: prompt ?? (summary?.startsWith('{') ? null : summary), - targetImage, - referenceImageCount: referenceImages.length, - }; -} - export function PendingToolCall({ messageId, toolCall, @@ -51,7 +25,7 @@ export function PendingToolCall({ onConfirm, onCancel, }: PendingToolCallProps) { - const details = readPendingToolDetails(toolCall); + const displayArgs = toolCall.displayArgs; const label = editorAgentToolLabel(toolCall.toolName); const isBusy = busyAction !== null; @@ -66,32 +40,66 @@ export function PendingToolCall({ - {details.prompt ? ( -

- {details.prompt} -

- ) : null} +
+ {displayArgs.stringArgs.map((argument, index) => ( +
+
+ {argument.label} +
+
+ {argument.value} +
+
+ ))} - {details.targetImage || details.referenceImageCount > 0 ? ( -
- {details.targetImage ? ( -
-
目标图片
-
- {details.targetImage} -
+ {displayArgs.imageArgs.map((argument, argumentIndex) => ( +
+
+ {argument.label} + + {argument.refs.length} 张 +
- ) : null} - {details.referenceImageCount > 0 ? ( -
-
参考图片
-
- {details.referenceImageCount} 张 -
-
- ) : null} -
- ) : null} + {argument.refs.length ? ( +
+ {argument.refs.map((image, imageIndex) => { + const imageLabel = + readString(image.label) ?? `图片 ${imageIndex + 1}`; + return ( +
+
+ +
+ {image.label ? ( +
+ {image.label} +
+ ) : null} +
+ ); + })} +
+ ) : null} +
+ ))} + + +
+