diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index c2205378e..9dabc9cf4 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -3204,4 +3204,12 @@ - 原因：`UnifiedModal` 默认 portal 到 `document.body`；若业务入口只在页面内层继承 `platform-theme`，portal 根节点不会继承该容器的 CSS 变量。此时 `.platform-modal-shell` 的 `background: var(--platform-modal-fill)` 和 `.platform-overlay` 的背景声明都会失效。 - 处理：平台白底工具弹窗优先复用 `PlatformToolModalShell`，由共享壳读取当前 `AuthUiContext.platformTheme`，并把 `platform-theme platform-theme--` 挂到 portal overlay；不要用硬编码白底掩盖主题变量缺失。必须直接使用 `UnifiedModal` 的特殊场景，也要在 `overlayClassName` 显式传递当前平台主题。 - 验证：在 light / dark 主题下打开 portal 弹窗，断言 dialog 的 overlay 携带对应主题类，并在真实浏览器核对 panel 与遮罩的 computed background 均非透明。 -- 关联：`src/components/project/ProjectGalleryView.tsx`、`src/components/common/PlatformToolModalShell.tsx`、`src/components/common/UnifiedModal.tsx`。 +- 关联：`src/components/project/ProjectGalleryView.tsx`、`src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx`、`src/components/common/PlatformToolModalShell.tsx`、`src/components/common/UnifiedModal.tsx`。 + +## 待用户确认的 Agent 工具不能依赖模型自行结束回合 + +- 现象：画布 Agent 已生成有效工具规划，却最终只保存 `ERROR max turns reached: 3`，助手文本和待确认工具卡都消失。 +- 原因：八类画布工具的 `call()` 只返回待用户确认的规划结果，但 function-calling runner 在成功工具后仍继续请求 LLM，只靠 prompt 要求模型不再重试；模型连续返回工具调用直到上限后，错误结果又丢弃此前累积的输出。 +- 处理：工具通过框架契约显式声明 `requires_user_confirmation`；当本批全部工具都成功且等待确认时，runner 在处理完整批次后立即返回已有助手文本和工具结果。未知工具、参数错误、hook skip、普通连续工具和不可解析响应仍继续受 `max_turns` 门禁保护。不要用单纯提高轮次上限掩盖终止条件缺失。 +- 验证：runner 回归测试必须同时覆盖“待确认工具只调用一次 LLM 并成功结束”和“普通连续工具仍会触发 max-turn 门禁”。 +- 关联：`server-rs/crates/platform-editor-agent/src/framework/run.rs`、`server-rs/crates/platform-editor-agent/src/framework/tool.rs`、`server-rs/crates/platform-editor-agent/src/agent/tools/`。 diff --git a/docs/【编辑器】画布Agent对话面板-2026-07-03.md b/docs/【编辑器】画布Agent对话面板-2026-07-03.md index bbb9f3286..6cf3cc2b6 100644 --- a/docs/【编辑器】画布Agent对话面板-2026-07-03.md +++ b/docs/【编辑器】画布Agent对话面板-2026-07-03.md @@ -71,6 +71,7 @@ - 「素材库」页签：账号级素材库（复用 `ImageCanvasAssetLibrary` 数据源）； - 多选 + 底部「取消 / 应用」。 - 网格末尾上传格为后续补齐项；在上传格未落地前，对话附件只从已有画布资源和账号素材库选择。后续若从对话入口上传图片，必须复用素材库 / 画布资源登记链路，不新增对话私有图片类型。 +- 附件选择弹窗使用 `PlatformToolModalShell` 承接 portal 主题变量和不透明 panel 背景；不能直接把未注入 `platform-theme` 的 `UnifiedModal` portal 到 `document.body`，否则 `--platform-modal-fill` 失效后面板会变透明。 - 应用后附件以胶囊 chip 挂在输入框上方；发出的消息内附件渲染为纯文本胶囊 chip（名称 + 小图标），**默认无缩略图，鼠标悬浮才浮出缩略图预览**。 - 附件领域形状：统一为画布资源 / 素材库对象引用（`resourceId` / `assetId` + 可选 `objectKey`），不存在只属于对话的第三种图；单条消息上限 9 张（前后端共同校验）。前端可携带展示用 `imageSrc` / `thumbnailSrc`，后端必须按当前工程和当前账号重新归一、校验归属与 `objectKey`。 @@ -100,6 +101,7 @@ - 用户使用「这张」「刚才那个」「上一张」「把衣服换成……」等方式指代或编辑上一张结果图时，LLM 默认选择 `edit_image` 并引用 `latestGeneratedImage` 作为源图；除非用户明确要求全新生成，否则不能因为本轮没有重新上传附件而降级为 `generate_image`。 - 规划 prompt 必须显式区分“规范展板”和“实际素材产出”：规范图、视觉规范图、风格规范图、素材规范展板、角色规范图等规范展板请求走 `generate_image`，并补齐统一视角、线条粗细、色卡、材质、阴影、圆角、状态层级、尺寸标注等要求；实际角色立绘才走 `generate_character`，多个图标素材 / 图集才走 `generate_icon_spritesheet`。 - 画布 Agent 规划请求使用 Chat Completions、1024 `max_tokens` 和 60 秒 Agent 专用请求超时；生成图片/编辑图片仍走对应生成工具和模型计费。 +- function-calling runner 必须把“等待用户确认”作为显式工具语义：当本批所有工具都校验成功并进入待确认状态时，立即以成功结果结束当前规划回合并持久化助手文本与待确认卡，不得继续依赖 LLM 自行停止；未知工具、参数错误、普通连续工具和不可解析响应仍受 `max_turns` 保护。 - **对话回合免费**（聊天、分析回复不扣泥点），仅 Agent 实际触发生成工具时按对应模型定价扣泥点。 - 工具调用前后端校验泥点余额；不足时该次生成失败并在对话中以明确错误气泡告知，对话本身可继续。 diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index fefa47d57..d5b2a85a0 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -4453,6 +4453,7 @@ dependencies = [ "serde_json", "sha2", "shared-contracts", + "tokio", ] [[package]] diff --git a/server-rs/crates/platform-editor-agent/Cargo.toml b/server-rs/crates/platform-editor-agent/Cargo.toml index a38849a97..e6ea9f19a 100644 --- a/server-rs/crates/platform-editor-agent/Cargo.toml +++ b/server-rs/crates/platform-editor-agent/Cargo.toml @@ -12,3 +12,6 @@ serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } shared-contracts = { workspace = true, features = ["oss-contracts"] } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs index 865b78eb8..d54b83b89 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/edit_image.rs @@ -99,6 +99,10 @@ impl Tool for EditImageTool { } } + fn requires_user_confirmation(&self) -> bool { + true + } + fn classify_error(&self, error: &Self::Error) -> ToolFailure { match error { EditImageError::AssetNotFound(_) => { diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs index 1b262a431..9c19fe2ef 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_background_music.rs @@ -62,6 +62,10 @@ impl Tool for GenerateBackgroundMusicTool { }) } } + fn requires_user_confirmation(&self) -> bool { + true + } + fn classify_error(&self, error: &Self::Error) -> ToolFailure { ToolFailure::invalid_args(error.to_string()) } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs index 6cd34f41f..0b9227420 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_character.rs @@ -48,6 +48,10 @@ impl Tool for GenerateCharacterTool { } } + fn requires_user_confirmation(&self) -> bool { + true + } + fn classify_error(&self, error: &Self::Error) -> ToolFailure { GenerateImageTool { context: self.context.clone(), diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs index d1892b901..dfd2a008c 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_icon_spritesheet.rs @@ -106,6 +106,10 @@ impl Tool for GenerateIconSpritesheetTool { } } + fn requires_user_confirmation(&self) -> bool { + true + } + fn classify_error(&self, error: &Self::Error) -> ToolFailure { match error { GenerateIconSpritesheetError::AssetNotFound(_) => { diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs index 959c1c31f..e3c7a687e 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_image.rs @@ -94,6 +94,10 @@ impl Tool for GenerateImageTool { } } + fn requires_user_confirmation(&self) -> bool { + true + } + fn classify_error(&self, error: &Self::Error) -> ToolFailure { match error { GenerateImageError::AssetNotFound(_) => { diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs index 15fff27ec..59e5c77d9 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_sound_effect.rs @@ -69,6 +69,10 @@ impl Tool for GenerateSoundEffectTool { } } + fn requires_user_confirmation(&self) -> bool { + true + } + fn classify_error(&self, error: &Self::Error) -> ToolFailure { ToolFailure::invalid_args(error.to_string()) } diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs index 2a7ee56b3..49660a6de 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_ui_design.rs @@ -47,6 +47,10 @@ impl Tool for GenerateUiDesignTool { } } + fn requires_user_confirmation(&self) -> bool { + true + } + fn classify_error(&self, error: &Self::Error) -> ToolFailure { GenerateImageTool { context: self.context.clone(), diff --git a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs index 81cf19361..b03ad5c4b 100644 --- a/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs +++ b/server-rs/crates/platform-editor-agent/src/agent/tools/generate_video.rs @@ -86,6 +86,10 @@ impl Tool for GenerateVideoTool { }) } } + fn requires_user_confirmation(&self) -> bool { + true + } + fn classify_error(&self, error: &Self::Error) -> ToolFailure { match error { GenerateVideoError::AssetNotFound(_) => { diff --git a/server-rs/crates/platform-editor-agent/src/framework/run.rs b/server-rs/crates/platform-editor-agent/src/framework/run.rs index 2b4a90a1c..7dd7b036e 100644 --- a/server-rs/crates/platform-editor-agent/src/framework/run.rs +++ b/server-rs/crates/platform-editor-agent/src/framework/run.rs @@ -4,7 +4,7 @@ use crate::framework::error::PromptError; use crate::framework::hook::Hook; use crate::framework::memory::VecMemory; use crate::framework::run::PromptOutput::{Text, Tool}; -use crate::framework::tool::{ToolCall, ToolDyn, ToolExecutionResult, ToolFailure, ToolOutcome}; +use crate::framework::tool::{ToolCall, ToolExecutionResult, ToolFailure, ToolOutcome}; use serde::Deserialize; use serde_json::Value; use std::pin::Pin; @@ -138,6 +138,7 @@ where return Ok(prompt_result); } + let mut all_tool_calls_await_user_confirmation = true; for (tc_id, tc) in tool_calls.iter().enumerate() { // inline run_hooks: before_tool_call hook let mut should_skip = false; @@ -160,26 +161,26 @@ where } } if should_skip { + all_tool_calls_await_user_confirmation = false; continue; } - let result = { - let tools: Vec<&Box> = agent.tools.iter().collect(); - let name = tc.name.clone(); - let args = tc.args.clone(); - let fut = async move { - match tools.iter().find(|tool| tool.tool_name() == name) { - Some(tool) => tool.call(args).await, - None => ToolExecutionResult::failed( - Value::Null, - ToolFailure::invalid_args(format!( - "unknown tool: {name}" - )), - ), - } - }; - fut.await + let matching_tool = + agent.tools.iter().find(|tool| tool.tool_name() == tc.name); + let requires_user_confirmation = + matching_tool.is_some_and(|tool| tool.requires_user_confirmation()); + let result = match matching_tool { + Some(tool) => tool.call(tc.args.clone()).await, + None => ToolExecutionResult::failed( + Value::Null, + ToolFailure::invalid_args(format!("unknown tool: {}", tc.name)), + ), }; + if !requires_user_confirmation + || !matches!(&result.outcome, ToolOutcome::InternalOk) + { + all_tool_calls_await_user_confirmation = false; + } match result.outcome { ToolOutcome::InternalOk => { @@ -193,6 +194,7 @@ where )); } ToolCallFlow::Skip => { + all_tool_calls_await_user_confirmation = false; json_output = serde_json::json!({"message":"tool call is ignored by hook"}); break; } @@ -227,6 +229,10 @@ where } } } + + if all_tool_calls_await_user_confirmation { + return Ok(prompt_result); + } } Err(_) => { // Not valid JSON — push as assistant message and continue to next _turn @@ -276,3 +282,154 @@ pub fn clean_json_response(text: &str) -> String { } text.to_string() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::framework::agent::LlmApiAdaptor; + use crate::framework::hook::Hook; + use crate::framework::tool::Tool; + use serde_json::json; + use std::convert::Infallible; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct RepeatingToolCallModel { + completion_count: Arc, + } + + impl LlmApiAdaptor for RepeatingToolCallModel { + async fn complete<'a>( + &self, + _messages: impl Iterator + Send, + ) -> Result { + self.completion_count.fetch_add(1, Ordering::SeqCst); + Ok(json!({ + "reply_text": "请确认这次生成", + "tool_calls": [{ + "tool_name": "test-tool", + "args": { "prompt": "生成一张图" } + }] + }) + .to_string()) + } + + fn tool_result_message(&self, tool_name: &str, output: &str) -> String { + format!("{tool_name}: {output}") + } + + fn build_assistant_message(&self, text: &str) -> String { + text.to_string() + } + } + + struct TestTool { + requires_user_confirmation: bool, + } + + struct SkipAfterToolCallHook; + + impl Hook for SkipAfterToolCallHook { + fn after_tool_call(&self, _tool_name: &str, _output: &mut Value) -> ToolCallFlow { + ToolCallFlow::Skip + } + } + + impl Tool for TestTool { + const NAME: &'static str = "test-tool"; + type Error = Infallible; + type Args = Value; + type Output = Value; + + fn description(&self) -> String { + "test tool".to_string() + } + + fn parameters(&self) -> Value { + json!({ "type": "object" }) + } + + fn call( + &self, + _args: Self::Args, + ) -> impl Future> + Send { + async { Ok(json!({ "message": "pending user confirmation" })) } + } + + fn requires_user_confirmation(&self) -> bool { + self.requires_user_confirmation + } + } + + #[tokio::test] + async fn pending_confirmation_tool_batch_finishes_without_another_completion() { + let completion_count = Arc::new(AtomicUsize::new(0)); + let model = RepeatingToolCallModel { + completion_count: completion_count.clone(), + }; + let mut agent = Agent::new(model) + .tool(TestTool { + requires_user_confirmation: true, + }) + .max_turns(3); + + let outputs = agent + .prompt("生成一张图".to_string()) + .await + .expect("pending confirmation should finish the planning turn"); + + assert_eq!(completion_count.load(Ordering::SeqCst), 1); + assert_eq!(outputs.len(), 2); + assert!(matches!(outputs[0], PromptOutput::Text(_))); + assert!(matches!(outputs[1], PromptOutput::Tool(_))); + } + + #[tokio::test] + async fn non_confirmation_tool_keeps_the_existing_max_turn_guard() { + let completion_count = Arc::new(AtomicUsize::new(0)); + let model = RepeatingToolCallModel { + completion_count: completion_count.clone(), + }; + let mut agent = Agent::new(model) + .tool(TestTool { + requires_user_confirmation: false, + }) + .max_turns(3); + + let error = agent + .prompt("生成一张图".to_string()) + .await + .expect_err("a continuing tool should still hit the max-turn guard"); + + assert_eq!(completion_count.load(Ordering::SeqCst), 3); + assert!(matches!( + error, + PromptError::MaxTurnsReached { max_turns: 3 } + )); + } + + #[tokio::test] + async fn skipped_confirmation_result_keeps_the_existing_max_turn_guard() { + let completion_count = Arc::new(AtomicUsize::new(0)); + let model = RepeatingToolCallModel { + completion_count: completion_count.clone(), + }; + let mut agent = Agent::new(model) + .tool(TestTool { + requires_user_confirmation: true, + }) + .hook(SkipAfterToolCallHook) + .max_turns(3); + + let error = agent + .prompt("生成一张图".to_string()) + .await + .expect_err("a skipped result must not finish as pending confirmation"); + + assert_eq!(completion_count.load(Ordering::SeqCst), 3); + assert!(matches!( + error, + PromptError::MaxTurnsReached { max_turns: 3 } + )); + } +} diff --git a/server-rs/crates/platform-editor-agent/src/framework/tool.rs b/server-rs/crates/platform-editor-agent/src/framework/tool.rs index 386a9ed36..8eb4cf91b 100644 --- a/server-rs/crates/platform-editor-agent/src/framework/tool.rs +++ b/server-rs/crates/platform-editor-agent/src/framework/tool.rs @@ -140,6 +140,10 @@ pub trait Tool: Sized { args: Self::Args, ) -> impl Future> + Send; + fn requires_user_confirmation(&self) -> bool { + false + } + fn classify_error(&self, error: &Self::Error) -> ToolFailure { ToolFailure::other(error.to_string()) } @@ -150,6 +154,7 @@ pub trait ToolDyn: Send + Sync { fn tool_name(&self) -> &'static str; fn description(&self) -> String; fn parameters(&self) -> serde_json::Value; + fn requires_user_confirmation(&self) -> bool; fn call( &self, args: serde_json::Value, @@ -169,6 +174,10 @@ impl ToolDyn for T { self.parameters() } + fn requires_user_confirmation(&self) -> bool { + T::requires_user_confirmation(self) + } + fn call( &self, args: serde_json::Value, diff --git a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx index 737068d1e..8556f4876 100644 --- a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx +++ b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx @@ -268,6 +268,10 @@ describe('EditorAgentConversationPanelView', () => { const attachmentDialog = screen.getByRole('dialog', { name: '选择图片附件', }); + expect(attachmentDialog.parentElement?.className).toContain( + 'platform-theme--light', + ); + expect(attachmentDialog.className).toContain('platform-remap-surface'); expect( within(attachmentDialog).queryByRole('checkbox', { name: '选择画布图片 未入库图层', diff --git a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx index 7846c3b3c..5b055e7c4 100644 --- a/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx +++ b/src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.tsx @@ -23,7 +23,7 @@ import { } from '@/packages/shared/src/contracts'; import { PlatformActionButton } from '@/src/components/common/PlatformActionButton.tsx'; import { PlatformDangerConfirmDialog } from '@/src/components/common/PlatformDangerConfirmDialog.tsx'; -import { UnifiedModal } from '@/src/components/common/UnifiedModal.tsx'; +import { PlatformToolModalShell } from '@/src/components/common/PlatformToolModalShell.tsx'; import AttachmentChip from '@/src/components/image-editor/EditorAgentConversation/AttachmentChip.tsx'; import { attachmentKey } from '@/src/components/image-editor/EditorAgentConversation/common.ts'; import { @@ -150,7 +150,7 @@ function AttachmentPickerModal({ const visibleOptions = tab === 'canvas' ? canvasOptions : libraryOptions; return ( - - + ); }