diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index a010f230e..efe8e1790 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -3153,8 +3153,12 @@ dependencies = [ name = "module-editor-agent" version = "0.1.0" dependencies = [ + "hmac", + "platform-llm", "serde", "serde_json", + "sha2", + "shared-contracts", "shared-kernel", "spacetimedb", ] diff --git a/server-rs/crates/api-server/src/editor_agent/agent.rs b/server-rs/crates/api-server/src/editor_agent/agent.rs deleted file mode 100644 index 98709e806..000000000 --- a/server-rs/crates/api-server/src/editor_agent/agent.rs +++ /dev/null @@ -1,174 +0,0 @@ -use module_editor_agent::agent::agent::{Agent, LlmApiAdaptor}; -use module_editor_agent::agent::agent_builder::AgentBuilder; -use module_editor_agent::agent::error::PromptError; -use module_editor_agent::agent::hook::Hook; -use module_editor_agent::agent::memory::AgentMemory; -use module_editor_agent::agent::tool::{Tool, ToolDyn}; -use platform_llm::{LlmClient, LlmMessage}; -use serde_json::Value; - -pub(crate) struct LlmCompletionModel { - client: LlmClient, -} - -impl LlmApiAdaptor for LlmCompletionModel { - async fn complete<'a>( - &self, - messages: impl Iterator + Send, - ) -> Result { - use platform_llm::LlmTextRequest; - let request = - LlmTextRequest::new(messages.cloned().collect()).with_request_timeout_ms(30_000); - let response = self - .client - .request_text(request) - .await - .map_err(|e| PromptError::CompletionError(e.to_string()))?; - Ok(response.content) - } - - fn tool_result_message(&self, tool_name: &str, output: &str) -> LlmMessage { - LlmMessage::system(format!("Tool '{tool_name}' returned: {output}")) - } - - fn build_assistant_message(&self, text: &str) -> LlmMessage { - LlmMessage::assistant(text) - } -} - -pub struct LlmChatAgentBuilder { - client: Option, - system_prompt_parts: Vec, - tools: Vec>, - hooks: Vec>, - max_turns: usize, - memory_data: Option>>, -} - -impl AgentBuilder for LlmChatAgentBuilder { - type Client = LlmClient; - - fn new() -> Self { - Self { - client: None, - system_prompt_parts: Vec::new(), - tools: Vec::new(), - hooks: Vec::new(), - max_turns: 10, - memory_data: None, - } - } - - fn with_client(mut self, client: LlmClient) -> Self { - self.client = Some(client); - self - } - - fn system_prompt(mut self, system_prompt: impl Into) -> Self { - self.system_prompt_parts.push(system_prompt.into()); - self - } - - fn tool(mut self, tool: impl Tool + Send + Sync + 'static) -> Self { - self.tools.push(Box::new(tool)); - self - } - - fn add_hook(mut self, hook: impl Hook + 'static) -> Self { - self.hooks.push(Box::new(hook)); - self - } - - fn max_turns(mut self, n: usize) -> Self { - self.max_turns = n; - self - } - - fn memory(mut self, memory: impl AgentMemory + 'static) -> Self { - self.memory_data = Some(Box::new(memory)); - self - } - - fn build(self) -> Agent { - let model = LlmCompletionModel { - client: self.client.expect("call .with_client() first"), - }; - let mut agent = Agent::new(model); - let tool_specs = self - .tools - .iter() - .map(|tool| ToolPromptSpec { - name: tool.tool_name().to_string(), - description: tool.description(), - parameters: tool.parameters(), - }) - .collect::>(); - let base_prompt = self.system_prompt_parts.join("\n\n"); - let system_prompt = build_tools_system_prompt(&base_prompt, &tool_specs); - agent.tools = self.tools; - agent.hooks = self.hooks; - agent.default_max_turns = self.max_turns; - agent.memory = self.memory_data; - agent.system_prompt = Some(LlmMessage::system(&system_prompt)); - agent - } -} - -struct ToolPromptSpec { - name: String, - description: String, - parameters: Value, -} - -fn build_tools_system_prompt(base_prompt: &str, tool_specs: &[ToolPromptSpec]) -> String { - let mut prompt = String::new(); - prompt.push_str(base_prompt); - prompt.push_str("\n\nYou have access to the following tools.\n\n"); - - if tool_specs.is_empty() { - prompt.push_str("(No tools available.)\n"); - } else { - prompt.push_str("## JSON Response Format\n"); - prompt.push_str( - "When you need to use a tool, respond with valid JSON only (no markdown fences):\n", - ); - prompt.push_str("{\n"); - prompt.push_str(" \"reply_text\": \"your message to the user\",\n"); - prompt.push_str(" \"tool_calls\": [\n {\n"); - prompt.push_str(" \"tool_name\": \"tool_name_here\",\n"); - prompt.push_str(" \"args\": { \"argument_name\": \"argument_value\" }\n"); - prompt.push_str(" }\n ]\n"); - prompt.push_str("}\n\n"); - prompt.push_str("If you don't need to use a tool, respond with:\n"); - prompt.push_str("{\n"); - prompt.push_str(" \"reply_text\": \"your message\",\n"); - prompt.push_str(" \"tool_calls\": []\n"); - prompt.push_str("}\n\n"); - prompt.push_str("## Available Tools\n\n"); - - for tool in tool_specs { - prompt.push_str(&format!("- {}\n", tool.name)); - prompt.push_str(&format!(" Description: {}\n", tool.description)); - prompt.push_str(" Arguments JSON Schema:\n"); - let parameters = serde_json::to_string_pretty(&tool.parameters) - .unwrap_or_else(|_| tool.parameters.to_string()); - prompt.push_str(¶meters); - prompt.push_str("\n"); - } - - prompt.push_str( - "return your response immediately without expecting further tool execution. ", - ); - prompt.push_str( - "as you see, tool_calls is an array, several tools calls can be executed in one turn concurrently. ", - ); - prompt.push_str("For example: `\"reply_text\": \"Task complete. \"`."); - prompt.push_str("\n"); - prompt.push_str("IMPORTANT: Always respond with valid JSON only. "); - prompt.push_str( - "Do not wrap the JSON in markdown code fences or add extra text outside the JSON.", - ); - } - - prompt -} diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index d29fad773..13846f4ab 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -1,75 +1,76 @@ use axum::extract::{Path, State}; use axum::{Extension, Json}; -use module_editor_agent::agent::agent_builder::AgentBuilder; -use module_editor_agent::agent::error::PromptError; -use module_editor_agent::agent::memory::VecMemory; -use module_editor_agent::agent::run::PromptOutput; -use module_editor_agent::agent::tool::Tool; +use module_editor_agent::framework::agent_builder::AgentBuilder; +use module_editor_agent::framework::error::PromptError; +use module_editor_agent::framework::memory::VecMemory; +use module_editor_agent::framework::run::PromptOutput; +use module_editor_agent::framework::tool::Tool; use module_editor_agent::{ - EDITOR_AGENT_CONVERSATION_ID_PREFIX, EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE, - derive_conversation_title, editor_agent_messages_object_key, + derive_conversation_title, editor_agent_messages_object_key, + EDITOR_AGENT_CONVERSATION_ID_PREFIX, EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE, }; use platform_llm::LlmMessage; use serde::Serialize; -use serde_json::{Value, json}; +use serde_json::{json, Value}; use sha2::{Digest, Sha256}; use shared_contracts::assets::{ - EditorBackgroundMusicGenerateRequest, EditorSoundEffectGenerateRequest, - EditorVideoGenerateRequest, + EditorBackgroundMusicGenerateRequest, EditorSoundEffectGenerateRequest, + EditorVideoGenerateRequest, }; use shared_contracts::editor_agent::{ - CreateEditorAgentConversationRequest, EditorAgentConversationListResponse, - EditorAgentConversationMessagesDocument, EditorAgentConversationResponse, - EditorAgentConversationSummary, EditorAgentMessage, EditorAgentMessageRequest, - EditorAgentMessageResponse, EditorAgentMessageRole, EditorAgentToolCall, - EditorAgentToolCallStatus, + CreateEditorAgentConversationRequest, EditorAgentConversationListResponse, + EditorAgentConversationMessagesDocument, EditorAgentConversationResponse, + EditorAgentConversationSummary, EditorAgentMessage, EditorAgentMessageRequest, + EditorAgentMessageResponse, EditorAgentMessageRole, EditorAgentToolCall, + EditorAgentToolCallStatus, }; use spacetime_client::{ - EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput, - EditorAgentConversationRecord, EditorAgentConversationTouchRecordInput, - EditorProjectGetRecordInput, + EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput, + EditorAgentConversationRecord, EditorAgentConversationTouchRecordInput, + EditorProjectGetRecordInput, }; use crate::api_response::json_success_body; use crate::auth::AuthenticatedAccessToken; -use crate::editor_agent::agent::LlmChatAgentBuilder; -use crate::editor_agent::editor_tools::common::{EditorAgentPricedTool, EditorToolContext}; -use crate::editor_agent::editor_tools::edit_image::{EditImageTool, EditImageToolArgs}; -use crate::editor_agent::editor_tools::generate_background_music::{ - GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, +use module_editor_agent::agent::agent::LlmChatAgentBuilder; +use crate::editor_agent::pricing::EditorAgentPricedTool; +use module_editor_agent::agent::tools::edit_image::{EditImageTool, EditImageToolArgs}; +use module_editor_agent::agent::tools::generate_background_music::{ + GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, }; -use crate::editor_agent::editor_tools::generate_character::GenerateCharacterTool; -use crate::editor_agent::editor_tools::generate_icon_spritesheet::{ - GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, +use module_editor_agent::agent::tools::generate_character::GenerateCharacterTool; +use module_editor_agent::agent::tools::generate_icon_spritesheet::{ + GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, }; -use crate::editor_agent::editor_tools::generate_image::{GenerateImageTool, GenerateImageToolArgs}; -use crate::editor_agent::editor_tools::generate_sound_effect::{ - GenerateSoundEffectTool, GenerateSoundEffectToolArgs, +use module_editor_agent::agent::tools::generate_image::{GenerateImageTool, GenerateImageToolArgs}; +use module_editor_agent::agent::tools::generate_sound_effect::{ + GenerateSoundEffectTool, GenerateSoundEffectToolArgs, }; -use crate::editor_agent::editor_tools::generate_ui_design::GenerateUiDesignTool; -use crate::editor_agent::editor_tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; +use module_editor_agent::agent::tools::generate_ui_design::GenerateUiDesignTool; +use module_editor_agent::agent::tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; use crate::editor_agent::utils::{ - IntoImageId, build_editor_agent_canvas_completion, conversation_detail_from_record, - conversation_summary_from_record, editor_agent_bad_request, empty_messages_document, - ensure_editor_project_access, normalize_editor_agent_attachments, now_rfc3339, - read_messages_document, require_editor_agent_sidebar_enabled, write_messages_document, + build_editor_agent_canvas_completion, conversation_detail_from_record, conversation_summary_from_record, + editor_agent_bad_request, empty_messages_document, ensure_editor_project_access, + normalize_editor_agent_attachments, now_rfc3339, read_messages_document, + require_editor_agent_sidebar_enabled, write_messages_document, IntoImageId, }; use crate::editor_agent::{context, display_args, reconcile}; use crate::editor_generation_config::EditorGenerationPricingConfig; use crate::editor_generation_queue::{ - EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND, - EDITOR_IMAGE_EDIT_JOB_KIND, EDITOR_IMAGE_GENERATION_JOB_KIND, - EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND, EDITOR_VIDEO_GENERATION_JOB_KIND, - enqueue_editor_generation_job_with_identity, + enqueue_editor_generation_job_with_identity, EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, + EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND, EDITOR_IMAGE_EDIT_JOB_KIND, + EDITOR_IMAGE_GENERATION_JOB_KIND, EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND, + EDITOR_VIDEO_GENERATION_JOB_KIND, }; use crate::editor_project::{ - EditorIconSpritesheetGenerationRequest, EditorImageEditRequest, EditorImageGenerationRequest, + EditorIconSpritesheetGenerationRequest, EditorImageEditRequest, EditorImageGenerationRequest, }; use crate::editor_project::{current_utc_micros, map_editor_project_error}; use crate::http_error::AppError; use crate::request_context::RequestContext; use crate::state::AppState; use shared_kernel::{build_prefixed_uuid_id, normalize_optional_string}; +use module_editor_agent::agent::tools::context::EditorToolContext; pub async fn editor_agent_message( State(state): State, @@ -219,7 +220,7 @@ pub async fn editor_agent_message( delta_messages: Vec::new(), error_message: Some(err.to_string()), })), - Ok(mut delta_messages) => { + Ok(delta_messages) => { for msg in &delta_messages { document.messages.push(msg.clone()); } diff --git a/server-rs/crates/api-server/src/editor_agent/context.rs b/server-rs/crates/api-server/src/editor_agent/context.rs index 87d1e3f5c..94c478de5 100644 --- a/server-rs/crates/api-server/src/editor_agent/context.rs +++ b/server-rs/crates/api-server/src/editor_agent/context.rs @@ -1,7 +1,8 @@ -use crate::editor_agent::editor_tools::common::EditorToolContext; -use crate::editor_agent::utils::{ImageId, ImageMetadata, IntoDataKey}; +use module_editor_agent::agent::tools::context::EditorToolContext; +use crate::editor_agent::utils::IntoDataKey; use shared_contracts::editor_agent::EditorAgentConversationMessagesDocument; use std::collections::HashMap; +use module_editor_agent::agent::asset::{ImageId, ImageMetadata}; pub fn build_tool_context(document: &EditorAgentConversationMessagesDocument) -> EditorToolContext { let mut images: HashMap = HashMap::new(); @@ -19,7 +20,7 @@ pub fn build_tool_context(document: &EditorAgentConversationMessagesDocument) -> images.insert(image_id, metadata); } } - + // user pointed for a in &msg.attachments { let data_key = a.clone().into_data_key(); @@ -33,4 +34,4 @@ pub fn build_tool_context(document: &EditorAgentConversationMessagesDocument) -> } EditorToolContext { images } -} \ No newline at end of file +} diff --git a/server-rs/crates/api-server/src/editor_agent/display_args.rs b/server-rs/crates/api-server/src/editor_agent/display_args.rs index 84ee705e4..0953b7f44 100644 --- a/server-rs/crates/api-server/src/editor_agent/display_args.rs +++ b/server-rs/crates/api-server/src/editor_agent/display_args.rs @@ -1,17 +1,19 @@ use serde_json::Value; -use module_editor_agent::agent::error::PromptError; -use module_editor_agent::agent::tool::Tool; +use module_editor_agent::framework::error::PromptError; +use module_editor_agent::framework::tool::Tool; use shared_contracts::editor_agent::{EditorAgentConversationMessagesDocument, EditorAgentToolCallDisplayArgs, EditorAgentToolCallDisplayExtras, EditorAgentToolCallImageArg, EditorAgentToolCallImageRef, EditorAgentToolCallStringArg}; -use crate::editor_agent::editor_tools::common::{EditorAgentPricedTool, EditorToolContext}; -use crate::editor_agent::editor_tools::edit_image::{EditImageTool, EditImageToolArgs}; -use crate::editor_agent::editor_tools::generate_background_music::{GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs}; -use crate::editor_agent::editor_tools::generate_character::GenerateCharacterTool; -use crate::editor_agent::editor_tools::generate_icon_spritesheet::{GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs}; -use crate::editor_agent::editor_tools::generate_image::{GenerateImageTool, GenerateImageToolArgs}; -use crate::editor_agent::editor_tools::generate_sound_effect::{GenerateSoundEffectTool, GenerateSoundEffectToolArgs}; -use crate::editor_agent::editor_tools::generate_ui_design::GenerateUiDesignTool; -use crate::editor_agent::editor_tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; -use crate::editor_agent::utils::{ImageId, IntoDataKey, IntoImageId}; +use crate::editor_agent::pricing::EditorAgentPricedTool; +use module_editor_agent::agent::tools::context::EditorToolContext; +use module_editor_agent::agent::tools::edit_image::{EditImageTool, EditImageToolArgs}; +use module_editor_agent::agent::tools::generate_background_music::{GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs}; +use module_editor_agent::agent::tools::generate_character::GenerateCharacterTool; +use module_editor_agent::agent::tools::generate_icon_spritesheet::{GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs}; +use module_editor_agent::agent::tools::generate_image::{GenerateImageTool, GenerateImageToolArgs}; +use module_editor_agent::agent::tools::generate_sound_effect::{GenerateSoundEffectTool, GenerateSoundEffectToolArgs}; +use module_editor_agent::agent::tools::generate_ui_design::GenerateUiDesignTool; +use module_editor_agent::agent::tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; +use module_editor_agent::agent::asset::ImageId; +use crate::editor_agent::utils::{IntoDataKey, IntoImageId}; use crate::editor_generation_config::EditorGenerationPricingConfig; pub fn build_tool_call_display_args( @@ -311,4 +313,4 @@ pub fn resolve_tool_call_image_ref( } None -} \ No newline at end of file +} diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs b/server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs deleted file mode 100644 index c13977897..000000000 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs +++ /dev/null @@ -1,56 +0,0 @@ -use crate::editor_agent::utils::{ImageId, ImageMetadata}; -use crate::editor_generation_config::EditorGenerationPricingConfig; -use crate::http_error::AppError; -use crate::openai_image_generation::GPT_IMAGE_2_MODEL; -use axum::http::StatusCode; -use module_editor_agent::agent::tool::Tool; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use std::collections::HashMap; - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct EditorToolContext { - pub images: HashMap, -} - -impl EditorToolContext { - /// Check if an image with the given ID exists in the context. - pub fn contains_image(&self, image_id: &ImageId) -> bool { - self.images.contains_key(image_id) - } - - pub fn image_data_key(&self, image_id: &ImageId) -> Option<&str> { - self.images - .get(image_id) - .map(|metadata| metadata.data_key.as_str()) - } -} - -/// api-server 侧的画布 Agent 工具计价扩展。 -/// -/// 通用 `Tool` 仍只负责参数校验;价格依赖 api-server 的运行时配置,不能下沉到 -/// `module-editor-agent`。实际执行和扣费仍由既有生成 BFF 负责。 -pub(crate) trait EditorAgentPricedTool: Tool { - fn pricing(&self, pricing: &EditorGenerationPricingConfig, args: &::Args) -> u32; -} - -pub(crate) fn editor_agent_image_mud_points( - pricing: &EditorGenerationPricingConfig, - kind: Option<&str>, - image_size: Option<&str>, -) -> u32 { - // 这些 Agent 工具当前向既有 BFF 传 model=None;BFF 会先归一为 gpt-image-2。 - // 尺寸同样只把精确的 2K 识别为 2K,其余值回落到 1K。 - let normalized_image_size = match image_size.map(str::trim) { - Some("2K") => "2K", - _ => "1K", - }; - pricing.image_generation_mud_points(kind, Some(GPT_IMAGE_2_MODEL), Some(normalized_image_size)) -} - -pub fn image_not_found(image_id: &ImageId) -> AppError { - AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ - "provider": "editor-agent", - "message": format!("asset {image_id} not found in context"), - })) -} diff --git a/server-rs/crates/api-server/src/editor_agent/mod.rs b/server-rs/crates/api-server/src/editor_agent/mod.rs index f65f6f288..ecbc880e9 100644 --- a/server-rs/crates/api-server/src/editor_agent/mod.rs +++ b/server-rs/crates/api-server/src/editor_agent/mod.rs @@ -1,15 +1,13 @@ -mod editor_tools; mod utils; pub mod api; -mod agent; mod display_args; mod context; mod resp_to_asset; mod reconcile; +pub mod pricing; pub use api::{ - create_editor_agent_conversation, delete_editor_agent_conversation, - get_editor_agent_conversation, list_editor_agent_conversations, - confirm_editor_agent_tool_call, cancel_editor_agent_tool_call, + cancel_editor_agent_tool_call, confirm_editor_agent_tool_call, + create_editor_agent_conversation, delete_editor_agent_conversation, + get_editor_agent_conversation, list_editor_agent_conversations, }; -pub use utils::build_editor_agent_canvas_completion; diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/pricing_tests.rs b/server-rs/crates/api-server/src/editor_agent/pricing.rs similarity index 50% rename from server-rs/crates/api-server/src/editor_agent/editor_tools/pricing_tests.rs rename to server-rs/crates/api-server/src/editor_agent/pricing.rs index 8c66aa585..db48ea729 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/pricing_tests.rs +++ b/server-rs/crates/api-server/src/editor_agent/pricing.rs @@ -1,22 +1,22 @@ use std::collections::HashMap; - -use crate::editor_agent::editor_tools::common::{EditorAgentPricedTool, EditorToolContext}; -use crate::editor_agent::editor_tools::edit_image::{EditImageTool, EditImageToolArgs}; -use crate::editor_agent::editor_tools::generate_background_music::{ - GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, +use module_editor_agent::framework::tool::Tool; +use platform_image::GPT_IMAGE_2_MODEL; +use module_editor_agent::agent::tools::context::EditorToolContext; +use module_editor_agent::agent::tools::edit_image::{EditImageTool, EditImageToolArgs}; +use module_editor_agent::agent::tools::generate_background_music::{ + GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, }; -use crate::editor_agent::editor_tools::generate_character::GenerateCharacterTool; -use crate::editor_agent::editor_tools::generate_icon_spritesheet::{ - GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, +use module_editor_agent::agent::tools::generate_character::GenerateCharacterTool; +use module_editor_agent::agent::tools::generate_icon_spritesheet::{ + GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, }; -use crate::editor_agent::editor_tools::generate_image::{GenerateImageTool, GenerateImageToolArgs}; -use crate::editor_agent::editor_tools::generate_sound_effect::{ - GenerateSoundEffectTool, GenerateSoundEffectToolArgs, +use module_editor_agent::agent::tools::generate_image::{GenerateImageTool, GenerateImageToolArgs}; +use module_editor_agent::agent::tools::generate_sound_effect::{ + GenerateSoundEffectTool, GenerateSoundEffectToolArgs, }; -use crate::editor_agent::editor_tools::generate_ui_design::GenerateUiDesignTool; -use crate::editor_agent::editor_tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; -use crate::editor_agent::utils::ImageId; -use crate::editor_generation_config::load_editor_generation_pricing_from_paths; +use module_editor_agent::agent::tools::generate_ui_design::GenerateUiDesignTool; +use module_editor_agent::agent::tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; +use crate::editor_generation_config::{load_editor_generation_pricing_from_paths, EditorGenerationPricingConfig}; fn context() -> EditorToolContext { EditorToolContext { @@ -188,3 +188,122 @@ fn pricing_uses_the_supplied_runtime_snapshot() { 19 ); } + +impl EditorAgentPricedTool for GenerateVideoTool { + fn pricing( + &self, + pricing: &EditorGenerationPricingConfig, + args: &GenerateVideoToolArgs, + ) -> u32 { + let model = args + .model + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(Self::DEFAULT_VIDEO_MODEL); + let resolution = args + .resolution + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(Self::DEFAULT_VIDEO_RESOLUTION); + let duration_seconds = args + .duration_seconds + .unwrap_or(Self::DEFAULT_VIDEO_DURATION_SECONDS); + pricing.video_model_mud_points(Some(model), resolution, duration_seconds) + } +} + +impl EditorAgentPricedTool for GenerateUiDesignTool { + fn pricing( + &self, + pricing: &EditorGenerationPricingConfig, + args: &GenerateImageToolArgs, + ) -> u32 { + editor_agent_image_mud_points(pricing, Some("ui-design"), args.image_size.as_deref()) + } +} + +impl EditorAgentPricedTool for GenerateSoundEffectTool { + fn pricing( + &self, + pricing: &EditorGenerationPricingConfig, + args: &GenerateSoundEffectToolArgs, + ) -> u32 { + let model = args + .model + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(Self::DEFAULT_MODEL); + pricing.sound_effect_model_mud_points(Some(model)) + } +} + +impl EditorAgentPricedTool for GenerateImageTool { + fn pricing( + &self, + pricing: &EditorGenerationPricingConfig, + args: &GenerateImageToolArgs, + ) -> u32 { + editor_agent_image_mud_points(pricing, None, args.image_size.as_deref()) + } +} + +impl EditorAgentPricedTool for GenerateIconSpritesheetTool { + fn pricing( + &self, + pricing: &EditorGenerationPricingConfig, + args: &GenerateIconSpritesheetToolArgs, + ) -> u32 { + editor_agent_image_mud_points(pricing, Some("icon"), args.image_size.as_deref()) + } +} + +impl EditorAgentPricedTool for GenerateCharacterTool { + fn pricing( + &self, + pricing: &EditorGenerationPricingConfig, + args: &GenerateImageToolArgs, + ) -> u32 { + editor_agent_image_mud_points(pricing, Some("character"), args.image_size.as_deref()) + } +} + +/// api-server 侧的画布 Agent 工具计价扩展。 +/// +/// 通用 `Tool` 仍只负责参数校验;价格依赖 api-server 的运行时配置,不能下沉到 +/// `module-editor-agent`。实际执行和扣费仍由既有生成 BFF 负责。 +pub(crate) trait EditorAgentPricedTool: Tool { + fn pricing(&self, pricing: &EditorGenerationPricingConfig, args: &::Args) -> u32; +} + +pub(crate) fn editor_agent_image_mud_points( + pricing: &EditorGenerationPricingConfig, + kind: Option<&str>, + image_size: Option<&str>, +) -> u32 { + // 这些 Agent 工具当前向既有 BFF 传 model=None;BFF 会先归一为 gpt-image-2。 + // 尺寸同样只把精确的 2K 识别为 2K,其余值回落到 1K。 + let normalized_image_size = match image_size.map(str::trim) { + Some("2K") => "2K", + _ => "1K", + }; + pricing.image_generation_mud_points(kind, Some(GPT_IMAGE_2_MODEL), Some(normalized_image_size)) +} + +impl EditorAgentPricedTool for GenerateBackgroundMusicTool { + fn pricing( + &self, + pricing: &EditorGenerationPricingConfig, + _args: &GenerateBackgroundMusicToolArgs, + ) -> u32 { + pricing.background_music_model_mud_points(Some(Self::DEFAULT_MODEL)) + } +} + +impl EditorAgentPricedTool for EditImageTool { + fn pricing(&self, pricing: &EditorGenerationPricingConfig, _args: &EditImageToolArgs) -> u32 { + editor_agent_image_mud_points(pricing, Some("quick-edit"), Some("1K")) + } +} diff --git a/server-rs/crates/api-server/src/editor_agent/reconcile.rs b/server-rs/crates/api-server/src/editor_agent/reconcile.rs index 5362e9888..6a4e44902 100644 --- a/server-rs/crates/api-server/src/editor_agent/reconcile.rs +++ b/server-rs/crates/api-server/src/editor_agent/reconcile.rs @@ -1,26 +1,26 @@ -use crate::editor_agent::editor_tools::common::EditorToolContext; -use crate::editor_agent::editor_tools::edit_image::{ +use module_editor_agent::agent::tools::context::EditorToolContext; +use module_editor_agent::agent::tools::edit_image::{ EditImageTool, EditImageToolArgs, EditorImageEditResult, }; -use crate::editor_agent::editor_tools::generate_background_music::{ +use module_editor_agent::agent::tools::generate_background_music::{ GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs, }; -use crate::editor_agent::editor_tools::generate_character::GenerateCharacterTool; -use crate::editor_agent::editor_tools::generate_icon_spritesheet::{ +use module_editor_agent::agent::tools::generate_character::GenerateCharacterTool; +use module_editor_agent::agent::tools::generate_icon_spritesheet::{ EditorIconSpritesheetResult, GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs, }; -use crate::editor_agent::editor_tools::generate_image::{ +use module_editor_agent::agent::tools::generate_image::{ EditorImageGenerationResult, GenerateImageTool, GenerateImageToolArgs, }; -use crate::editor_agent::editor_tools::generate_sound_effect::{ +use module_editor_agent::agent::tools::generate_sound_effect::{ GenerateSoundEffectTool, GenerateSoundEffectToolArgs, }; -use crate::editor_agent::editor_tools::generate_ui_design::GenerateUiDesignTool; -use crate::editor_agent::editor_tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; +use module_editor_agent::agent::tools::generate_ui_design::GenerateUiDesignTool; +use module_editor_agent::agent::tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs}; use crate::editor_agent::resp_to_asset; use crate::http_error::AppError; use crate::state::AppState; -use module_editor_agent::agent::tool::Tool; +use module_editor_agent::framework::tool::Tool; use serde_json::Value; use shared_contracts::assets::{EditorAudioGenerateResponse, EditorVideoGenerateResponse}; use shared_contracts::editor_agent::{ diff --git a/server-rs/crates/api-server/src/editor_agent/resp_to_asset.rs b/server-rs/crates/api-server/src/editor_agent/resp_to_asset.rs index a365e72a1..12195de11 100644 --- a/server-rs/crates/api-server/src/editor_agent/resp_to_asset.rs +++ b/server-rs/crates/api-server/src/editor_agent/resp_to_asset.rs @@ -1,9 +1,9 @@ use serde_json::Value; use shared_contracts::assets::{EditorAudioGenerateResponse, EditorVideoGenerateResponse}; use shared_contracts::editor_agent::{EditorAgentGeneratedAudio, EditorAgentGeneratedImage, EditorAgentGeneratedVideo}; -use crate::editor_agent::editor_tools::edit_image::EditorImageEditResult; -use crate::editor_agent::editor_tools::generate_icon_spritesheet::EditorIconSpritesheetResult; -use crate::editor_agent::editor_tools::generate_image::EditorImageGenerationResult; +use module_editor_agent::agent::tools::edit_image::EditorImageEditResult; +use module_editor_agent::agent::tools::generate_icon_spritesheet::EditorIconSpritesheetResult; +use module_editor_agent::agent::tools::generate_image::EditorImageGenerationResult; fn value_string(value: Option<&Value>, field: &str) -> Option { value diff --git a/server-rs/crates/api-server/src/editor_agent/utils.rs b/server-rs/crates/api-server/src/editor_agent/utils.rs index cc5ee9d24..d746a2fae 100644 --- a/server-rs/crates/api-server/src/editor_agent/utils.rs +++ b/server-rs/crates/api-server/src/editor_agent/utils.rs @@ -4,27 +4,25 @@ use crate::platform_errors::map_oss_error; use crate::state::AppState; use axum::http::StatusCode; use platform_oss::{ - LegacyAssetPrefix, OssObjectAccess, OssPutObjectRequest, OssSignedGetObjectUrlRequest, + LegacyAssetPrefix, OssObjectAccess, OssPutObjectRequest, OssSignedGetObjectUrlRequest, }; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; -use sha2::{Digest, Sha256}; +use serde_json::{json, Value}; use shared_contracts::assets::{ - EditorCanvasGenerationCompletionPayload, EditorCanvasGenerationPlaceholderPayload, + EditorCanvasGenerationCompletionPayload, EditorCanvasGenerationPlaceholderPayload, }; use shared_contracts::editor_agent::{ - EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, EditorAgentAttachmentRef, EditorAgentAttachmentSource, - EditorAgentConversationDetail, EditorAgentConversationMessagesDocument, EditorAgentGeneratedImage, - EditorAgentConversationSummary, EditorAgentMessage, + EditorAgentAttachmentRef, EditorAgentAttachmentSource, EditorAgentConversationDetail, + EditorAgentConversationMessagesDocument, EditorAgentConversationSummary, EditorAgentGeneratedImage, + EditorAgentMessage, EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, }; use shared_kernel::{normalize_optional_string, normalize_required_string}; use spacetime_client::{ - EditorAgentConversationRecord, EditorAssetLibraryRecord, EditorAssetRecord, - EditorProjectGetRecordInput, EditorProjectRecord, EditorProjectResourceRecord, + EditorAgentConversationRecord, EditorAssetLibraryRecord, EditorAssetRecord, + EditorProjectGetRecordInput, EditorProjectRecord, EditorProjectResourceRecord, }; use std::collections::BTreeMap; -use std::fmt::Display; use std::sync::{Arc, Mutex, OnceLock}; +use module_editor_agent::agent::asset::ImageId; pub trait IntoDataKey { fn into_data_key(self) -> String; @@ -107,48 +105,9 @@ impl IntoResourceId for EditorAssetRecord { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ImageId { - pub id: String, -} - -impl ImageId { - /// Keep data keys server-side while giving the model a stable image reference. - pub fn from_data_key(data_key: impl AsRef) -> Self { - let digest = Sha256::digest(data_key.as_ref().as_bytes()); - Self { - id: format!("sha256:{digest:x}"), - } - } -} - -impl Serialize for ImageId { - fn serialize(&self, serializer: S) -> Result { - self.id.serialize(serializer) - } -} - -impl<'de> Deserialize<'de> for ImageId { - fn deserialize>(deserializer: D) -> Result { - String::deserialize(deserializer).map(|id| ImageId { id }) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ImageMetadata { - pub tag: String, - pub data_key: String, -} - -impl Display for ImageId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.id) - } -} - #[cfg(test)] mod image_id_tests { - use super::ImageId; + use module_editor_agent::agent::asset::ImageId; #[test] fn image_id_is_a_stable_hash_of_the_data_key() { diff --git a/server-rs/crates/module-editor-agent/Cargo.toml b/server-rs/crates/module-editor-agent/Cargo.toml index 188b483f0..f1a9aca94 100644 --- a/server-rs/crates/module-editor-agent/Cargo.toml +++ b/server-rs/crates/module-editor-agent/Cargo.toml @@ -13,3 +13,7 @@ serde = { workspace = true } serde_json = { workspace = true } shared-kernel = { workspace = true } spacetimedb = { workspace = true, optional = true } +platform-llm = { workspace = true } +shared-contracts = { workspace = true } +hmac = { workspace = true } +sha2 = { workspace = true } diff --git a/server-rs/crates/module-editor-agent/src/agent/agent.rs b/server-rs/crates/module-editor-agent/src/agent/agent.rs index e6a656300..0b8ed41c6 100644 --- a/server-rs/crates/module-editor-agent/src/agent/agent.rs +++ b/server-rs/crates/module-editor-agent/src/agent/agent.rs @@ -1,80 +1,174 @@ -use crate::agent::error::PromptError; -use crate::agent::hook::Hook; -use crate::agent::memory::AgentMemory; -use crate::agent::run::PromptRequest; -use crate::agent::{Tool, ToolDyn}; +use crate::framework::agent::{Agent, LlmApiAdaptor}; +use crate::framework::agent_builder::AgentBuilder; +use crate::framework::error::PromptError; +use crate::framework::hook::Hook; +use crate::framework::memory::AgentMemory; +use crate::framework::tool::{Tool, ToolDyn}; +use platform_llm::{LlmClient, LlmMessage}; +use serde_json::Value; -pub struct Agent, Message> { - pub model: M, - pub tools: Vec>, - pub hooks: Vec>, - pub default_max_turns: usize, - pub system_prompt: Option, - pub memory: Option>>, +pub struct LlmCompletionModel { + client: LlmClient, } -impl Agent -where - M: LlmApiAdaptor + 'static, - Message: Send + 'static, -{ - pub fn new(model: M) -> Self { +impl LlmApiAdaptor for LlmCompletionModel { + async fn complete<'a>( + &self, + messages: impl Iterator + Send, + ) -> Result { + use platform_llm::LlmTextRequest; + let request = + LlmTextRequest::new(messages.cloned().collect()).with_request_timeout_ms(30_000); + let response = self + .client + .request_text(request) + .await + .map_err(|e| PromptError::CompletionError(e.to_string()))?; + Ok(response.content) + } + + fn tool_result_message(&self, tool_name: &str, output: &str) -> LlmMessage { + LlmMessage::system(format!("Tool '{tool_name}' returned: {output}")) + } + + fn build_assistant_message(&self, text: &str) -> LlmMessage { + LlmMessage::assistant(text) + } +} + +pub struct LlmChatAgentBuilder { + client: Option, + system_prompt_parts: Vec, + tools: Vec>, + hooks: Vec>, + max_turns: usize, + memory_data: Option>>, +} + +impl AgentBuilder for LlmChatAgentBuilder { + type Client = LlmClient; + + fn new() -> Self { Self { - model, + client: None, + system_prompt_parts: Vec::new(), tools: Vec::new(), hooks: Vec::new(), - default_max_turns: 10, - system_prompt: None, - memory: None, + max_turns: 10, + memory_data: None, } } - pub fn tool(mut self, tool: impl Tool + Send + Sync + 'static) -> Self { + fn with_client(mut self, client: LlmClient) -> Self { + self.client = Some(client); + self + } + + fn system_prompt(mut self, system_prompt: impl Into) -> Self { + self.system_prompt_parts.push(system_prompt.into()); + self + } + + fn tool(mut self, tool: impl Tool + Send + Sync + 'static) -> Self { self.tools.push(Box::new(tool)); self } - pub fn system_prompt(mut self, msg: Message) -> Self { - self.system_prompt = Some(msg); - self - } - - pub fn memory(mut self, mem: impl AgentMemory + 'static) -> Self { - self.memory = Some(Box::new(mem)); - self - } - - pub fn hook(mut self, hook: impl Hook + 'static) -> Self { + fn add_hook(mut self, hook: impl Hook + 'static) -> Self { self.hooks.push(Box::new(hook)); self } - pub fn max_turns(mut self, n: usize) -> Self { - self.default_max_turns = n; + fn max_turns(mut self, n: usize) -> Self { + self.max_turns = n; self } - pub fn tools(&self) -> &[Box] { - &self.tools + fn memory(mut self, memory: impl AgentMemory + 'static) -> Self { + self.memory_data = Some(Box::new(memory)); + self } - - pub fn prompt(&mut self, message: impl Into + Send) -> PromptRequest<'_, M, Message> - where - Message: 'static, - { - PromptRequest::new(self, message.into()) + fn build(self) -> Agent { + let model = LlmCompletionModel { + client: self.client.expect("call .with_client() first"), + }; + let mut agent = Agent::new(model); + let tool_specs = self + .tools + .iter() + .map(|tool| ToolPromptSpec { + name: tool.tool_name().to_string(), + description: tool.description(), + parameters: tool.parameters(), + }) + .collect::>(); + let base_prompt = self.system_prompt_parts.join("\n\n"); + let system_prompt = build_tools_system_prompt(&base_prompt, &tool_specs); + agent.tools = self.tools; + agent.hooks = self.hooks; + agent.default_max_turns = self.max_turns; + agent.memory = self.memory_data; + agent.system_prompt = Some(LlmMessage::system(&system_prompt)); + agent } } -pub trait LlmApiAdaptor: Send + Sync { - fn complete<'a>( - &self, - messages: impl Iterator + Send, - ) -> impl Future> + Send - where - Message: 'a; - fn tool_result_message(&self, tool_name: &str, output: &str) -> Message; - - fn build_assistant_message(&self, text: &str) -> Message; +struct ToolPromptSpec { + name: String, + description: String, + parameters: Value, +} + +fn build_tools_system_prompt(base_prompt: &str, tool_specs: &[ToolPromptSpec]) -> String { + let mut prompt = String::new(); + prompt.push_str(base_prompt); + prompt.push_str("\n\nYou have access to the following tools.\n\n"); + + if tool_specs.is_empty() { + prompt.push_str("(No tools available.)\n"); + } else { + prompt.push_str("## JSON Response Format\n"); + prompt.push_str( + "When you need to use a tool, respond with valid JSON only (no markdown fences):\n", + ); + prompt.push_str("{\n"); + prompt.push_str(" \"reply_text\": \"your message to the user\",\n"); + prompt.push_str(" \"tool_calls\": [\n {\n"); + prompt.push_str(" \"tool_name\": \"tool_name_here\",\n"); + prompt.push_str(" \"args\": { \"argument_name\": \"argument_value\" }\n"); + prompt.push_str(" }\n ]\n"); + prompt.push_str("}\n\n"); + prompt.push_str("If you don't need to use a tool, respond with:\n"); + prompt.push_str("{\n"); + prompt.push_str(" \"reply_text\": \"your message\",\n"); + prompt.push_str(" \"tool_calls\": []\n"); + prompt.push_str("}\n\n"); + prompt.push_str("## Available Tools\n\n"); + + for tool in tool_specs { + prompt.push_str(&format!("- {}\n", tool.name)); + prompt.push_str(&format!(" Description: {}\n", tool.description)); + prompt.push_str(" Arguments JSON Schema:\n"); + let parameters = serde_json::to_string_pretty(&tool.parameters) + .unwrap_or_else(|_| tool.parameters.to_string()); + prompt.push_str(¶meters); + prompt.push_str("\n"); + } + + prompt.push_str( + "return your response immediately without expecting further tool execution. ", + ); + prompt.push_str( + "as you see, tool_calls is an array, several tools calls can be executed in one turn concurrently. ", + ); + prompt.push_str("For example: `\"reply_text\": \"Task complete. \"`."); + prompt.push_str("\n"); + prompt.push_str("IMPORTANT: Always respond with valid JSON only. "); + prompt.push_str( + "Do not wrap the JSON in markdown code fences or add extra text outside the JSON.", + ); + } + + prompt } diff --git a/server-rs/crates/module-editor-agent/src/agent/asset.rs b/server-rs/crates/module-editor-agent/src/agent/asset.rs new file mode 100644 index 000000000..027ac007d --- /dev/null +++ b/server-rs/crates/module-editor-agent/src/agent/asset.rs @@ -0,0 +1,43 @@ +use std::fmt::Display; +use sha2::Sha256; +use serde::{Deserialize, Serialize}; +use hmac::digest::Digest; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ImageId { + pub id: String, +} + +impl ImageId { + /// Keep data keys server-side while giving the model a stable image reference. + pub fn from_data_key(data_key: impl AsRef) -> Self { + let digest = Sha256::digest(data_key.as_ref().as_bytes()); + Self { + id: format!("sha256:{digest:x}"), + } + } +} + +impl Serialize for ImageId { + fn serialize(&self, serializer: S) -> Result { + self.id.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for ImageId { + fn deserialize>(deserializer: D) -> Result { + String::deserialize(deserializer).map(|id| ImageId { id }) + } +} + +impl Display for ImageId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.id) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ImageMetadata { + pub tag: String, + pub data_key: String, +} \ No newline at end of file diff --git a/server-rs/crates/module-editor-agent/src/agent/mod.rs b/server-rs/crates/module-editor-agent/src/agent/mod.rs index 1a5ab526a..26ffab9c4 100644 --- a/server-rs/crates/module-editor-agent/src/agent/mod.rs +++ b/server-rs/crates/module-editor-agent/src/agent/mod.rs @@ -1,9 +1,3 @@ -use tool::{Tool, ToolDyn}; - pub mod agent; -pub mod agent_builder; -pub mod error; -pub mod hook; -pub mod memory; -pub mod run; -pub mod tool; +pub mod tools; +pub mod asset; diff --git a/server-rs/crates/module-editor-agent/src/agent/tools/context.rs b/server-rs/crates/module-editor-agent/src/agent/tools/context.rs new file mode 100644 index 000000000..0220b1634 --- /dev/null +++ b/server-rs/crates/module-editor-agent/src/agent/tools/context.rs @@ -0,0 +1,22 @@ +use std::collections::HashMap; +use serde::{Deserialize, Serialize}; +use crate::agent::asset::ImageId; +use crate::agent::asset::ImageMetadata; + +impl EditorToolContext { + /// Check if an image with the given ID exists in the context. + pub fn contains_image(&self, image_id: &ImageId) -> bool { + self.images.contains_key(image_id) + } + + pub fn image_data_key(&self, image_id: &ImageId) -> Option<&str> { + self.images + .get(image_id) + .map(|metadata| metadata.data_key.as_str()) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct EditorToolContext { + pub images: HashMap, +} \ No newline at end of file diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs b/server-rs/crates/module-editor-agent/src/agent/tools/edit_image.rs similarity index 91% rename from server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs rename to server-rs/crates/module-editor-agent/src/agent/tools/edit_image.rs index 5f91bdea4..a93d4ff6f 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs +++ b/server-rs/crates/module-editor-agent/src/agent/tools/edit_image.rs @@ -1,14 +1,10 @@ -use crate::editor_agent::editor_tools::common::{ - EditorAgentPricedTool, EditorToolContext, editor_agent_image_mud_points, -}; -use crate::editor_agent::utils::ImageId; -use crate::editor_generation_config::EditorGenerationPricingConfig; -use module_editor_agent::agent::tool::{Tool, ToolFailure, ToolFailureKind}; +use crate::agent::tools::context::EditorToolContext; +use crate::agent::asset::ImageId; +use crate::framework::tool::{Tool, ToolFailure, ToolFailureKind}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::error::Error; use std::fmt::Display; - pub struct EditImageTool { pub context: EditorToolContext, } @@ -115,12 +111,6 @@ impl Tool for EditImageTool { } } -impl EditorAgentPricedTool for EditImageTool { - fn pricing(&self, pricing: &EditorGenerationPricingConfig, _args: &EditImageToolArgs) -> u32 { - editor_agent_image_mud_points(pricing, Some("quick-edit"), Some("1K")) - } -} - /// Typed result deserialized from `edit_editor_image_for_owner` response. /// /// Mirrors `EditorImageGenerationResponse` but uses `String` instead of `&'static str` diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_background_music.rs b/server-rs/crates/module-editor-agent/src/agent/tools/generate_background_music.rs similarity index 83% rename from server-rs/crates/api-server/src/editor_agent/editor_tools/generate_background_music.rs rename to server-rs/crates/module-editor-agent/src/agent/tools/generate_background_music.rs index 62357170f..fe8a5d162 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_background_music.rs +++ b/server-rs/crates/module-editor-agent/src/agent/tools/generate_background_music.rs @@ -1,8 +1,4 @@ -use crate::editor_agent::editor_tools::common::EditorAgentPricedTool; -use crate::editor_generation_config::{ - EditorGenerationPricingConfig, EDITOR_BACKGROUND_MUSIC_MODEL_SUNO, -}; -use module_editor_agent::agent::tool::{Tool, ToolFailure}; +use crate::framework::tool::{Tool, ToolFailure}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use shared_contracts::assets::EditorAudioGenerateResponse; @@ -71,17 +67,9 @@ impl Tool for GenerateBackgroundMusicTool { } } -impl EditorAgentPricedTool for GenerateBackgroundMusicTool { - fn pricing( - &self, - pricing: &EditorGenerationPricingConfig, - _args: &GenerateBackgroundMusicToolArgs, - ) -> u32 { - pricing.background_music_model_mud_points(Some(EDITOR_BACKGROUND_MUSIC_MODEL_SUNO)) - } -} - impl GenerateBackgroundMusicTool { + pub const DEFAULT_MODEL: &'static str = "chirp-v5"; + pub fn format_execute_message( &self, args: &GenerateBackgroundMusicToolArgs, diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_character.rs b/server-rs/crates/module-editor-agent/src/agent/tools/generate_character.rs similarity index 76% rename from server-rs/crates/api-server/src/editor_agent/editor_tools/generate_character.rs rename to server-rs/crates/module-editor-agent/src/agent/tools/generate_character.rs index f6a0b8711..9394d762b 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_character.rs +++ b/server-rs/crates/module-editor-agent/src/agent/tools/generate_character.rs @@ -1,13 +1,7 @@ -use crate::editor_agent::editor_tools::common::{ - editor_agent_image_mud_points, EditorAgentPricedTool, EditorToolContext, -}; -use crate::editor_agent::editor_tools::generate_image::{ - EditorImageGenerationResult, GenerateImageError, GenerateImageTool, GenerateImageToolArgs, - GenerateImageToolOutput, -}; -use crate::editor_generation_config::EditorGenerationPricingConfig; -use module_editor_agent::agent::tool::{Tool, ToolFailure}; +use crate::framework::tool::{Tool, ToolFailure}; use serde_json::{json, Value}; +use crate::agent::tools::context::EditorToolContext; +use crate::agent::tools::generate_image::{EditorImageGenerationResult, GenerateImageError, GenerateImageTool, GenerateImageToolArgs, GenerateImageToolOutput}; pub struct GenerateCharacterTool { pub context: EditorToolContext, @@ -59,16 +53,6 @@ impl Tool for GenerateCharacterTool { } } -impl EditorAgentPricedTool for GenerateCharacterTool { - fn pricing( - &self, - pricing: &EditorGenerationPricingConfig, - args: &GenerateImageToolArgs, - ) -> u32 { - editor_agent_image_mud_points(pricing, Some("character"), args.image_size.as_deref()) - } -} - impl GenerateCharacterTool { pub fn format_execute_message( &self, @@ -84,7 +68,7 @@ impl GenerateCharacterTool { .filter(|value| !value.is_empty()) .map(|value| value.trim_start_matches('/').to_string()) .unwrap_or_else(|| result.image_src.clone()); - let image_id = crate::editor_agent::utils::ImageId::from_data_key(data_key); + let image_id = crate::agent::asset::ImageId::from_data_key(data_key); format!( "[tool_call:{tool_name}] args: {args} output: generated result saved as image: {image_id}" ) diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_icon_spritesheet.rs b/server-rs/crates/module-editor-agent/src/agent/tools/generate_icon_spritesheet.rs similarity index 86% rename from server-rs/crates/api-server/src/editor_agent/editor_tools/generate_icon_spritesheet.rs rename to server-rs/crates/module-editor-agent/src/agent/tools/generate_icon_spritesheet.rs index 6b09e3abb..3d9f16968 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_icon_spritesheet.rs +++ b/server-rs/crates/module-editor-agent/src/agent/tools/generate_icon_spritesheet.rs @@ -1,11 +1,6 @@ -use crate::editor_agent::editor_tools::common::{ - editor_agent_image_mud_points, EditorAgentPricedTool, EditorToolContext, -}; -use crate::editor_agent::utils::ImageId; -use crate::editor_generation_config::EditorGenerationPricingConfig; -use crate::http_error::AppError; -use axum::http::StatusCode; -use module_editor_agent::agent::tool::{Tool, ToolFailure, ToolFailureKind}; +use crate::agent::tools::context::EditorToolContext; +use crate::agent::asset::ImageId; +use crate::framework::tool::{Tool, ToolFailure, ToolFailureKind}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::error::Error; @@ -113,16 +108,6 @@ impl Tool for GenerateIconSpritesheetTool { } } -impl EditorAgentPricedTool for GenerateIconSpritesheetTool { - fn pricing( - &self, - pricing: &EditorGenerationPricingConfig, - args: &GenerateIconSpritesheetToolArgs, - ) -> u32 { - editor_agent_image_mud_points(pricing, Some("icon"), args.image_size.as_deref()) - } -} - impl GenerateIconSpritesheetTool { fn validate_args( &self, @@ -174,10 +159,3 @@ impl GenerateIconSpritesheetTool { ) } } - -fn image_not_found(image_id: &ImageId) -> AppError { - AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({ - "provider": "editor-agent", - "message": format!("asset {image_id} not found in context"), - })) -} diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_image.rs b/server-rs/crates/module-editor-agent/src/agent/tools/generate_image.rs similarity index 89% rename from server-rs/crates/api-server/src/editor_agent/editor_tools/generate_image.rs rename to server-rs/crates/module-editor-agent/src/agent/tools/generate_image.rs index 553f3e5bc..1c13df998 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_image.rs +++ b/server-rs/crates/module-editor-agent/src/agent/tools/generate_image.rs @@ -1,9 +1,6 @@ -use crate::editor_agent::editor_tools::common::{ - editor_agent_image_mud_points, EditorAgentPricedTool, EditorToolContext, -}; -use crate::editor_agent::utils::ImageId; -use crate::editor_generation_config::EditorGenerationPricingConfig; -use module_editor_agent::agent::tool::{Tool, ToolFailure, ToolFailureKind}; +use crate::agent::tools::context::EditorToolContext; +use crate::agent::asset::ImageId; +use crate::framework::tool::{Tool, ToolFailure, ToolFailureKind}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::error::Error; @@ -107,16 +104,6 @@ impl Tool for GenerateImageTool { } } -impl EditorAgentPricedTool for GenerateImageTool { - fn pricing( - &self, - pricing: &EditorGenerationPricingConfig, - args: &GenerateImageToolArgs, - ) -> u32 { - editor_agent_image_mud_points(pricing, None, args.image_size.as_deref()) - } -} - #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct EditorImageGenerationResult { @@ -137,7 +124,7 @@ pub struct EditorImageGenerationResult { } impl GenerateImageTool { - pub(crate) fn validate_args( + pub fn validate_args( &self, args: &GenerateImageToolArgs, ) -> Result<(), GenerateImageError> { diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_sound_effect.rs b/server-rs/crates/module-editor-agent/src/agent/tools/generate_sound_effect.rs similarity index 80% rename from server-rs/crates/api-server/src/editor_agent/editor_tools/generate_sound_effect.rs rename to server-rs/crates/module-editor-agent/src/agent/tools/generate_sound_effect.rs index 4931096e8..57f1c0c36 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_sound_effect.rs +++ b/server-rs/crates/module-editor-agent/src/agent/tools/generate_sound_effect.rs @@ -1,8 +1,4 @@ -use crate::editor_agent::editor_tools::common::EditorAgentPricedTool; -use crate::editor_generation_config::{ - EditorGenerationPricingConfig, EDITOR_SOUND_EFFECT_MODEL_VIDU, -}; -use module_editor_agent::agent::tool::{Tool, ToolFailure}; +use crate::framework::tool::{Tool, ToolFailure}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use shared_contracts::assets::EditorAudioGenerateResponse; @@ -80,23 +76,9 @@ impl Tool for GenerateSoundEffectTool { } } -impl EditorAgentPricedTool for GenerateSoundEffectTool { - fn pricing( - &self, - pricing: &EditorGenerationPricingConfig, - args: &GenerateSoundEffectToolArgs, - ) -> u32 { - let model = args - .model - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or(EDITOR_SOUND_EFFECT_MODEL_VIDU); - pricing.sound_effect_model_mud_points(Some(model)) - } -} - impl GenerateSoundEffectTool { + pub const DEFAULT_MODEL: &'static str = "audio1.0"; + pub fn format_execute_message( &self, args: &GenerateSoundEffectToolArgs, diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_ui_design.rs b/server-rs/crates/module-editor-agent/src/agent/tools/generate_ui_design.rs similarity index 75% rename from server-rs/crates/api-server/src/editor_agent/editor_tools/generate_ui_design.rs rename to server-rs/crates/module-editor-agent/src/agent/tools/generate_ui_design.rs index 3cb0eae1a..203e85b3f 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_ui_design.rs +++ b/server-rs/crates/module-editor-agent/src/agent/tools/generate_ui_design.rs @@ -1,13 +1,10 @@ -use crate::editor_agent::editor_tools::common::{ - EditorAgentPricedTool, EditorToolContext, editor_agent_image_mud_points, +use crate::agent::tools::generate_image::{ + EditorImageGenerationResult, GenerateImageError, GenerateImageTool, GenerateImageToolArgs, + GenerateImageToolOutput, }; -use crate::editor_agent::editor_tools::generate_image::{ - EditorImageGenerationResult, GenerateImageError, GenerateImageTool, GenerateImageToolArgs, - GenerateImageToolOutput, -}; -use crate::editor_generation_config::EditorGenerationPricingConfig; -use module_editor_agent::agent::tool::{Tool, ToolFailure}; -use serde_json::{Value, json}; +use crate::framework::tool::{Tool, ToolFailure}; +use serde_json::{json, Value}; +use crate::agent::tools::context::EditorToolContext; pub struct GenerateUiDesignTool { pub context: EditorToolContext, @@ -58,16 +55,6 @@ impl Tool for GenerateUiDesignTool { } } -impl EditorAgentPricedTool for GenerateUiDesignTool { - fn pricing( - &self, - pricing: &EditorGenerationPricingConfig, - args: &GenerateImageToolArgs, - ) -> u32 { - editor_agent_image_mud_points(pricing, Some("ui-design"), args.image_size.as_deref()) - } -} - impl GenerateUiDesignTool { pub fn format_execute_message( &self, @@ -83,7 +70,7 @@ impl GenerateUiDesignTool { .filter(|value| !value.is_empty()) .map(|value| value.trim_start_matches('/').to_string()) .unwrap_or_else(|| result.image_src.clone()); - let image_id = crate::editor_agent::utils::ImageId::from_data_key(data_key); + let image_id = crate::agent::asset::ImageId::from_data_key(data_key); format!( "[tool_call:{tool_name}] args: {args} output: generated result saved as image: {image_id}" ) diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs b/server-rs/crates/module-editor-agent/src/agent/tools/generate_video.rs similarity index 76% rename from server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs rename to server-rs/crates/module-editor-agent/src/agent/tools/generate_video.rs index 7a8cccc6d..882f776bb 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs +++ b/server-rs/crates/module-editor-agent/src/agent/tools/generate_video.rs @@ -1,20 +1,16 @@ -use crate::editor_agent::editor_tools::common::EditorAgentPricedTool; -use crate::editor_agent::utils::ImageId; -use crate::editor_generation_config::EditorGenerationPricingConfig; -use module_editor_agent::agent::tool::{Tool, ToolFailure, ToolFailureKind}; +use crate::agent::asset::ImageId; +use crate::framework::tool::{Tool, ToolFailure, ToolFailureKind}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use shared_contracts::assets::EditorVideoGenerateResponse; use std::error::Error; use std::fmt::Display; +use crate::agent::tools::context::EditorToolContext; pub struct GenerateVideoTool { - pub context: crate::editor_agent::editor_tools::common::EditorToolContext, + pub context: EditorToolContext, } -const DEFAULT_VIDEO_MODEL: &str = "seedance2.0-fast"; -const DEFAULT_VIDEO_RESOLUTION: &str = "720p"; -const DEFAULT_VIDEO_DURATION_SECONDS: u32 = 4; #[derive(Debug, Clone)] pub enum GenerateVideoError { @@ -101,32 +97,11 @@ impl Tool for GenerateVideoTool { } } -impl EditorAgentPricedTool for GenerateVideoTool { - fn pricing( - &self, - pricing: &EditorGenerationPricingConfig, - args: &GenerateVideoToolArgs, - ) -> u32 { - let model = args - .model - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or(DEFAULT_VIDEO_MODEL); - let resolution = args - .resolution - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or(DEFAULT_VIDEO_RESOLUTION); - let duration_seconds = args - .duration_seconds - .unwrap_or(DEFAULT_VIDEO_DURATION_SECONDS); - pricing.video_model_mud_points(Some(model), resolution, duration_seconds) - } -} - impl GenerateVideoTool { + pub const DEFAULT_VIDEO_MODEL: &'static str = "seedance2.0-fast"; + pub const DEFAULT_VIDEO_RESOLUTION: &'static str = "720p"; + pub const DEFAULT_VIDEO_DURATION_SECONDS: u32 = 4; + pub fn format_execute_message( &self, args: &GenerateVideoToolArgs, @@ -140,4 +115,4 @@ impl GenerateVideoTool { video_id ) } -} \ No newline at end of file +} diff --git a/server-rs/crates/api-server/src/editor_agent/editor_tools/mod.rs b/server-rs/crates/module-editor-agent/src/agent/tools/mod.rs similarity index 82% rename from server-rs/crates/api-server/src/editor_agent/editor_tools/mod.rs rename to server-rs/crates/module-editor-agent/src/agent/tools/mod.rs index ec3c580d2..afee4c893 100644 --- a/server-rs/crates/api-server/src/editor_agent/editor_tools/mod.rs +++ b/server-rs/crates/module-editor-agent/src/agent/tools/mod.rs @@ -1,4 +1,3 @@ -pub mod common; pub mod edit_image; pub mod generate_background_music; pub mod generate_character; @@ -7,6 +6,4 @@ pub mod generate_image; pub mod generate_sound_effect; pub mod generate_ui_design; pub mod generate_video; - -#[cfg(test)] -mod pricing_tests; +pub mod context; \ No newline at end of file diff --git a/server-rs/crates/module-editor-agent/src/framework/agent.rs b/server-rs/crates/module-editor-agent/src/framework/agent.rs new file mode 100644 index 000000000..7aac0adf5 --- /dev/null +++ b/server-rs/crates/module-editor-agent/src/framework/agent.rs @@ -0,0 +1,80 @@ +use crate::framework::error::PromptError; +use crate::framework::hook::Hook; +use crate::framework::memory::AgentMemory; +use crate::framework::run::PromptRequest; +use crate::framework::tool::{Tool, ToolDyn}; + +pub struct Agent, Message> { + pub model: M, + pub tools: Vec>, + pub hooks: Vec>, + pub default_max_turns: usize, + pub system_prompt: Option, + pub memory: Option>>, +} + +impl Agent +where + M: LlmApiAdaptor + 'static, + Message: Send + 'static, +{ + pub fn new(model: M) -> Self { + Self { + model, + tools: Vec::new(), + hooks: Vec::new(), + default_max_turns: 10, + system_prompt: None, + memory: None, + } + } + + pub fn tool(mut self, tool: impl Tool + Send + Sync + 'static) -> Self { + self.tools.push(Box::new(tool)); + self + } + + pub fn system_prompt(mut self, msg: Message) -> Self { + self.system_prompt = Some(msg); + self + } + + pub fn memory(mut self, mem: impl AgentMemory + 'static) -> Self { + self.memory = Some(Box::new(mem)); + self + } + + pub fn hook(mut self, hook: impl Hook + 'static) -> Self { + self.hooks.push(Box::new(hook)); + self + } + + pub fn max_turns(mut self, n: usize) -> Self { + self.default_max_turns = n; + self + } + + pub fn tools(&self) -> &[Box] { + &self.tools + } + + + pub fn prompt(&mut self, message: impl Into + Send) -> PromptRequest<'_, M, Message> + where + Message: 'static, + { + PromptRequest::new(self, message.into()) + } +} + +pub trait LlmApiAdaptor: Send + Sync { + fn complete<'a>( + &self, + messages: impl Iterator + Send, + ) -> impl Future> + Send + where + Message: 'a; + fn tool_result_message(&self, tool_name: &str, output: &str) -> Message; + + fn build_assistant_message(&self, text: &str) -> Message; +} diff --git a/server-rs/crates/module-editor-agent/src/agent/agent_builder.rs b/server-rs/crates/module-editor-agent/src/framework/agent_builder.rs similarity index 75% rename from server-rs/crates/module-editor-agent/src/agent/agent_builder.rs rename to server-rs/crates/module-editor-agent/src/framework/agent_builder.rs index f090c7057..f2f90d8cc 100644 --- a/server-rs/crates/module-editor-agent/src/agent/agent_builder.rs +++ b/server-rs/crates/module-editor-agent/src/framework/agent_builder.rs @@ -1,7 +1,7 @@ -use crate::agent::agent::{Agent, LlmApiAdaptor}; -use crate::agent::hook::Hook; -use crate::agent::memory::AgentMemory; -use crate::agent::tool::Tool; +use crate::framework::agent::{Agent, LlmApiAdaptor}; +use crate::framework::hook::Hook; +use crate::framework::memory::AgentMemory; +use crate::framework::tool::Tool; pub trait AgentBuilder> { type Client; diff --git a/server-rs/crates/module-editor-agent/src/agent/error.rs b/server-rs/crates/module-editor-agent/src/framework/error.rs similarity index 100% rename from server-rs/crates/module-editor-agent/src/agent/error.rs rename to server-rs/crates/module-editor-agent/src/framework/error.rs diff --git a/server-rs/crates/module-editor-agent/src/agent/hook.rs b/server-rs/crates/module-editor-agent/src/framework/hook.rs similarity index 80% rename from server-rs/crates/module-editor-agent/src/agent/hook.rs rename to server-rs/crates/module-editor-agent/src/framework/hook.rs index 5193c3d18..c5fd255a3 100644 --- a/server-rs/crates/module-editor-agent/src/agent/hook.rs +++ b/server-rs/crates/module-editor-agent/src/framework/hook.rs @@ -1,5 +1,5 @@ -use crate::agent::run::{TextFlow, ToolCallFlow}; -use crate::agent::tool::ToolCall; +use crate::framework::run::{TextFlow, ToolCallFlow}; +use crate::framework::tool::ToolCall; pub trait Hook: Send + Sync { fn on_text_reply(&self, _text: &str) -> TextFlow { diff --git a/server-rs/crates/module-editor-agent/src/agent/memory.rs b/server-rs/crates/module-editor-agent/src/framework/memory.rs similarity index 100% rename from server-rs/crates/module-editor-agent/src/agent/memory.rs rename to server-rs/crates/module-editor-agent/src/framework/memory.rs diff --git a/server-rs/crates/module-editor-agent/src/framework/mod.rs b/server-rs/crates/module-editor-agent/src/framework/mod.rs new file mode 100644 index 000000000..0f84bfb15 --- /dev/null +++ b/server-rs/crates/module-editor-agent/src/framework/mod.rs @@ -0,0 +1,7 @@ +pub mod agent; +pub mod agent_builder; +pub mod error; +pub mod hook; +pub mod memory; +pub mod run; +pub mod tool; diff --git a/server-rs/crates/module-editor-agent/src/agent/run.rs b/server-rs/crates/module-editor-agent/src/framework/run.rs similarity index 96% rename from server-rs/crates/module-editor-agent/src/agent/run.rs rename to server-rs/crates/module-editor-agent/src/framework/run.rs index 04c9eba8e..c3e7355db 100644 --- a/server-rs/crates/module-editor-agent/src/agent/run.rs +++ b/server-rs/crates/module-editor-agent/src/framework/run.rs @@ -1,10 +1,10 @@ -use crate::agent::agent::Agent; -use crate::agent::agent::LlmApiAdaptor; -use crate::agent::error::PromptError; -use crate::agent::hook::Hook; -use crate::agent::memory::VecMemory; -use crate::agent::run::PromptOutput::{Text, Tool}; -use crate::agent::tool::{ToolCall, ToolDyn, ToolExecutionResult, ToolFailure, ToolOutcome}; +use crate::framework::agent::Agent; +use crate::framework::agent::LlmApiAdaptor; +use crate::framework::error::PromptError; +use crate::framework::hook::Hook; +use crate::framework::memory::VecMemory; +use crate::framework::run::PromptOutput::{Text, Tool}; +use crate::framework::tool::{ToolCall, ToolDyn, ToolExecutionResult, ToolFailure, ToolOutcome}; use serde::Deserialize; use serde_json::Value; use std::pin::Pin; diff --git a/server-rs/crates/module-editor-agent/src/agent/tool.rs b/server-rs/crates/module-editor-agent/src/framework/tool.rs similarity index 100% rename from server-rs/crates/module-editor-agent/src/agent/tool.rs rename to server-rs/crates/module-editor-agent/src/framework/tool.rs diff --git a/server-rs/crates/module-editor-agent/src/lib.rs b/server-rs/crates/module-editor-agent/src/lib.rs index 4e0e32619..3e7a37941 100644 --- a/server-rs/crates/module-editor-agent/src/lib.rs +++ b/server-rs/crates/module-editor-agent/src/lib.rs @@ -3,6 +3,7 @@ mod application; mod commands; mod domain; mod errors; +pub mod framework; pub use application::*; pub use commands::*;