reimpl the agent

server-rs/crates/api-server/src/editor_agent/agent.rs

@@ -0,0 +1,211 @@
+use serde_json::Value;
+use module_editor_agent::agent::agent::{Agent, LlmApiAdaptor};
+use module_editor_agent::agent::agent_builder::AgentBuilder;
+use module_editor_agent::agent::error::PromptError;
+use module_editor_agent::agent::hook::Hook;
+use module_editor_agent::agent::memory::AgentMemory;
+use module_editor_agent::agent::run::ToolCallFlow;
+use module_editor_agent::agent::tool::{Tool, ToolCall, ToolDyn};
+use platform_llm::{LlmClient, LlmMessage};
+
+struct LlmCompletionModel {
+  client: LlmClient,
+}
+
+impl LlmApiAdaptor<LlmMessage> for LlmCompletionModel {
+  async fn complete(&self, messages: &[LlmMessage]) -> Result<String, PromptError> {
+    use platform_llm::LlmTextRequest;
+    let request = LlmTextRequest::new(messages.to_vec()).with_request_timeout_ms(30_000);
+    let response = self
+      .client
+      .request_text(request)
+      .await
+      .map_err(|e| PromptError::CompletionError(e.to_string()))?;
+    Ok(response.content)
+  }
+
+  fn tool_result_message(&self, tool_name: &str, output: &str) -> LlmMessage {
+    LlmMessage::system(format!("Tool '{tool_name}' returned: {output}"))
+  }
+
+  fn build_assistant_message(&self, text: &str) -> LlmMessage {
+    LlmMessage::assistant(text)
+  }
+}
+
+pub struct LlmChatAgentBuilder {
+  client: Option<LlmClient>,
+  system_prompt_parts: Vec<String>,
+  tools: Vec<Box<dyn ToolDyn>>,
+  hooks: Vec<Box<dyn Hook>>,
+  max_turns: usize,
+  memory_data: Option<Box<dyn AgentMemory<LlmMessage>>>,
+  context: Option<serde_json::Value>,
+}
+
+impl AgentBuilder<LlmMessage, LlmCompletionModel> for LlmChatAgentBuilder {
+  type Client = LlmClient;
+
+  fn new() -> Self {
+    Self {
+      client: None,
+      system_prompt_parts: Vec::new(),
+      tools: Vec::new(),
+      hooks: Vec::new(),
+      max_turns: 10,
+      memory_data: None,
+      context: None,
+    }
+  }
+
+  fn with_client(mut self, client: LlmClient) -> Self {
+    self.client = Some(client);
+    self
+  }
+
+  fn system_prompt(mut self, system_prompt: impl Into<String>) -> Self {
+    self.system_prompt_parts.push(system_prompt.into());
+    self
+  }
+
+  fn tool(mut self, tool: impl Tool + Send + Sync + 'static) -> Self {
+    self.tools.push(Box::new(tool));
+    self
+  }
+
+  fn add_hook(mut self, hook: impl Hook + 'static) -> Self {
+    self.hooks.push(Box::new(hook));
+    self
+  }
+
+  fn max_turns(mut self, n: usize) -> Self {
+    self.max_turns = n;
+    self
+  }
+
+  fn memory(mut self, memory: impl AgentMemory<LlmMessage> + 'static) -> Self {
+    self.memory_data = Some(Box::new(memory));
+    self
+  }
+
+  fn context(mut self, context: Value) -> Self {
+    self.context = Some(context);
+    self
+  }
+
+  fn build(self) -> Agent<LlmCompletionModel, LlmMessage> {
+    let model = LlmCompletionModel {
+      client: self.client.expect("call .with_client() first"),
+    };
+    let mut agent = Agent::new(model);
+    let tool_specs = self
+      .tools
+      .iter()
+      .map(|tool| ToolPromptSpec {
+        name: tool.tool_name().to_string(),
+        description: tool.description(),
+        parameters: tool.parameters(),
+      })
+      .collect::<Vec<_>>();
+    let base_prompt = self.system_prompt_parts.join("\n\n");
+    let system_prompt = build_tools_system_prompt(&base_prompt, &tool_specs);
+    agent.tools = self.tools;
+    agent.hooks = self.hooks;
+    agent.default_max_turns = self.max_turns;
+    agent.memory = self.memory_data;
+    agent.context = self.context;
+    agent.system_prompt = Some(LlmMessage::system(&system_prompt));
+    agent
+  }
+}
+
+struct ToolPromptSpec {
+  name: String,
+  description: String,
+  parameters: Value,
+}
+
+fn build_tools_system_prompt(base_prompt: &str, tool_specs: &[ToolPromptSpec]) -> String {
+  let mut prompt = String::new();
+  prompt.push_str(base_prompt);
+  prompt.push_str("\n\nYou have access to the following tools.\n\n");
+
+  if tool_specs.is_empty() {
+    prompt.push_str("(No tools available.)\n");
+  } else {
+    prompt.push_str("## JSON Response Format\n");
+    prompt.push_str(
+      "When you need to use a tool, respond with valid JSON only (no markdown fences):\n",
+    );
+    prompt.push_str("{\n");
+    prompt.push_str("  \"reply_text\": \"your message to the user\",\n");
+    prompt.push_str("  \"tool_calls\": [\n    {\n");
+    prompt.push_str("      \"tool_name\": \"tool_name_here\",\n");
+    prompt.push_str("      \"args\": { \"argument_name\": \"argument_value\" }\n");
+    prompt.push_str("    }\n  ]\n");
+    prompt.push_str("}\n\n");
+    prompt.push_str("If you don't need to use a tool, respond with:\n");
+    prompt.push_str("{\n");
+    prompt.push_str("  \"reply_text\": \"your message\",\n");
+    prompt.push_str("  \"tool_calls\": []\n");
+    prompt.push_str("}\n\n");
+    prompt.push_str("## Available Tools\n\n");
+
+    for tool in tool_specs {
+      prompt.push_str(&format!("- {}\n", tool.name));
+      prompt.push_str(&format!("  Description: {}\n", tool.description));
+      prompt.push_str("  Arguments JSON Schema:\n");
+      let parameters = serde_json::to_string_pretty(&tool.parameters)
+        .unwrap_or_else(|_| tool.parameters.to_string());
+      prompt.push_str(&parameters);
+      prompt.push_str("\n");
+    }
+
+    prompt.push_str(
+      "Use `<end/>` in `reply_text` when you want to stop the conversation turn and ",
+    );
+    prompt.push_str(
+      "return your response immediately without expecting further tool execution. ",
+    );
+    prompt.push_str("For example: `\"reply_text\": \"Task complete. <end/>\"`.");
+    prompt.push_str("\n");
+    prompt.push_str("IMPORTANT: Always respond with valid JSON only. ");
+    prompt.push_str(
+      "Do not wrap the JSON in markdown code fences or add extra text outside the JSON.",
+    );
+  }
+
+  prompt
+}
+
+
+//
+// async fn run_chat_agent(config: LlmConfig) -> Result<(), LlmError> {
+//   let client = LlmClient::new(config)?;
+//   let agent = LlmChatAgentBuilder::new()
+//     .with_client(client)
+//     .system_prompt("You are a helpful assistant with an echo tool.")
+//     .build();
+//
+//   let names = agent
+//     .tools()
+//     .iter()
+//     .map(|t| t.tool_name().to_string())
+//     .collect();
+//   let tool_hook = ToolValidationHook::new(names.clone());
+//
+//   let output = agent
+//     .prompt(LlmMessage::user(
+//       "Use the echo tool to echo 'Hello from the JSON harness!', then tell me what it said.",
+//     ))
+//     .max_turns(5)
+//     .add_hook(tool_hook)
+//     .await
+//     .expect("agent should succeed");
+//
+//   println!("--- Final Agent Response ---");
+//   println!("{}", output.text);
+//   println!("-----------------------------");
+//
+//   Ok(())
+// }

server-rs/crates/api-server/src/editor_agent/api.rs

@@ -0,0 +1,207 @@
+use std::collections::HashMap;
+
+use axum::extract::{Path, State};
+use axum::{Extension, Json};
+use module_editor_agent::agent::agent_builder::AgentBuilder;
+use module_editor_agent::agent::error::PromptError;
+use module_editor_agent::agent::memory::VecMemory;
+use module_editor_agent::agent::run::PromptOutput;
+use module_editor_agent::derive_conversation_title;
+use platform_llm::LlmMessage;
+use serde_json::{Value, json};
+use shared_contracts::editor_agent::{
+    EditorAgentConversationMessagesDocument, EditorAgentMessage, EditorAgentMessageRole,
+    EditorAgentToolCall, EditorAgentToolCallStatus, StreamEditorAgentMessageRequest,
+};
+use spacetime_client::EditorAgentConversationTouchRecordInput;
+
+use crate::auth::AuthenticatedAccessToken;
+use crate::editor_agent::agent::LlmChatAgentBuilder;
+use crate::editor_agent::editor_tools::edit_image::EditImageTool;
+use crate::editor_agent::utils::{EditorAgentMessageResponse, ImageId, ImageMetadata, now_rfc3339, require_editor_agent_sidebar_enabled, write_messages_document, normalize_editor_agent_attachments, read_messages_document};
+use crate::editor_project::current_utc_micros;
+use crate::http_error::AppError;
+use crate::request_context::RequestContext;
+use crate::state::AppState;
+
+pub async fn editor_agent_message(
+    State(state): State<AppState>,
+    Path(conversation_id): Path<String>,
+    Extension(_request_context): Extension<RequestContext>,
+    Extension(authenticated): Extension<AuthenticatedAccessToken>,
+    Json(payload): Json<StreamEditorAgentMessageRequest>,
+) -> Result<Json<EditorAgentMessageResponse>, AppError> {
+    let owner_user_id = authenticated.claims().user_id().to_string();
+    require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
+    // Load conversation & attachments
+    let conversation = state
+        .spacetime_client()
+        .get_editor_agent_conversation(conversation_id.clone(), owner_user_id.clone())
+        .await
+        .map_err(|e| {
+            AppError::from_status(axum::http::StatusCode::NOT_FOUND)
+                .with_details(json!({ "message": format!("conversation not found: {e}") }))
+        })?;
+
+    let attachments = normalize_editor_agent_attachments(
+        &state,
+        &conversation,
+        payload.attachments.as_slice(),
+    )
+    .await?;
+
+    let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
+        conversation.conversation_id.as_str(),
+    );
+    let _conversation_lock_guard = conversation_lock.lock_owned().await;
+
+    let mut document: EditorAgentConversationMessagesDocument =
+        read_messages_document(&state, &conversation).await?;
+
+    // Build conversation history as LlmMessage vec
+    let previous_messages: Vec<LlmMessage> = document
+        .messages
+        .iter()
+        .map(|msg| match msg.role {
+            EditorAgentMessageRole::User => LlmMessage::user(&msg.text),
+            EditorAgentMessageRole::Assistant => LlmMessage::assistant(&msg.text),
+            EditorAgentMessageRole::System => LlmMessage::system(&msg.text),
+        })
+        .collect();
+
+    // Save user message to document
+    let was_empty = document.messages.is_empty();
+    let user_now = now_rfc3339();
+    let user_message = EditorAgentMessage {
+        id: document.messages.len(),
+        role: EditorAgentMessageRole::User,
+        text: payload.text.trim().to_string(),
+        attachments,
+        tool_call: None,
+        created_at: user_now,
+    };
+    document.messages.push(user_message.clone());
+    write_messages_document(&state, &conversation, &document).await?;
+
+    // Touch conversation (set title if first message)
+    if was_empty {
+        let title = derive_conversation_title(user_message.text.as_str());
+        let _ = state
+            .spacetime_client()
+            .touch_editor_agent_conversation(EditorAgentConversationTouchRecordInput {
+                conversation_id: conversation.conversation_id.clone(),
+                owner_user_id: conversation.owner_user_id.clone(),
+                title: Some(title),
+                updated_at_micros: current_utc_micros(),
+            })
+            .await;
+    }
+
+    // Build tool context from document
+    let tool_context_value = build_tool_context(&document);
+
+    // Build and run agent
+    let llm_client = state.llm_client().ok_or_else(|| {
+        AppError::from_status(axum::http::StatusCode::SERVICE_UNAVAILABLE)
+            .with_details(json!({ "message": "LLM client not configured" }))
+    })?;
+    let llm_client = llm_client.clone();
+
+    let memory = VecMemory::new(previous_messages);
+
+    let agent = LlmChatAgentBuilder::new()
+        .with_client(llm_client)
+        .tool(EditImageTool {})
+        .max_turns(3)
+        .memory(memory)
+        .context(tool_context_value)
+        .build();
+
+    let agent_result = agent.prompt(LlmMessage::user("")).await;
+
+    let assistant_now = now_rfc3339();
+
+    match build_delta_messages(agent_result, &assistant_now, document.messages.len()) {
+        Err(err) => Ok(Json(EditorAgentMessageResponse {
+            delta_messages: vec![],
+            error_message: Some(err.to_string()),
+        })),
+        Ok(delta_messages) => {
+            for msg in &delta_messages {
+                document.messages.push(msg.clone());
+            }
+            write_messages_document(&state, &conversation, &document).await?;
+
+            Ok(Json(EditorAgentMessageResponse {
+                delta_messages,
+                error_message: None,
+            }))
+        }
+    }
+}
+
+fn build_delta_messages(
+    result: Result<Vec<PromptOutput>, PromptError>,
+    created_at: &str,
+    messages_offset: usize,
+) -> Result<Vec<EditorAgentMessage>, PromptError> {
+    let outputs = result?;
+    let mut messages = Vec::with_capacity(outputs.len());
+
+    for (i, out) in outputs.into_iter().enumerate() {
+        let absolute_idx = messages_offset + i;
+        match out {
+            PromptOutput::Text(text) => {
+                messages.push(EditorAgentMessage {
+                    id: absolute_idx,
+                    role: EditorAgentMessageRole::Assistant,
+                    text,
+                    attachments: Vec::new(),
+                    tool_call: None,
+                    created_at: created_at.to_string(),
+                });
+            }
+            PromptOutput::Tool(tco) => {
+                let summary = tco.tool_call.args.to_string();
+                messages.push(EditorAgentMessage {
+                    id: absolute_idx,
+                    role: EditorAgentMessageRole::System,
+                    text: tco.message,
+                    attachments: Vec::new(),
+                    tool_call: Some(EditorAgentToolCall {
+                        tool_name: tco.tool_call.name,
+                        summary,
+                        status: EditorAgentToolCallStatus::PendingConfirmation,
+                        args: tco.tool_call.args,
+                        images: Vec::new(),
+                        error: None,
+                    }),
+                    created_at: created_at.to_string(),
+                });
+            }
+        }
+    }
+
+    Ok(messages)
+}
+
+fn build_tool_context(document: &EditorAgentConversationMessagesDocument) -> Value {
+    let mut images: HashMap<ImageId, ImageMetadata> = HashMap::new();
+
+    for msg in document.messages.iter().rev() {
+        if let Some(tc) = &msg.tool_call {
+            if !tc.images.is_empty() {
+                for img in &tc.images {
+                    let image_id = ImageId {
+                        id: img.object_key.clone().unwrap_or_default(),
+                    };
+                    let metadata = ImageMetadata { tag: String::new() };
+                    images.insert(image_id, metadata);
+                }
+                break;
+            }
+        }
+    }
+
+    json!({ "images": images })
+}

server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs

@@ -0,0 +1,16 @@
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use crate::editor_agent::utils::{ImageId, ImageMetadata};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct EditorToolContext {
+    pub images: HashMap<ImageId, ImageMetadata>,
+}
+
+impl EditorToolContext {
+    /// Check if an image with the given ID exists in the context.
+    pub fn contains_image(&self, image_id: &ImageId) -> bool {
+        self.images.contains_key(image_id)
+    }
+}
+

server-rs/crates/api-server/src/editor_agent/editor_tools/edit_image.rs

@@ -0,0 +1,245 @@
+use crate::editor_agent::editor_tools::common::EditorToolContext;
+use crate::editor_agent::utils::ImageId;
+use crate::editor_project::{
+    EditorGenerationCaller, EditorImageEditRequest, edit_editor_image_for_owner,
+};
+use crate::http_error::AppError;
+use crate::request_context::RequestContext;
+use crate::state::AppState;
+use axum::http::StatusCode;
+use module_editor_agent::agent::tool::{Tool, ToolFailure, ToolFailureKind};
+use serde::{Deserialize, Serialize};
+use serde_json::{Value, json};
+use shared_contracts::api::ApiSuccessEnvelope;
+use shared_contracts::assets::EditorCanvasGenerationCompletionPayload;
+use std::error::Error;
+use std::fmt::Display;
+
+pub struct EditImageTool {}
+
+#[derive(Debug, Clone)]
+pub enum EditImageError {
+    ObjectImageNotProvided,
+    PromptNotProvided,
+    AssetNotFound(ImageId),
+    ContextParseError(String),
+}
+
+impl Display for EditImageError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            EditImageError::ObjectImageNotProvided => {
+                write!(f, "object image not provided")
+            }
+            EditImageError::PromptNotProvided => write!(f, "prompt not provided"),
+            EditImageError::AssetNotFound(image_id) => {
+                write!(f, "asset {image_id} not found in context")
+            }
+            EditImageError::ContextParseError(msg) => {
+                write!(f, "failed to parse tool context: {msg}")
+            }
+        }
+    }
+}
+
+impl Error for EditImageError {}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct EditImageToolArgs {
+    pub object_image_id: ImageId,
+    #[serde(default)]
+    pub reference_image_ids: Vec<ImageId>,
+    pub prompt: String,
+    // #[serde(default)]
+    // pub tag: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct EditImageToolOutput {
+    pub message: String,
+}
+
+impl Tool for EditImageTool {
+    const NAME: &'static str = "edit-image";
+    type Error = EditImageError;
+    type Args = EditImageToolArgs;
+    type Output = EditImageToolOutput;
+
+    fn description(&self) -> String {
+        "根据用户提供的文本描述，对选定的图片进行编辑修改，例如调整颜色、替换元素、修改风格等。"
+            .to_string()
+    }
+
+    fn parameters(&self) -> Value {
+        json!({
+            "type": "object",
+            "properties": {
+                "object_image_id": {
+                    "type": "string",
+                    "description": "要修改的目标图片。"
+                },
+                "reference_image_ids": {
+                    "type": "array",
+                    "items": { "type": "string" },
+                    "description": "修改参考图 ID 列表（可选的，用于提供风格或元素参考）。"
+                },
+                "prompt": {
+                    "type": "string",
+                    "description": "编辑提示词，描述希望如何修改图片。例如「把背景换成红色」、「把人物改成坐着」。"
+                },
+                // "tag": {
+                //     "type": "string",
+                //     "description": "为新生成的图片添加标签，用于后续在上下文中引用。"
+                // }
+            },
+            "required": ["object_image_id", "prompt"],
+            "additionalProperties": false
+        })
+    }
+
+    fn call(
+        &self,
+        args: Self::Args,
+        context: Value,
+    ) -> impl Future<Output = Result<Self::Output, Self::Error>> + Send {
+        async move {
+            Self::validate_context_images(&args, &context)?;
+            if let Some(error) = Self::validate_args(&args) {
+                return Err(error);
+            }
+            Ok(EditImageToolOutput {
+                message: "this tool call is pending user confirmation. if all is pending, just end this turn".to_string(),
+            })
+        }
+    }
+
+    fn classify_error(&self, error: &Self::Error) -> ToolFailure {
+        match error {
+            EditImageError::ContextParseError(_) => {
+                ToolFailure::new(ToolFailureKind::Internal, error.to_string())
+            }
+            EditImageError::AssetNotFound(_) => {
+                ToolFailure::new(ToolFailureKind::NotFound, error.to_string())
+            }
+            EditImageError::ObjectImageNotProvided | EditImageError::PromptNotProvided => {
+                ToolFailure::invalid_args(error.to_string())
+            }
+        }
+    }
+}
+
+/// Typed result deserialized from `edit_editor_image_for_owner` response.
+///
+/// Mirrors `EditorImageGenerationResponse` but uses `String` instead of `&'static str`
+/// and `Value` for nested payload types so that `#[derive(Deserialize)]` works.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct EditorImageEditResult {
+    pub image_src: String,
+    pub object_key: Option<String>,
+    pub asset_object_id: Option<String>,
+    pub width: u32,
+    pub height: u32,
+    pub source_type: String,
+    pub prompt: String,
+    pub actual_prompt: Option<String>,
+    pub model: String,
+    pub provider: String,
+    pub task_id: String,
+    pub resource: Option<Value>,
+    pub asset: Option<Value>,
+    pub project: Option<Value>,
+}
+
+impl EditImageTool {
+    /// Validate the semantic correctness of the arguments.
+    fn validate_args(args: &EditImageToolArgs) -> Option<EditImageError> {
+        if args.prompt.trim().is_empty() {
+            return Some(EditImageError::PromptNotProvided);
+        }
+        None
+    }
+
+    /// Validate that all referenced images exist in the context.
+    fn validate_context_images(
+        args: &EditImageToolArgs,
+        context: &Value,
+    ) -> Result<(), EditImageError> {
+        let tool_context: EditorToolContext = serde_json::from_value(context.clone())
+            .map_err(|e| EditImageError::ContextParseError(e.to_string()))?;
+
+        if !tool_context.contains_image(&args.object_image_id) {
+            return Err(EditImageError::AssetNotFound(args.object_image_id.clone()));
+        }
+
+        for ref_id in &args.reference_image_ids {
+            if !tool_context.contains_image(ref_id) {
+                return Err(EditImageError::AssetNotFound(ref_id.clone()));
+            }
+        }
+
+        Ok(())
+    }
+
+    pub async fn execute(
+        &self,
+        state: &AppState,
+        request_context: &RequestContext,
+        caller: EditorGenerationCaller,
+        args: EditImageToolArgs,
+        model: Option<String>,
+        project_id: Option<String>,
+        generation_inputs: Option<Value>,
+        asset_label: Option<String>,
+        source_resource_id: Option<String>,
+        canvas_completion: Option<EditorCanvasGenerationCompletionPayload>,
+    ) -> Result<EditorImageEditResult, AppError> {
+        let source_image_src = args.object_image_id.id;
+        let reference_image_srcs: Vec<String> = args
+            .reference_image_ids
+            .into_iter()
+            .map(|id| id.id)
+            .collect();
+
+        let result = edit_editor_image_for_owner(
+            state,
+            request_context,
+            caller,
+            EditorImageEditRequest {
+                prompt: args.prompt,
+                source_image_src,
+                size: None,
+                model,
+                reference_image_srcs: Some(reference_image_srcs),
+                project_id,
+                asset_kind: Some("editor_agent_edit_image".to_string()),
+                generation_inputs,
+                asset_folder_id: Some("project".to_string()),
+                asset_label,
+                source_resource_id,
+                target_layer_id: None,
+                canvas_completion,
+            },
+        )
+        .await?
+        .0;
+
+        let data = if request_context.wants_envelope() {
+            serde_json::from_value::<ApiSuccessEnvelope<Value>>(result)
+                .map_err(|e| {
+                    AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({
+                        "message": format!("failed to parse success envelope: {e}"),
+                    }))
+                })?
+                .data
+        } else {
+            result
+        };
+
+        serde_json::from_value::<EditorImageEditResult>(data).map_err(|e| {
+            AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({
+                "message": format!("failed to deserialize edit image result: {e}"),
+            }))
+        })
+    }
+}

server-rs/crates/api-server/src/editor_agent/editor_tools/mod.rs

@@ -0,0 +1,2 @@
+pub mod common;
+pub mod edit_image;

server-rs/crates/api-server/src/editor_agent/mod.rs

@@ -0,0 +1,4 @@
+mod editor_tools;
+mod utils;
+pub mod api;
+mod agent;