911 lines
37 KiB
Rust
911 lines
37 KiB
Rust
use axum::extract::{Path, State};
|
|
use axum::{Extension, Json};
|
|
use module_editor_agent::framework::agent_builder::AgentBuilder;
|
|
use module_editor_agent::framework::error::PromptError;
|
|
use module_editor_agent::framework::memory::VecMemory;
|
|
use module_editor_agent::framework::run::PromptOutput;
|
|
use module_editor_agent::framework::tool::Tool;
|
|
use module_editor_agent::{
|
|
EDITOR_AGENT_CONVERSATION_ID_PREFIX, EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE,
|
|
derive_conversation_title, editor_agent_messages_object_key,
|
|
};
|
|
use platform_llm::LlmMessage;
|
|
use serde::Serialize;
|
|
use serde_json::{Value, json};
|
|
use sha2::{Digest, Sha256};
|
|
use shared_contracts::assets::{
|
|
EditorBackgroundMusicGenerateRequest, EditorSoundEffectGenerateRequest,
|
|
EditorVideoGenerateRequest,
|
|
};
|
|
use shared_contracts::editor_agent::{
|
|
CreateEditorAgentConversationRequest, EditorAgentConversationListResponse,
|
|
EditorAgentConversationMessagesDocument, EditorAgentConversationResponse,
|
|
EditorAgentConversationSummary, EditorAgentMessage, EditorAgentMessageRequest,
|
|
EditorAgentMessageResponse, EditorAgentMessageRole, EditorAgentToolCall,
|
|
EditorAgentToolCallStatus,
|
|
};
|
|
use spacetime_client::{
|
|
EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput,
|
|
EditorAgentConversationRecord, EditorAgentConversationTouchRecordInput,
|
|
EditorProjectGetRecordInput,
|
|
};
|
|
|
|
use crate::api_response::json_success_body;
|
|
use crate::auth::AuthenticatedAccessToken;
|
|
use crate::editor_agent::pricing::EditorAgentPricedTool;
|
|
use crate::editor_agent::utils::{
|
|
IntoImageId, build_editor_agent_canvas_completion, conversation_detail_from_record,
|
|
conversation_summary_from_record, editor_agent_bad_request, empty_messages_document,
|
|
ensure_editor_project_access, normalize_editor_agent_attachments, now_rfc3339,
|
|
read_messages_document, require_editor_agent_sidebar_enabled, write_messages_document,
|
|
};
|
|
use crate::editor_agent::{context, display_args, reconcile};
|
|
use crate::editor_generation_config::EditorGenerationPricingConfig;
|
|
use crate::editor_generation_queue::{
|
|
EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND,
|
|
EDITOR_IMAGE_EDIT_JOB_KIND, EDITOR_IMAGE_GENERATION_JOB_KIND,
|
|
EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND, EDITOR_VIDEO_GENERATION_JOB_KIND,
|
|
enqueue_editor_generation_job_with_identity,
|
|
};
|
|
use crate::editor_project::{
|
|
EditorIconSpritesheetGenerationRequest, EditorImageEditRequest, EditorImageGenerationRequest,
|
|
};
|
|
use crate::editor_project::{current_utc_micros, map_editor_project_error};
|
|
use crate::http_error::AppError;
|
|
use crate::request_context::RequestContext;
|
|
use crate::state::AppState;
|
|
use module_editor_agent::agent::agent::LlmChatAgentBuilder;
|
|
use module_editor_agent::agent::tools::context::EditorToolContext;
|
|
use module_editor_agent::agent::tools::edit_image::{EditImageTool, EditImageToolArgs};
|
|
use module_editor_agent::agent::tools::generate_background_music::{
|
|
GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs,
|
|
};
|
|
use module_editor_agent::agent::tools::generate_character::GenerateCharacterTool;
|
|
use module_editor_agent::agent::tools::generate_icon_spritesheet::{
|
|
GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs,
|
|
};
|
|
use module_editor_agent::agent::tools::generate_image::{GenerateImageTool, GenerateImageToolArgs};
|
|
use module_editor_agent::agent::tools::generate_sound_effect::{
|
|
GenerateSoundEffectTool, GenerateSoundEffectToolArgs,
|
|
};
|
|
use module_editor_agent::agent::tools::generate_ui_design::GenerateUiDesignTool;
|
|
use module_editor_agent::agent::tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs};
|
|
use shared_kernel::{build_prefixed_uuid_id, normalize_optional_string};
|
|
|
|
pub async fn editor_agent_message(
|
|
State(state): State<AppState>,
|
|
Path(conversation_id): Path<String>,
|
|
Extension(_request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
Json(payload): Json<EditorAgentMessageRequest>,
|
|
) -> Result<Json<EditorAgentMessageResponse>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
// Load conversation & attachments
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id.clone(), owner_user_id.clone())
|
|
.await
|
|
.map_err(|e| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": format!("conversation not found: {e}") }))
|
|
})?;
|
|
|
|
let attachments =
|
|
normalize_editor_agent_attachments(&state, &conversation, payload.attachments.as_slice())
|
|
.await?;
|
|
|
|
let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
|
|
conversation.conversation_id.as_str(),
|
|
);
|
|
let _conversation_lock_guard = conversation_lock.lock_owned().await;
|
|
|
|
let mut document: EditorAgentConversationMessagesDocument =
|
|
read_messages_document(&state, &conversation).await?;
|
|
// Determine initialization before attachment bookkeeping adds a system message.
|
|
let was_empty = document.messages.is_empty();
|
|
let now = now_rfc3339();
|
|
if !attachments.is_empty() {
|
|
let mut attachment_info = String::new();
|
|
attachment_info.push_str("user has just uploaded attachments of the order: ");
|
|
for a in &attachments {
|
|
attachment_info.push_str(&format!("{} ,", a.clone().into_image_id()))
|
|
}
|
|
document.messages.push(EditorAgentMessage {
|
|
id: 0,
|
|
role: EditorAgentMessageRole::System,
|
|
text: attachment_info,
|
|
attachments: Vec::new(),
|
|
tool_call: None,
|
|
created_at: now.clone(),
|
|
});
|
|
}
|
|
|
|
// Build conversation history as LlmMessage vec
|
|
let previous_messages: Vec<LlmMessage> = document
|
|
.messages
|
|
.iter()
|
|
.map(|msg| match msg.role {
|
|
EditorAgentMessageRole::User => LlmMessage::user(&msg.text),
|
|
EditorAgentMessageRole::Assistant => LlmMessage::assistant(&msg.text),
|
|
EditorAgentMessageRole::System => LlmMessage::system(&msg.text),
|
|
})
|
|
.collect();
|
|
|
|
// Save user message to document.
|
|
let user_message = EditorAgentMessage {
|
|
id: document.messages.len(),
|
|
role: EditorAgentMessageRole::User,
|
|
text: payload.text.trim().to_string(),
|
|
attachments,
|
|
tool_call: None,
|
|
created_at: now,
|
|
};
|
|
document.messages.push(user_message.clone());
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
|
|
// Persist and return the authoritative summary for every turn. Initialization sets the
|
|
// title from the first user prompt; a metadata write failure must fail the request.
|
|
let updated_conversation = state
|
|
.spacetime_client()
|
|
.touch_editor_agent_conversation(EditorAgentConversationTouchRecordInput {
|
|
conversation_id: conversation.conversation_id.clone(),
|
|
owner_user_id: conversation.owner_user_id.clone(),
|
|
title: was_empty.then(|| derive_conversation_title(user_message.text.as_str())),
|
|
updated_at_micros: current_utc_micros(),
|
|
})
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
let conversation_summary = conversation_summary_from_record(updated_conversation);
|
|
|
|
// Build tool context from document
|
|
let tool_context = context::build_tool_context(&document);
|
|
|
|
// Build and run agent
|
|
let llm_client = state.llm_client().ok_or_else(|| {
|
|
AppError::from_status(axum::http::StatusCode::SERVICE_UNAVAILABLE)
|
|
.with_details(json!({ "message": "LLM client not configured" }))
|
|
})?;
|
|
let llm_client = llm_client.clone();
|
|
let pricing = state.editor_generation_pricing().await.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({
|
|
"provider": "editor-generation-pricing",
|
|
"message": error.to_string(),
|
|
}))
|
|
})?;
|
|
|
|
let memory = VecMemory::new(previous_messages);
|
|
|
|
let mut agent = LlmChatAgentBuilder::new()
|
|
.with_client(llm_client)
|
|
.system_prompt(editor_agent_system_prompt())
|
|
.tool(EditImageTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateImageTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateCharacterTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
// TODO icons gen's result query is not prepared for now.
|
|
// .tool(GenerateIconSpritesheetTool {
|
|
// context: tool_context.clone(),
|
|
// })
|
|
.tool(GenerateSoundEffectTool)
|
|
.tool(GenerateBackgroundMusicTool)
|
|
.tool(GenerateVideoTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateUiDesignTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.max_turns(3)
|
|
.memory(memory)
|
|
.build();
|
|
|
|
let agent_result = agent.prompt(LlmMessage::user(user_message.text)).await;
|
|
|
|
let assistant_now = now_rfc3339();
|
|
|
|
match build_delta_messages(
|
|
agent_result,
|
|
&assistant_now,
|
|
document.messages.len(),
|
|
&document,
|
|
&tool_context,
|
|
&pricing,
|
|
) {
|
|
Err(err) => Ok(Json(EditorAgentMessageResponse {
|
|
conversation: conversation_summary,
|
|
delta_messages: Vec::new(),
|
|
error_message: Some(err.to_string()),
|
|
})),
|
|
Ok(delta_messages) => {
|
|
for msg in &delta_messages {
|
|
document.messages.push(msg.clone());
|
|
}
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
|
|
Ok(Json(EditorAgentMessageResponse {
|
|
conversation: conversation_summary,
|
|
delta_messages,
|
|
error_message: None,
|
|
}))
|
|
}
|
|
}
|
|
}
|
|
fn editor_agent_system_prompt() -> &'static str {
|
|
r#"
|
|
你是 Genarrative 图片画布 Agent，只负责帮助用户理解、规划和触发画布生成工具。
|
|
对话回复要简短，不能承诺免费生成；实际生成工具由后端按模型定价扣泥点。
|
|
"#
|
|
}
|
|
|
|
fn build_delta_messages(
|
|
result: Result<Vec<PromptOutput>, PromptError>,
|
|
created_at: &str,
|
|
messages_offset: usize,
|
|
document: &EditorAgentConversationMessagesDocument,
|
|
tool_context: &EditorToolContext,
|
|
pricing: &EditorGenerationPricingConfig,
|
|
) -> Result<Vec<EditorAgentMessage>, PromptError> {
|
|
let outputs = result?;
|
|
let mut messages = Vec::with_capacity(outputs.len());
|
|
|
|
for (i, out) in outputs.into_iter().enumerate() {
|
|
let absolute_idx = messages_offset + i;
|
|
match out {
|
|
PromptOutput::Text(text) => {
|
|
messages.push(EditorAgentMessage {
|
|
id: absolute_idx,
|
|
role: EditorAgentMessageRole::Assistant,
|
|
text,
|
|
attachments: Vec::new(),
|
|
tool_call: None,
|
|
created_at: created_at.to_string(),
|
|
});
|
|
}
|
|
PromptOutput::Tool(tco) => {
|
|
let display_args = display_args::build_tool_call_display_args(
|
|
tco.tool_call.name.as_str(),
|
|
&tco.tool_call.args,
|
|
document,
|
|
tool_context,
|
|
pricing,
|
|
)?;
|
|
messages.push(EditorAgentMessage {
|
|
id: absolute_idx,
|
|
role: EditorAgentMessageRole::System,
|
|
text: tco.message,
|
|
attachments: Vec::new(),
|
|
tool_call: Some(EditorAgentToolCall {
|
|
tool_name: tco.tool_call.name,
|
|
status: EditorAgentToolCallStatus::NotCompleted,
|
|
args: tco.tool_call.args,
|
|
display_args,
|
|
external_job_id: None,
|
|
images: Vec::new(),
|
|
videos: Vec::new(),
|
|
audios: Vec::new(),
|
|
error: None,
|
|
}),
|
|
created_at: created_at.to_string(),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(messages)
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
struct EditorAgentConversationDeleteResponse {
|
|
deleted_conversation_id: String,
|
|
conversation: EditorAgentConversationSummary,
|
|
}
|
|
|
|
pub async fn list_editor_agent_conversations(
|
|
State(state): State<AppState>,
|
|
Path(project_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
ensure_editor_project_access(&state, project_id.as_str(), owner_user_id.as_str()).await?;
|
|
let conversations = state
|
|
.spacetime_client()
|
|
.list_editor_agent_conversations(project_id, owner_user_id)
|
|
.await
|
|
.map_err(map_editor_project_error)?
|
|
.into_iter()
|
|
.map(conversation_summary_from_record)
|
|
.collect();
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationListResponse { conversations },
|
|
))
|
|
}
|
|
|
|
pub async fn create_editor_agent_conversation(
|
|
State(state): State<AppState>,
|
|
Path(project_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
Json(payload): Json<CreateEditorAgentConversationRequest>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
ensure_editor_project_access(&state, project_id.as_str(), owner_user_id.as_str()).await?;
|
|
|
|
let conversation_id = build_prefixed_uuid_id(EDITOR_AGENT_CONVERSATION_ID_PREFIX);
|
|
let messages_object_key = editor_agent_messages_object_key(conversation_id.as_str());
|
|
let title = normalize_optional_string(payload.title)
|
|
.unwrap_or_else(|| EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE.to_string());
|
|
let now_micros = current_utc_micros();
|
|
let seed_record = EditorAgentConversationRecord {
|
|
conversation_id: conversation_id.clone(),
|
|
project_id: project_id.clone(),
|
|
owner_user_id: owner_user_id.clone(),
|
|
title: title.clone(),
|
|
messages_object_key: messages_object_key.clone(),
|
|
deleted: false,
|
|
created_at: now_rfc3339(),
|
|
updated_at: now_rfc3339(),
|
|
updated_at_micros: now_micros,
|
|
};
|
|
write_messages_document(
|
|
&state,
|
|
&seed_record,
|
|
&empty_messages_document(conversation_id.as_str()),
|
|
)
|
|
.await?;
|
|
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.create_editor_agent_conversation(EditorAgentConversationCreateRecordInput {
|
|
conversation_id,
|
|
project_id,
|
|
owner_user_id,
|
|
title,
|
|
messages_object_key,
|
|
created_at_micros: now_micros,
|
|
})
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
let document = read_messages_document(&state, &conversation).await?;
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationResponse {
|
|
conversation: conversation_detail_from_record(conversation, document.messages),
|
|
},
|
|
))
|
|
}
|
|
|
|
pub async fn get_editor_agent_conversation(
|
|
State(state): State<AppState>,
|
|
Path(conversation_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id, owner_user_id)
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
|
|
conversation.conversation_id.as_str(),
|
|
);
|
|
let _conversation_lock_guard = conversation_lock.lock_owned().await;
|
|
let mut document = read_messages_document(&state, &conversation).await?;
|
|
let reconciled_messages =
|
|
reconcile::reconcile_editor_agent_tool_calls(&state, &conversation, &mut document).await?;
|
|
if !reconciled_messages.is_empty() {
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
}
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationResponse {
|
|
conversation: conversation_detail_from_record(conversation, document.messages),
|
|
},
|
|
))
|
|
}
|
|
|
|
pub async fn delete_editor_agent_conversation(
|
|
State(state): State<AppState>,
|
|
Path(conversation_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.delete_editor_agent_conversation(EditorAgentConversationDeleteRecordInput {
|
|
conversation_id,
|
|
owner_user_id,
|
|
updated_at_micros: current_utc_micros(),
|
|
})
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationDeleteResponse {
|
|
deleted_conversation_id: conversation.conversation_id.clone(),
|
|
conversation: conversation_summary_from_record(conversation),
|
|
},
|
|
))
|
|
}
|
|
|
|
pub async fn cancel_editor_agent_tool_call(
|
|
State(state): State<AppState>,
|
|
Path((conversation_id, message_id)): Path<(String, usize)>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id, owner_user_id)
|
|
.await
|
|
.map_err(|e| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": format!("conversation not found: {e}") }))
|
|
})?;
|
|
|
|
let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
|
|
conversation.conversation_id.as_str(),
|
|
);
|
|
let _conversation_lock_guard = conversation_lock.lock_owned().await;
|
|
|
|
let mut document: EditorAgentConversationMessagesDocument =
|
|
read_messages_document(&state, &conversation).await?;
|
|
|
|
// Validate message index
|
|
if message_id >= document.messages.len() {
|
|
return Err(AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": "message not found" })));
|
|
}
|
|
|
|
let msg = &mut document.messages[message_id];
|
|
|
|
// Validate role and tool_call
|
|
if msg.role != EditorAgentMessageRole::System {
|
|
return Err(editor_agent_bad_request("message is not a system message"));
|
|
}
|
|
let tc = msg
|
|
.tool_call
|
|
.as_mut()
|
|
.ok_or_else(|| editor_agent_bad_request("message has no tool call"))?;
|
|
if tc.status != EditorAgentToolCallStatus::NotCompleted || tc.external_job_id.is_some() {
|
|
return Err(editor_agent_bad_request(
|
|
"tool call is no longer pending confirmation",
|
|
));
|
|
}
|
|
|
|
tc.status = EditorAgentToolCallStatus::Cancelled;
|
|
let arg_json = tc.args.to_string();
|
|
msg.text = format!(
|
|
"[tool_call:{tool_name}] args: {arg_json} output: 用户已取消该操作",
|
|
tool_name = tc.tool_name,
|
|
arg_json = arg_json,
|
|
);
|
|
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
&document.messages[message_id],
|
|
))
|
|
}
|
|
|
|
pub async fn confirm_editor_agent_tool_call(
|
|
State(state): State<AppState>,
|
|
Path((conversation_id, message_id)): Path<(String, usize)>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id, owner_user_id)
|
|
.await
|
|
.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND).with_details(json!({
|
|
"message": format!("conversation not found: {error}"),
|
|
}))
|
|
})?;
|
|
let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
|
|
conversation.conversation_id.as_str(),
|
|
);
|
|
let _conversation_lock_guard = conversation_lock.lock_owned().await;
|
|
let mut document = read_messages_document(&state, &conversation).await?;
|
|
let message = document
|
|
.messages
|
|
.get(message_id)
|
|
.ok_or_else(|| AppError::from_status(axum::http::StatusCode::NOT_FOUND))?;
|
|
if message.role != EditorAgentMessageRole::System {
|
|
return Err(editor_agent_bad_request("message is not a system message"));
|
|
}
|
|
let tool_call = message
|
|
.tool_call
|
|
.as_ref()
|
|
.ok_or_else(|| editor_agent_bad_request("message has no tool call"))?;
|
|
if tool_call.status == EditorAgentToolCallStatus::Cancelled {
|
|
return Err(editor_agent_bad_request("tool call was cancelled"));
|
|
}
|
|
if tool_call.status != EditorAgentToolCallStatus::NotCompleted
|
|
|| tool_call.external_job_id.is_some()
|
|
{
|
|
return Ok(json_success_body(Some(&request_context), message));
|
|
}
|
|
|
|
let tool_name = tool_call.tool_name.clone();
|
|
let tool_args = tool_call.args.clone();
|
|
let pricing = state.editor_generation_pricing().await.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({
|
|
"provider": "editor-generation-pricing",
|
|
"message": error.to_string(),
|
|
}))
|
|
})?;
|
|
let project = load_editor_agent_project(&state, &conversation).await?;
|
|
let context = context::build_tool_context(&document);
|
|
let generation_inputs = |title: &str, value: &str| {
|
|
Some(json!({
|
|
"source": "editor-agent",
|
|
"conversationId": conversation.conversation_id,
|
|
"toolCallMessageId": message_id,
|
|
"fields": [{ "title": title, "value": value }],
|
|
}))
|
|
};
|
|
|
|
let (job_kind, request_label, price_mud_points, payload) = match tool_name.as_str() {
|
|
GenerateImageTool::NAME | GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => {
|
|
let args: GenerateImageToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let tool = GenerateImageTool {
|
|
context: context.clone(),
|
|
};
|
|
tool.validate_args(&args).map_err(|error| {
|
|
editor_agent_bad_request(format!("invalid tool call args: {error}"))
|
|
})?;
|
|
let kind = match tool_name.as_str() {
|
|
GenerateCharacterTool::NAME => Some("character"),
|
|
GenerateUiDesignTool::NAME => Some("ui-design"),
|
|
_ => None,
|
|
};
|
|
let price = match kind {
|
|
Some("character") => GenerateCharacterTool {
|
|
context: context.clone(),
|
|
}
|
|
.pricing(&pricing, &args),
|
|
Some("ui-design") => GenerateUiDesignTool {
|
|
context: context.clone(),
|
|
}
|
|
.pricing(&pricing, &args),
|
|
_ => tool.pricing(&pricing, &args),
|
|
};
|
|
let title = args.prompt.clone();
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let payload = EditorImageGenerationRequest {
|
|
prompt: args.prompt,
|
|
size: None,
|
|
kind: kind.map(ToOwned::to_owned),
|
|
model: None,
|
|
screen_color: (kind == Some("character")).then(|| "auto".to_string()),
|
|
seg_model: (kind == Some("character")).then(|| "birefnet".to_string()),
|
|
aspect_ratio: args.aspect_ratio,
|
|
image_size: args.image_size,
|
|
reference_image_srcs: Some(reference_image_srcs),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
asset_kind: Some(kind.unwrap_or("editor_agent_generated_image").to_string()),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title.clone()),
|
|
source_resource_id: None,
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project, &tool_name, &title,
|
|
)),
|
|
};
|
|
(
|
|
EDITOR_IMAGE_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成图片",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
EditImageTool::NAME => {
|
|
let args: EditImageToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
if args.prompt.trim().is_empty() {
|
|
return Err(editor_agent_bad_request("prompt not provided"));
|
|
}
|
|
let source_image_src = context
|
|
.image_data_key(&args.object_image_id)
|
|
.ok_or_else(|| editor_agent_bad_request("object image not found"))?
|
|
.to_string();
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let title = args.prompt.clone();
|
|
let price = EditImageTool {
|
|
context: context.clone(),
|
|
}
|
|
.pricing(&pricing, &args);
|
|
let payload = EditorImageEditRequest {
|
|
prompt: args.prompt,
|
|
source_image_src,
|
|
size: None,
|
|
model: None,
|
|
reference_image_srcs: Some(reference_image_srcs),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
asset_kind: Some("editor_agent_edit_image".to_string()),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title.clone()),
|
|
source_resource_id: None,
|
|
target_layer_id: None,
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
EditImageTool::NAME,
|
|
&title,
|
|
)),
|
|
};
|
|
(
|
|
EDITOR_IMAGE_EDIT_JOB_KIND,
|
|
"画布 Agent 修改图片",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateIconSpritesheetTool::NAME => {
|
|
let args: GenerateIconSpritesheetToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let tool = GenerateIconSpritesheetTool {
|
|
context: context.clone(),
|
|
};
|
|
let price = tool.pricing(&pricing, &args);
|
|
let reference_image_src = context
|
|
.image_data_key(&args.reference_image_id)
|
|
.ok_or_else(|| editor_agent_bad_request("reference image not found"))?
|
|
.to_string();
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let payload = EditorIconSpritesheetGenerationRequest {
|
|
reference_image_src,
|
|
reference_image_srcs: Some(reference_image_srcs),
|
|
icon_descriptions: args.icon_descriptions.clone(),
|
|
model: None,
|
|
screen_color: Some("auto".to_string()),
|
|
seg_model: Some("birefnet".to_string()),
|
|
aspect_ratio: args.aspect_ratio,
|
|
image_size: args.image_size,
|
|
project_id: Some(conversation.project_id.clone()),
|
|
generation_inputs: Some(
|
|
json!({"source":"editor-agent","conversationId":conversation.conversation_id,"toolCallMessageId":message_id}),
|
|
),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some("图标素材图集".to_string()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateIconSpritesheetTool::NAME,
|
|
"图标素材图集",
|
|
)),
|
|
};
|
|
(
|
|
EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成图标素材",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateVideoTool::NAME => {
|
|
let args: GenerateVideoToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let title = args.prompt.clone();
|
|
let price = GenerateVideoTool {
|
|
context: context.clone(),
|
|
}
|
|
.pricing(&pricing, &args);
|
|
let payload = EditorVideoGenerateRequest {
|
|
prompt: args.prompt,
|
|
model: args.model.unwrap_or_else(|| "seedance2.0-fast".to_string()),
|
|
aspect_ratio: args.aspect_ratio.unwrap_or_else(|| "16:9".to_string()),
|
|
duration_seconds: args.duration_seconds.unwrap_or(4),
|
|
resolution: args.resolution.unwrap_or_else(|| "720p".to_string()),
|
|
mode: "std".to_string(),
|
|
sound: args.sound.unwrap_or_else(|| "off".to_string()),
|
|
web_search_enabled: false,
|
|
reference_image_srcs,
|
|
reference_video_srcs: Vec::new(),
|
|
reference_audio_srcs: Vec::new(),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateVideoTool::NAME,
|
|
&title,
|
|
)),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
source_resource_id: None,
|
|
asset_kind: Some("video".to_string()),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title),
|
|
};
|
|
(
|
|
EDITOR_VIDEO_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成视频",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateSoundEffectTool::NAME => {
|
|
let args: GenerateSoundEffectToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let price = GenerateSoundEffectTool.pricing(&pricing, &args);
|
|
let title = args.prompt.clone();
|
|
let payload = EditorSoundEffectGenerateRequest {
|
|
prompt: args.prompt,
|
|
model: args.model,
|
|
duration: args.duration.unwrap_or(3),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateSoundEffectTool::NAME,
|
|
&title,
|
|
)),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title),
|
|
};
|
|
(
|
|
EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成音效",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateBackgroundMusicTool::NAME => {
|
|
let args: GenerateBackgroundMusicToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let price = GenerateBackgroundMusicTool.pricing(&pricing, &args);
|
|
let title = args.prompt.clone();
|
|
let payload = EditorBackgroundMusicGenerateRequest {
|
|
gpt_description_prompt: args.prompt,
|
|
make_instrumental: args.make_instrumental,
|
|
project_id: Some(conversation.project_id.clone()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateBackgroundMusicTool::NAME,
|
|
"生成背景音乐",
|
|
)),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some("生成背景音乐".to_string()),
|
|
};
|
|
(
|
|
EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成背景音乐",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
_ => {
|
|
return Err(editor_agent_bad_request(format!(
|
|
"unsupported tool: {tool_name}"
|
|
)));
|
|
}
|
|
};
|
|
let payload = payload.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR)
|
|
.with_details(json!({"message": format!("failed to serialize tool payload: {error}")}))
|
|
})?;
|
|
let (job_id, dedupe_key) = editor_agent_tool_job_identity(
|
|
conversation.conversation_id.as_str(),
|
|
message_id,
|
|
tool_name.as_str(),
|
|
);
|
|
let job = enqueue_editor_generation_job_with_identity(
|
|
&state,
|
|
conversation.owner_user_id.as_str(),
|
|
job_kind,
|
|
conversation.project_id.clone(),
|
|
request_label,
|
|
u64::from(price_mud_points),
|
|
&payload,
|
|
job_id,
|
|
dedupe_key,
|
|
)
|
|
.await?;
|
|
let message = &mut document.messages[message_id];
|
|
let tool_call = message
|
|
.tool_call
|
|
.as_mut()
|
|
.ok_or_else(|| editor_agent_bad_request("message has no tool call"))?;
|
|
tool_call.external_job_id = Some(job.job_id);
|
|
tool_call.status = EditorAgentToolCallStatus::NotCompleted;
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
&document.messages[message_id],
|
|
))
|
|
}
|
|
|
|
fn parse_confirm_tool_args<T: serde::de::DeserializeOwned>(value: &Value) -> Result<T, AppError> {
|
|
serde_json::from_value(value.clone())
|
|
.map_err(|error| editor_agent_bad_request(format!("invalid tool call args: {error}")))
|
|
}
|
|
|
|
fn editor_agent_tool_job_identity(
|
|
conversation_id: &str,
|
|
message_id: usize,
|
|
tool_name: &str,
|
|
) -> (String, String) {
|
|
let dedupe_key = format!("editor-agent:{conversation_id}:{message_id}:{tool_name}");
|
|
let digest = Sha256::digest(dedupe_key.as_bytes());
|
|
(format!("task-editor-agent-{digest:x}"), dedupe_key)
|
|
}
|
|
|
|
async fn load_editor_agent_project(
|
|
state: &AppState,
|
|
conversation: &EditorAgentConversationRecord,
|
|
) -> Result<spacetime_client::EditorProjectRecord, AppError> {
|
|
state
|
|
.spacetime_client()
|
|
.get_editor_project(EditorProjectGetRecordInput {
|
|
project_id: conversation.project_id.clone(),
|
|
owner_user_id: conversation.owner_user_id.clone(),
|
|
})
|
|
.await
|
|
.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": format!("project not found: {error}") }))
|
|
})
|
|
}
|