1421 lines
56 KiB
Rust
1421 lines
56 KiB
Rust
use std::future::IntoFuture;
|
|
use std::time::Duration;
|
|
|
|
use axum::extract::{Path, State};
|
|
use axum::{Extension, Json};
|
|
use module_editor_agent::{
|
|
EDITOR_AGENT_CONVERSATION_ID_PREFIX, EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE,
|
|
derive_conversation_title, editor_agent_messages_object_key, validate_user_message,
|
|
};
|
|
use platform_editor_agent::framework::agent_builder::AgentBuilder;
|
|
use platform_editor_agent::framework::error::PromptError;
|
|
use platform_editor_agent::framework::memory::VecMemory;
|
|
use platform_editor_agent::framework::run::PromptOutput;
|
|
use platform_editor_agent::framework::tool::Tool;
|
|
use platform_llm::LlmMessage;
|
|
use serde::Serialize;
|
|
use serde_json::{Value, json};
|
|
use sha2::{Digest, Sha256};
|
|
use shared_contracts::assets::{
|
|
EditorBackgroundMusicGenerateRequest, EditorSoundEffectGenerateRequest,
|
|
EditorVideoGenerateRequest,
|
|
};
|
|
use shared_contracts::editor_agent::{
|
|
CreateEditorAgentConversationRequest, EDITOR_AGENT_ERROR_MESSAGE_PREFIX,
|
|
EditorAgentConversationListResponse, EditorAgentConversationMessagesDocument,
|
|
EditorAgentConversationResponse, EditorAgentConversationSummary, EditorAgentMessage,
|
|
EditorAgentMessageRequest, EditorAgentMessageResponse, EditorAgentMessageRole,
|
|
EditorAgentToolCall, EditorAgentToolCallStatus,
|
|
};
|
|
use spacetime_client::{
|
|
EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput,
|
|
EditorAgentConversationRecord, EditorAgentConversationTouchRecordInput,
|
|
EditorProjectGetRecordInput,
|
|
};
|
|
|
|
use crate::api_response::json_success_body;
|
|
use crate::auth::AuthenticatedAccessToken;
|
|
use crate::editor_agent::pricing::EditorAgentPricedTool;
|
|
use crate::editor_agent::utils::{
|
|
IntoImageId, build_editor_agent_canvas_completion, conversation_detail_from_record,
|
|
conversation_summary_from_record, editor_agent_bad_request, empty_messages_document,
|
|
ensure_editor_project_access, normalize_editor_agent_attachments, now_rfc3339,
|
|
read_messages_document, require_editor_agent_sidebar_enabled, write_messages_document,
|
|
};
|
|
use crate::editor_agent::{context, display_args, reconcile, tool_args};
|
|
use crate::editor_generation_config::EditorGenerationPricingConfig;
|
|
use crate::editor_generation_queue::{
|
|
EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND,
|
|
EDITOR_IMAGE_EDIT_JOB_KIND, EDITOR_IMAGE_GENERATION_JOB_KIND,
|
|
EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND, EDITOR_VIDEO_GENERATION_JOB_KIND,
|
|
enqueue_editor_generation_job_with_identity,
|
|
};
|
|
use crate::editor_project::{
|
|
EditorIconSpritesheetGenerationRequest, EditorImageEditRequest, EditorImageGenerationRequest,
|
|
};
|
|
use crate::editor_project::{current_utc_micros, map_editor_project_error};
|
|
use crate::http_error::AppError;
|
|
use crate::request_context::RequestContext;
|
|
use crate::state::AppState;
|
|
use platform_editor_agent::agent::agent::LlmChatAgentBuilder;
|
|
use platform_editor_agent::agent::tools::context::EditorToolContext;
|
|
use platform_editor_agent::agent::tools::edit_image::{EditImageTool, EditImageToolArgs};
|
|
use platform_editor_agent::agent::tools::generate_background_music::{
|
|
GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs,
|
|
};
|
|
use platform_editor_agent::agent::tools::generate_character::{
|
|
GenerateCharacterTool, GenerateCharacterToolArgs,
|
|
};
|
|
use platform_editor_agent::agent::tools::generate_icon_spritesheet::{
|
|
GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs,
|
|
};
|
|
use platform_editor_agent::agent::tools::generate_image::{
|
|
GenerateImageTool, GenerateImageToolArgs,
|
|
};
|
|
use platform_editor_agent::agent::tools::generate_sound_effect::{
|
|
GenerateSoundEffectTool, GenerateSoundEffectToolArgs,
|
|
};
|
|
use platform_editor_agent::agent::tools::generate_ui_design::{
|
|
GenerateUiDesignTool, GenerateUiDesignToolArgs,
|
|
};
|
|
use platform_editor_agent::agent::tools::generate_video::{
|
|
GenerateVideoTool, GenerateVideoToolArgs,
|
|
};
|
|
use shared_kernel::{build_prefixed_uuid_id, normalize_optional_string, normalize_required_string};
|
|
use tokio::time::{Instant, timeout};
|
|
|
|
const EDITOR_AGENT_CLIENT_MESSAGE_ID_MAX_CHARS: usize = 128;
|
|
const EDITOR_AGENT_PROMPT_TIMEOUT_MS: u64 = 18 * 60_000;
|
|
const EDITOR_AGENT_PROMPT_TIMEOUT_MESSAGE: &str = "规划总时长已达到 18 分钟安全上限";
|
|
const EDITOR_AGENT_LLM_UNAVAILABLE_MESSAGE: &str = "美术 Agent 服务暂不可用，请稍后重试";
|
|
const EDITOR_AGENT_PRICING_UNAVAILABLE_MESSAGE: &str = "美术 Agent 生成定价暂不可用，请稍后重试";
|
|
|
|
pub async fn editor_agent_message(
|
|
State(state): State<AppState>,
|
|
Path(conversation_id): Path<String>,
|
|
Extension(_request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
Json(payload): Json<EditorAgentMessageRequest>,
|
|
) -> Result<Json<EditorAgentMessageResponse>, AppError> {
|
|
let message_started_at = Instant::now();
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
let client_message_id = validate_editor_agent_message_request(&payload)?;
|
|
let normalized_text = payload.text.trim().to_string();
|
|
// Load conversation & attachments
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id.clone(), owner_user_id.clone())
|
|
.await
|
|
.map_err(|e| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": format!("conversation not found: {e}") }))
|
|
})?;
|
|
|
|
let attachments =
|
|
normalize_editor_agent_attachments(&state, &conversation, payload.attachments.as_slice())
|
|
.await?;
|
|
|
|
let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
|
|
conversation.conversation_id.as_str(),
|
|
);
|
|
let _conversation_lock_guard = conversation_lock.lock_owned().await;
|
|
|
|
let mut document: EditorAgentConversationMessagesDocument =
|
|
read_messages_document(&state, &conversation).await?;
|
|
|
|
let existing_user_index = find_idempotent_editor_agent_user_message(
|
|
&document,
|
|
client_message_id.as_str(),
|
|
normalized_text.as_str(),
|
|
attachments.as_slice(),
|
|
)?;
|
|
|
|
let (user_message, history_end, conversation_summary) =
|
|
if let Some(user_index) = existing_user_index {
|
|
let delta_messages = document.messages[user_index + 1..]
|
|
.iter()
|
|
.take_while(|message| message.role != EditorAgentMessageRole::User)
|
|
.cloned()
|
|
.collect::<Vec<_>>();
|
|
if !delta_messages.is_empty() {
|
|
return Ok(Json(EditorAgentMessageResponse {
|
|
conversation: conversation_summary_from_record(conversation),
|
|
delta_messages,
|
|
error_message: None,
|
|
}));
|
|
}
|
|
|
|
(
|
|
document.messages[user_index].clone(),
|
|
user_index,
|
|
conversation_summary_from_record(conversation.clone()),
|
|
)
|
|
} else {
|
|
// Determine initialization before attachment bookkeeping adds a system message.
|
|
let was_empty = document.messages.is_empty();
|
|
let now = now_rfc3339();
|
|
if !attachments.is_empty() {
|
|
let mut attachment_info = String::new();
|
|
attachment_info.push_str("user has just uploaded attachments of the order: ");
|
|
for attachment in &attachments {
|
|
attachment_info.push_str(&format!("{} ,", attachment.clone().into_image_id()));
|
|
}
|
|
document.messages.push(EditorAgentMessage {
|
|
id: document.messages.len(),
|
|
client_message_id: None,
|
|
role: EditorAgentMessageRole::System,
|
|
text: attachment_info,
|
|
attachments: Vec::new(),
|
|
tool_call: None,
|
|
created_at: now.clone(),
|
|
});
|
|
}
|
|
|
|
let history_end = document.messages.len();
|
|
let user_message = EditorAgentMessage {
|
|
id: document.messages.len(),
|
|
client_message_id: Some(client_message_id),
|
|
role: EditorAgentMessageRole::User,
|
|
text: normalized_text,
|
|
attachments,
|
|
tool_call: None,
|
|
created_at: now,
|
|
};
|
|
document.messages.push(user_message.clone());
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
|
|
// Persist and return the authoritative summary for every turn. Initialization sets the
|
|
// title from the first user prompt; a metadata write failure must fail the request.
|
|
let updated_conversation = state
|
|
.spacetime_client()
|
|
.touch_editor_agent_conversation(EditorAgentConversationTouchRecordInput {
|
|
conversation_id: conversation.conversation_id.clone(),
|
|
owner_user_id: conversation.owner_user_id.clone(),
|
|
title: was_empty.then(|| derive_conversation_title(user_message.text.as_str())),
|
|
updated_at_micros: current_utc_micros(),
|
|
})
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
|
|
(
|
|
user_message,
|
|
history_end,
|
|
conversation_summary_from_record(updated_conversation),
|
|
)
|
|
};
|
|
|
|
// The current user message is passed separately to prompt(), so memory stops before it.
|
|
let previous_messages: Vec<LlmMessage> = document.messages[..history_end]
|
|
.iter()
|
|
.map(|message| match message.role {
|
|
EditorAgentMessageRole::User => LlmMessage::user(&message.text),
|
|
EditorAgentMessageRole::Assistant => LlmMessage::assistant(&message.text),
|
|
EditorAgentMessageRole::System => LlmMessage::system(&message.text),
|
|
})
|
|
// Tool calls and attachment bookkeeping are separate system messages.
|
|
.rev()
|
|
.take(18)
|
|
.rev()
|
|
.collect();
|
|
|
|
// Build tool context from document
|
|
let tool_context = context::build_tool_context(&document);
|
|
|
|
// Build and run agent
|
|
let Some(llm_client) = state.editor_agent_llm_client() else {
|
|
tracing::warn!(
|
|
conversation_id = %conversation.conversation_id,
|
|
"美术 Agent LLM 客户端未配置"
|
|
);
|
|
return persist_editor_agent_planning_error(
|
|
&state,
|
|
&conversation,
|
|
&mut document,
|
|
conversation_summary,
|
|
EDITOR_AGENT_LLM_UNAVAILABLE_MESSAGE,
|
|
)
|
|
.await;
|
|
};
|
|
let llm_client = llm_client.clone();
|
|
let pricing = match state.editor_generation_pricing().await {
|
|
Ok(pricing) => pricing,
|
|
Err(error) => {
|
|
tracing::warn!(
|
|
conversation_id = %conversation.conversation_id,
|
|
error = %error,
|
|
"读取美术 Agent 生成定价失败"
|
|
);
|
|
return persist_editor_agent_planning_error(
|
|
&state,
|
|
&conversation,
|
|
&mut document,
|
|
conversation_summary,
|
|
EDITOR_AGENT_PRICING_UNAVAILABLE_MESSAGE,
|
|
)
|
|
.await;
|
|
}
|
|
};
|
|
|
|
let memory = VecMemory::new(previous_messages);
|
|
|
|
let mut agent = LlmChatAgentBuilder::new()
|
|
.with_client(llm_client)
|
|
.system_prompt(editor_agent_system_prompt())
|
|
.tool(EditImageTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateImageTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateCharacterTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateIconSpritesheetTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateSoundEffectTool)
|
|
.tool(GenerateBackgroundMusicTool)
|
|
.tool(GenerateVideoTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateUiDesignTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.max_turns(3)
|
|
.memory(memory)
|
|
.build();
|
|
|
|
let remaining_prompt_duration =
|
|
remaining_editor_agent_prompt_duration(message_started_at.elapsed());
|
|
let agent_result = run_editor_agent_prompt_with_timeout(
|
|
agent.prompt(LlmMessage::user(user_message.text.clone())),
|
|
remaining_prompt_duration,
|
|
)
|
|
.await;
|
|
|
|
let assistant_now = now_rfc3339();
|
|
|
|
match build_delta_messages(
|
|
agent_result,
|
|
&assistant_now,
|
|
document.messages.len(),
|
|
&document,
|
|
&tool_context,
|
|
&pricing,
|
|
) {
|
|
Err(error) => {
|
|
persist_editor_agent_planning_error(
|
|
&state,
|
|
&conversation,
|
|
&mut document,
|
|
conversation_summary,
|
|
error.to_string(),
|
|
)
|
|
.await
|
|
}
|
|
Ok(delta_messages) => {
|
|
for msg in &delta_messages {
|
|
document.messages.push(msg.clone());
|
|
}
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
|
|
Ok(Json(EditorAgentMessageResponse {
|
|
conversation: conversation_summary,
|
|
delta_messages,
|
|
error_message: None,
|
|
}))
|
|
}
|
|
}
|
|
}
|
|
|
|
fn remaining_editor_agent_prompt_duration(elapsed: Duration) -> Duration {
|
|
Duration::from_millis(EDITOR_AGENT_PROMPT_TIMEOUT_MS).saturating_sub(elapsed)
|
|
}
|
|
|
|
async fn run_editor_agent_prompt_with_timeout<F>(
|
|
future: F,
|
|
duration: Duration,
|
|
) -> Result<Vec<PromptOutput>, PromptError>
|
|
where
|
|
F: IntoFuture<Output = Result<Vec<PromptOutput>, PromptError>>,
|
|
{
|
|
timeout(duration, future.into_future())
|
|
.await
|
|
.unwrap_or_else(|_| {
|
|
Err(PromptError::CompletionError(
|
|
EDITOR_AGENT_PROMPT_TIMEOUT_MESSAGE.to_string(),
|
|
))
|
|
})
|
|
}
|
|
|
|
fn build_editor_agent_error_message(
|
|
message_id: usize,
|
|
error: impl std::fmt::Display,
|
|
) -> EditorAgentMessage {
|
|
EditorAgentMessage {
|
|
id: message_id,
|
|
client_message_id: None,
|
|
role: EditorAgentMessageRole::System,
|
|
text: format!("{EDITOR_AGENT_ERROR_MESSAGE_PREFIX}{error}"),
|
|
attachments: Vec::new(),
|
|
tool_call: None,
|
|
created_at: now_rfc3339(),
|
|
}
|
|
}
|
|
|
|
async fn persist_editor_agent_planning_error(
|
|
state: &AppState,
|
|
conversation: &EditorAgentConversationRecord,
|
|
document: &mut EditorAgentConversationMessagesDocument,
|
|
conversation_summary: EditorAgentConversationSummary,
|
|
error: impl std::fmt::Display,
|
|
) -> Result<Json<EditorAgentMessageResponse>, AppError> {
|
|
let error_message = build_editor_agent_error_message(document.messages.len(), error);
|
|
document.messages.push(error_message.clone());
|
|
write_messages_document(state, conversation, document).await?;
|
|
|
|
Ok(Json(EditorAgentMessageResponse {
|
|
conversation: conversation_summary,
|
|
delta_messages: vec![error_message],
|
|
error_message: None,
|
|
}))
|
|
}
|
|
|
|
fn validate_editor_agent_message_request(
|
|
payload: &EditorAgentMessageRequest,
|
|
) -> Result<String, AppError> {
|
|
let client_message_id = normalize_required_string(payload.client_message_id.as_str())
|
|
.ok_or_else(|| editor_agent_bad_request("clientMessageId is required"))?;
|
|
if client_message_id.chars().count() > EDITOR_AGENT_CLIENT_MESSAGE_ID_MAX_CHARS {
|
|
return Err(editor_agent_bad_request(format!(
|
|
"clientMessageId must not exceed {EDITOR_AGENT_CLIENT_MESSAGE_ID_MAX_CHARS} characters"
|
|
)));
|
|
}
|
|
let attachment_reference_ids = payload
|
|
.attachments
|
|
.iter()
|
|
.map(|attachment| attachment.reference_id.clone())
|
|
.collect::<Vec<_>>();
|
|
validate_user_message(payload.text.as_str(), attachment_reference_ids.as_slice())
|
|
.map_err(|error| editor_agent_bad_request(error.to_string()))?;
|
|
Ok(client_message_id)
|
|
}
|
|
|
|
fn find_idempotent_editor_agent_user_message(
|
|
document: &EditorAgentConversationMessagesDocument,
|
|
client_message_id: &str,
|
|
text: &str,
|
|
attachments: &[shared_contracts::editor_agent::EditorAgentAttachmentRef],
|
|
) -> Result<Option<usize>, AppError> {
|
|
let Some((index, message)) = document
|
|
.messages
|
|
.iter()
|
|
.enumerate()
|
|
.find(|(_, message)| message.client_message_id.as_deref() == Some(client_message_id))
|
|
else {
|
|
return Ok(None);
|
|
};
|
|
|
|
if message.role != EditorAgentMessageRole::User
|
|
|| message.text != text
|
|
|| !editor_agent_attachment_requests_match(&message.attachments, attachments)
|
|
{
|
|
return Err(
|
|
AppError::from_status(axum::http::StatusCode::CONFLICT).with_details(json!({
|
|
"provider": "editor-agent",
|
|
"field": "clientMessageId",
|
|
"message": "clientMessageId already exists with different message content",
|
|
})),
|
|
);
|
|
}
|
|
|
|
Ok(Some(index))
|
|
}
|
|
|
|
fn editor_agent_attachment_requests_match(
|
|
stored: &[shared_contracts::editor_agent::EditorAgentAttachmentRef],
|
|
submitted: &[shared_contracts::editor_agent::EditorAgentAttachmentRef],
|
|
) -> bool {
|
|
stored.len() == submitted.len()
|
|
&& stored.iter().zip(submitted).all(|(left, right)| {
|
|
left.source == right.source && left.reference_id == right.reference_id
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use shared_contracts::editor_agent::{EditorAgentAttachmentRef, EditorAgentAttachmentSource};
|
|
|
|
fn attachment(reference_id: impl Into<String>) -> EditorAgentAttachmentRef {
|
|
EditorAgentAttachmentRef {
|
|
source: EditorAgentAttachmentSource::CanvasResource,
|
|
reference_id: reference_id.into(),
|
|
object_key: None,
|
|
image_src: "/generated/test.png".to_string(),
|
|
thumbnail_src: None,
|
|
label: None,
|
|
width: None,
|
|
height: None,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn validates_editor_agent_message_before_normalizing_attachments() {
|
|
let empty_payload = EditorAgentMessageRequest {
|
|
client_message_id: "client-message-empty".to_string(),
|
|
text: " ".to_string(),
|
|
attachments: Vec::new(),
|
|
};
|
|
assert!(validate_editor_agent_message_request(&empty_payload).is_err());
|
|
|
|
let too_many_payload = EditorAgentMessageRequest {
|
|
client_message_id: "client-message-many".to_string(),
|
|
text: "生成一张图".to_string(),
|
|
attachments: (0..10)
|
|
.map(|index| attachment(format!("res-{index}")))
|
|
.collect(),
|
|
};
|
|
assert!(validate_editor_agent_message_request(&too_many_payload).is_err());
|
|
|
|
let attachment_only_payload = EditorAgentMessageRequest {
|
|
client_message_id: "client-message-attachment".to_string(),
|
|
text: String::new(),
|
|
attachments: vec![attachment("res-1")],
|
|
};
|
|
assert!(validate_editor_agent_message_request(&attachment_only_payload).is_ok());
|
|
|
|
let missing_client_message_id = EditorAgentMessageRequest {
|
|
client_message_id: " ".to_string(),
|
|
text: "生成一张图".to_string(),
|
|
attachments: Vec::new(),
|
|
};
|
|
assert!(validate_editor_agent_message_request(&missing_client_message_id).is_err());
|
|
|
|
let oversized_client_message_id = EditorAgentMessageRequest {
|
|
client_message_id: "x".repeat(EDITOR_AGENT_CLIENT_MESSAGE_ID_MAX_CHARS + 1),
|
|
text: "生成一张图".to_string(),
|
|
attachments: Vec::new(),
|
|
};
|
|
assert!(validate_editor_agent_message_request(&oversized_client_message_id).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn detects_idempotent_message_replays_and_content_conflicts() {
|
|
let stored_attachment = attachment("res-1");
|
|
let document = EditorAgentConversationMessagesDocument {
|
|
version: 2,
|
|
conversation_id: "conversation-1".to_string(),
|
|
messages: vec![EditorAgentMessage {
|
|
id: 0,
|
|
client_message_id: Some("client-message-1".to_string()),
|
|
role: EditorAgentMessageRole::User,
|
|
text: "生成一张图".to_string(),
|
|
attachments: vec![stored_attachment.clone()],
|
|
tool_call: None,
|
|
created_at: "2026-07-16T00:00:00Z".to_string(),
|
|
}],
|
|
};
|
|
|
|
assert_eq!(
|
|
find_idempotent_editor_agent_user_message(
|
|
&document,
|
|
"client-message-1",
|
|
"生成一张图",
|
|
&[stored_attachment.clone()],
|
|
)
|
|
.expect("same request should be an idempotent replay"),
|
|
Some(0),
|
|
);
|
|
assert!(
|
|
find_idempotent_editor_agent_user_message(
|
|
&document,
|
|
"client-message-1",
|
|
"生成另一张图",
|
|
&[stored_attachment],
|
|
)
|
|
.is_err()
|
|
);
|
|
assert_eq!(
|
|
find_idempotent_editor_agent_user_message(
|
|
&document,
|
|
"client-message-2",
|
|
"生成一张图",
|
|
&[],
|
|
)
|
|
.expect("new request should not match"),
|
|
None,
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn builds_system_error_message_with_wire_prefix() {
|
|
let message = build_editor_agent_error_message(3, "planning failed");
|
|
|
|
assert_eq!(message.id, 3);
|
|
assert_eq!(message.role, EditorAgentMessageRole::System);
|
|
assert_eq!(message.text, "ERROR planning failed");
|
|
assert!(message.tool_call.is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn direct_planning_failures_use_user_facing_chinese_copy() {
|
|
assert_eq!(
|
|
build_editor_agent_error_message(1, EDITOR_AGENT_LLM_UNAVAILABLE_MESSAGE).text,
|
|
"ERROR 美术 Agent 服务暂不可用，请稍后重试"
|
|
);
|
|
assert_eq!(
|
|
build_editor_agent_error_message(2, EDITOR_AGENT_PRICING_UNAVAILABLE_MESSAGE).text,
|
|
"ERROR 美术 Agent 生成定价暂不可用，请稍后重试"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn legacy_confirm_args_use_current_defaults_and_ignore_retired_fields() {
|
|
let music: GenerateBackgroundMusicToolArgs = parse_confirm_tool_args(&json!({
|
|
"prompt": "温暖舒缓的钢琴背景音乐",
|
|
"make_instrumental": false
|
|
}))
|
|
.expect("旧背景音乐字段不应阻断确认");
|
|
assert_eq!(music.model, GenerateBackgroundMusicTool::DEFAULT_MODEL);
|
|
|
|
let ui: GenerateUiDesignToolArgs = parse_confirm_tool_args(&json!({
|
|
"prompt": "生成游戏主界面"
|
|
}))
|
|
.expect("旧 UI 消息缺少 model 时应使用当前固定模型");
|
|
assert_eq!(ui.model, platform_image::GPT_IMAGE_2_MODEL);
|
|
|
|
let video: GenerateVideoToolArgs = parse_confirm_tool_args(&json!({
|
|
"prompt": "镜头向前推进"
|
|
}))
|
|
.expect("旧视频消息缺少可选字段时应使用当前默认值");
|
|
assert_eq!(video.sound.as_deref(), Some("on"));
|
|
|
|
let sound: GenerateSoundEffectToolArgs = parse_confirm_tool_args(&json!({
|
|
"prompt": "按钮点击声"
|
|
}))
|
|
.expect("旧音效消息缺少时长时应使用当前默认值");
|
|
assert_eq!(sound.duration, Some(5));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn prompt_timeout_applies_to_the_whole_agent_run() {
|
|
let error = run_editor_agent_prompt_with_timeout(
|
|
std::future::pending::<Result<Vec<PromptOutput>, PromptError>>(),
|
|
Duration::from_millis(1),
|
|
)
|
|
.await
|
|
.expect_err("pending agent run should hit the prompt deadline");
|
|
|
|
assert_eq!(EDITOR_AGENT_PROMPT_TIMEOUT_MS, 1_080_000);
|
|
assert_eq!(
|
|
remaining_editor_agent_prompt_duration(Duration::from_secs(17 * 60)),
|
|
Duration::from_secs(60)
|
|
);
|
|
assert_eq!(
|
|
remaining_editor_agent_prompt_duration(Duration::from_secs(18 * 60)),
|
|
Duration::ZERO
|
|
);
|
|
assert_eq!(
|
|
error.to_string(),
|
|
"美术 Agent 规划失败：规划总时长已达到 18 分钟安全上限"
|
|
);
|
|
}
|
|
}
|
|
fn editor_agent_system_prompt() -> &'static str {
|
|
r#"
|
|
你是 Genarrative 图片画布 Agent，只负责帮助用户理解、规划和触发画布生成工具。
|
|
对话回复要简短，不能承诺免费生成；实际生成工具由后端按模型定价扣泥点。
|
|
"#
|
|
}
|
|
|
|
fn build_delta_messages(
|
|
result: Result<Vec<PromptOutput>, PromptError>,
|
|
created_at: &str,
|
|
messages_offset: usize,
|
|
document: &EditorAgentConversationMessagesDocument,
|
|
tool_context: &EditorToolContext,
|
|
pricing: &EditorGenerationPricingConfig,
|
|
) -> Result<Vec<EditorAgentMessage>, PromptError> {
|
|
let outputs = result?;
|
|
let mut messages = Vec::with_capacity(outputs.len());
|
|
|
|
for (i, out) in outputs.into_iter().enumerate() {
|
|
let absolute_idx = messages_offset + i;
|
|
match out {
|
|
PromptOutput::Text(text) => {
|
|
messages.push(EditorAgentMessage {
|
|
id: absolute_idx,
|
|
client_message_id: None,
|
|
role: EditorAgentMessageRole::Assistant,
|
|
text,
|
|
attachments: Vec::new(),
|
|
tool_call: None,
|
|
created_at: created_at.to_string(),
|
|
});
|
|
}
|
|
PromptOutput::Tool(tco) => {
|
|
let tool_name = tco.tool_call.name;
|
|
let normalized_args = tool_args::normalize_tool_args(
|
|
tool_name.as_str(),
|
|
&tco.tool_call.args,
|
|
tool_context,
|
|
)?;
|
|
let display_args = display_args::build_tool_call_display_args(
|
|
tool_name.as_str(),
|
|
&normalized_args,
|
|
document,
|
|
tool_context,
|
|
pricing,
|
|
)?;
|
|
let text =
|
|
format!("[tool_call:{tool_name}] args: {normalized_args} output: 等待用户确认");
|
|
messages.push(EditorAgentMessage {
|
|
id: absolute_idx,
|
|
client_message_id: None,
|
|
role: EditorAgentMessageRole::System,
|
|
text,
|
|
attachments: Vec::new(),
|
|
tool_call: Some(EditorAgentToolCall {
|
|
tool_name,
|
|
status: EditorAgentToolCallStatus::NotCompleted,
|
|
args: normalized_args,
|
|
display_args,
|
|
external_job_id: None,
|
|
images: Vec::new(),
|
|
videos: Vec::new(),
|
|
audios: Vec::new(),
|
|
error: None,
|
|
}),
|
|
created_at: created_at.to_string(),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(messages)
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
struct EditorAgentConversationDeleteResponse {
|
|
deleted_conversation_id: String,
|
|
conversation: EditorAgentConversationSummary,
|
|
}
|
|
|
|
pub async fn list_editor_agent_conversations(
|
|
State(state): State<AppState>,
|
|
Path(project_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
ensure_editor_project_access(&state, project_id.as_str(), owner_user_id.as_str()).await?;
|
|
let conversations = state
|
|
.spacetime_client()
|
|
.list_editor_agent_conversations(project_id, owner_user_id)
|
|
.await
|
|
.map_err(map_editor_project_error)?
|
|
.into_iter()
|
|
.map(conversation_summary_from_record)
|
|
.collect();
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationListResponse { conversations },
|
|
))
|
|
}
|
|
|
|
pub async fn create_editor_agent_conversation(
|
|
State(state): State<AppState>,
|
|
Path(project_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
Json(payload): Json<CreateEditorAgentConversationRequest>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
ensure_editor_project_access(&state, project_id.as_str(), owner_user_id.as_str()).await?;
|
|
|
|
let conversation_id = build_prefixed_uuid_id(EDITOR_AGENT_CONVERSATION_ID_PREFIX);
|
|
let messages_object_key = editor_agent_messages_object_key(conversation_id.as_str());
|
|
let title = normalize_optional_string(payload.title)
|
|
.unwrap_or_else(|| EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE.to_string());
|
|
let now_micros = current_utc_micros();
|
|
let seed_record = EditorAgentConversationRecord {
|
|
conversation_id: conversation_id.clone(),
|
|
project_id: project_id.clone(),
|
|
owner_user_id: owner_user_id.clone(),
|
|
title: title.clone(),
|
|
messages_object_key: messages_object_key.clone(),
|
|
deleted: false,
|
|
created_at: now_rfc3339(),
|
|
updated_at: now_rfc3339(),
|
|
updated_at_micros: now_micros,
|
|
};
|
|
write_messages_document(
|
|
&state,
|
|
&seed_record,
|
|
&empty_messages_document(conversation_id.as_str()),
|
|
)
|
|
.await?;
|
|
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.create_editor_agent_conversation(EditorAgentConversationCreateRecordInput {
|
|
conversation_id,
|
|
project_id,
|
|
owner_user_id,
|
|
title,
|
|
messages_object_key,
|
|
created_at_micros: now_micros,
|
|
})
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
let document = read_messages_document(&state, &conversation).await?;
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationResponse {
|
|
conversation: conversation_detail_from_record(conversation, document.messages),
|
|
},
|
|
))
|
|
}
|
|
|
|
pub async fn get_editor_agent_conversation(
|
|
State(state): State<AppState>,
|
|
Path(conversation_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id, owner_user_id)
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
|
|
conversation.conversation_id.as_str(),
|
|
);
|
|
let _conversation_lock_guard = conversation_lock.lock_owned().await;
|
|
let mut document = read_messages_document(&state, &conversation).await?;
|
|
let reconciled_messages =
|
|
reconcile::reconcile_editor_agent_tool_calls(&state, &conversation, &mut document).await?;
|
|
if !reconciled_messages.is_empty() {
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
}
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationResponse {
|
|
conversation: conversation_detail_from_record(conversation, document.messages),
|
|
},
|
|
))
|
|
}
|
|
|
|
pub async fn delete_editor_agent_conversation(
|
|
State(state): State<AppState>,
|
|
Path(conversation_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.delete_editor_agent_conversation(EditorAgentConversationDeleteRecordInput {
|
|
conversation_id,
|
|
owner_user_id,
|
|
updated_at_micros: current_utc_micros(),
|
|
})
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationDeleteResponse {
|
|
deleted_conversation_id: conversation.conversation_id.clone(),
|
|
conversation: conversation_summary_from_record(conversation),
|
|
},
|
|
))
|
|
}
|
|
|
|
pub async fn cancel_editor_agent_tool_call(
|
|
State(state): State<AppState>,
|
|
Path((conversation_id, message_id)): Path<(String, usize)>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id, owner_user_id)
|
|
.await
|
|
.map_err(|e| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": format!("conversation not found: {e}") }))
|
|
})?;
|
|
|
|
let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
|
|
conversation.conversation_id.as_str(),
|
|
);
|
|
let _conversation_lock_guard = conversation_lock.lock_owned().await;
|
|
|
|
let mut document: EditorAgentConversationMessagesDocument =
|
|
read_messages_document(&state, &conversation).await?;
|
|
|
|
// Validate message index
|
|
if message_id >= document.messages.len() {
|
|
return Err(AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": "message not found" })));
|
|
}
|
|
|
|
let msg = &mut document.messages[message_id];
|
|
|
|
// Validate role and tool_call
|
|
if msg.role != EditorAgentMessageRole::System {
|
|
return Err(editor_agent_bad_request("message is not a system message"));
|
|
}
|
|
let tc = msg
|
|
.tool_call
|
|
.as_mut()
|
|
.ok_or_else(|| editor_agent_bad_request("message has no tool call"))?;
|
|
if tc.status != EditorAgentToolCallStatus::NotCompleted || tc.external_job_id.is_some() {
|
|
return Err(editor_agent_bad_request(
|
|
"tool call is no longer pending confirmation",
|
|
));
|
|
}
|
|
|
|
tc.status = EditorAgentToolCallStatus::Cancelled;
|
|
let arg_json = tc.args.to_string();
|
|
msg.text = format!(
|
|
"[tool_call:{tool_name}] args: {arg_json} output: 用户已取消该操作",
|
|
tool_name = tc.tool_name,
|
|
arg_json = arg_json,
|
|
);
|
|
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
json!({ "ok": true }),
|
|
))
|
|
}
|
|
|
|
pub async fn confirm_editor_agent_tool_call(
|
|
State(state): State<AppState>,
|
|
Path((conversation_id, message_id)): Path<(String, usize)>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id, owner_user_id)
|
|
.await
|
|
.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND).with_details(json!({
|
|
"message": format!("conversation not found: {error}"),
|
|
}))
|
|
})?;
|
|
let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
|
|
conversation.conversation_id.as_str(),
|
|
);
|
|
let _conversation_lock_guard = conversation_lock.lock_owned().await;
|
|
let mut document = read_messages_document(&state, &conversation).await?;
|
|
let message = document
|
|
.messages
|
|
.get(message_id)
|
|
.ok_or_else(|| AppError::from_status(axum::http::StatusCode::NOT_FOUND))?;
|
|
if message.role != EditorAgentMessageRole::System {
|
|
return Err(editor_agent_bad_request("message is not a system message"));
|
|
}
|
|
let tool_call = message
|
|
.tool_call
|
|
.as_ref()
|
|
.ok_or_else(|| editor_agent_bad_request("message has no tool call"))?;
|
|
if tool_call.status == EditorAgentToolCallStatus::Cancelled {
|
|
return Err(editor_agent_bad_request("tool call was cancelled"));
|
|
}
|
|
if tool_call.status != EditorAgentToolCallStatus::NotCompleted
|
|
|| tool_call.external_job_id.is_some()
|
|
{
|
|
return Ok(json_success_body(
|
|
Some(&request_context),
|
|
json!({ "ok": true }),
|
|
));
|
|
}
|
|
|
|
let tool_name = tool_call.tool_name.clone();
|
|
let tool_args = tool_call.args.clone();
|
|
let pricing = state.editor_generation_pricing().await.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({
|
|
"provider": "editor-generation-pricing",
|
|
"message": error.to_string(),
|
|
}))
|
|
})?;
|
|
let project = load_editor_agent_project(&state, &conversation).await?;
|
|
let context = context::build_tool_context(&document);
|
|
let generation_inputs = |title: &str, value: &str| {
|
|
Some(json!({
|
|
// TODO extract const
|
|
"source": "editor-agent",
|
|
"conversationId": conversation.conversation_id,
|
|
"toolCallMessageId": message_id,
|
|
"fields": [{ "title": title, "value": value }],
|
|
}))
|
|
};
|
|
|
|
let (job_kind, request_label, price_mud_points, payload) = match tool_name.as_str() {
|
|
GenerateImageTool::NAME => {
|
|
let args: GenerateImageToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let tool = GenerateImageTool {
|
|
context: context.clone(),
|
|
};
|
|
tool.validate_args(&args).map_err(|error| {
|
|
editor_agent_bad_request(format!("invalid tool call args: {error}"))
|
|
})?;
|
|
let price = tool.pricing(&pricing, &args);
|
|
let title = args.prompt.clone();
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let payload = EditorImageGenerationRequest {
|
|
prompt: args.prompt,
|
|
size: None,
|
|
kind: None,
|
|
model: Some(args.model),
|
|
screen_color: None,
|
|
seg_model: None,
|
|
aspect_ratio: args.aspect_ratio,
|
|
image_size: args.image_size,
|
|
reference_image_srcs: Some(reference_image_srcs),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
asset_kind: Some("editor_agent_generated_image".to_string()),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title.clone()),
|
|
source_resource_id: None,
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project, &tool_name, &title,
|
|
)),
|
|
};
|
|
(
|
|
EDITOR_IMAGE_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成图片",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateCharacterTool::NAME => {
|
|
let args: GenerateCharacterToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let tool = GenerateCharacterTool {
|
|
context: context.clone(),
|
|
};
|
|
tool.validate_args(&args).map_err(|error| {
|
|
editor_agent_bad_request(format!("invalid tool call args: {error}"))
|
|
})?;
|
|
let price = tool.pricing(&pricing, &args);
|
|
let title = args.prompt.clone();
|
|
let reference_image_srcs =
|
|
resolve_editor_agent_image_ids(args.reference_image_ids.as_slice(), &context)?;
|
|
let payload = EditorImageGenerationRequest {
|
|
prompt: args.prompt,
|
|
size: None,
|
|
kind: Some("character".to_string()),
|
|
model: Some(args.model),
|
|
screen_color: Some("auto".to_string()),
|
|
seg_model: Some("birefnet".to_string()),
|
|
aspect_ratio: args.aspect_ratio,
|
|
image_size: args.image_size,
|
|
reference_image_srcs: Some(reference_image_srcs),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
asset_kind: Some("character".to_string()),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title.clone()),
|
|
source_resource_id: None,
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project, &tool_name, &title,
|
|
)),
|
|
};
|
|
(
|
|
EDITOR_IMAGE_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成图片",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateUiDesignTool::NAME => {
|
|
let args: GenerateUiDesignToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let tool = GenerateUiDesignTool {
|
|
context: context.clone(),
|
|
};
|
|
tool.validate_args(&args).map_err(|error| {
|
|
editor_agent_bad_request(format!("invalid tool call args: {error}"))
|
|
})?;
|
|
let price = tool.pricing(&pricing, &args);
|
|
let title = args.prompt.clone();
|
|
let reference_image_srcs =
|
|
resolve_editor_agent_image_ids(args.reference_image_ids.as_slice(), &context)?;
|
|
let payload = EditorImageGenerationRequest {
|
|
prompt: args.prompt,
|
|
size: None,
|
|
kind: Some("ui-design".to_string()),
|
|
model: Some(args.model),
|
|
screen_color: None,
|
|
seg_model: None,
|
|
aspect_ratio: args.aspect_ratio,
|
|
image_size: args.image_size,
|
|
reference_image_srcs: Some(reference_image_srcs),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
asset_kind: Some("ui-design".to_string()),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title.clone()),
|
|
source_resource_id: None,
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project, &tool_name, &title,
|
|
)),
|
|
};
|
|
(
|
|
EDITOR_IMAGE_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成图片",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
EditImageTool::NAME => {
|
|
let args: EditImageToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let tool = EditImageTool {
|
|
context: context.clone(),
|
|
};
|
|
if let Some(error) = tool.validate_args(&args) {
|
|
return Err(editor_agent_bad_request(format!(
|
|
"invalid tool call args: {error}"
|
|
)));
|
|
}
|
|
let source_image_src = context
|
|
.image_data_key(&args.object_image_id)
|
|
.ok_or_else(|| editor_agent_bad_request("object image not found"))?
|
|
.to_string();
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let title = args.prompt.clone();
|
|
let price = tool.pricing(&pricing, &args);
|
|
let payload = EditorImageEditRequest {
|
|
prompt: args.prompt,
|
|
source_image_src,
|
|
size: None,
|
|
model: Some(args.model),
|
|
aspect_ratio: None,
|
|
image_size: None,
|
|
reference_image_srcs: Some(reference_image_srcs),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
asset_kind: Some("editor_agent_edit_image".to_string()),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title.clone()),
|
|
source_resource_id: None,
|
|
target_layer_id: None,
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
EditImageTool::NAME,
|
|
&title,
|
|
)),
|
|
};
|
|
(
|
|
EDITOR_IMAGE_EDIT_JOB_KIND,
|
|
"画布 Agent 修改图片",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateIconSpritesheetTool::NAME => {
|
|
let args: GenerateIconSpritesheetToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let tool = GenerateIconSpritesheetTool {
|
|
context: context.clone(),
|
|
};
|
|
tool.validate_args(&args).map_err(|error| {
|
|
editor_agent_bad_request(format!("invalid tool call args: {error}"))
|
|
})?;
|
|
let price = tool.pricing(&pricing, &args);
|
|
let reference_image_src = context
|
|
.image_data_key(&args.reference_image_id)
|
|
.ok_or_else(|| editor_agent_bad_request("reference image not found"))?
|
|
.to_string();
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let payload = EditorIconSpritesheetGenerationRequest {
|
|
reference_image_src,
|
|
reference_image_srcs: Some(reference_image_srcs),
|
|
icon_descriptions: args.icon_descriptions.clone(),
|
|
model: Some(args.model),
|
|
screen_color: Some("auto".to_string()),
|
|
seg_model: Some("birefnet".to_string()),
|
|
aspect_ratio: args.aspect_ratio,
|
|
image_size: args.image_size,
|
|
project_id: Some(conversation.project_id.clone()),
|
|
generation_inputs: Some(
|
|
json!({"source":"editor-agent","conversationId":conversation.conversation_id,"toolCallMessageId":message_id}),
|
|
),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some("图标素材图集".to_string()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateIconSpritesheetTool::NAME,
|
|
"图标素材图集",
|
|
)),
|
|
};
|
|
(
|
|
EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成图标素材",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateVideoTool::NAME => {
|
|
let args: GenerateVideoToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let tool = GenerateVideoTool {
|
|
context: context.clone(),
|
|
};
|
|
tool.validate_args(&args).map_err(|error| {
|
|
editor_agent_bad_request(format!("invalid tool call args: {error}"))
|
|
})?;
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let title = args.prompt.clone();
|
|
let price = tool.pricing(&pricing, &args);
|
|
let payload = EditorVideoGenerateRequest {
|
|
prompt: args.prompt,
|
|
model: args.model,
|
|
aspect_ratio: args
|
|
.aspect_ratio
|
|
.unwrap_or_else(|| GenerateVideoTool::DEFAULT_VIDEO_ASPECT_RATIO.to_string()),
|
|
duration_seconds: args
|
|
.duration_seconds
|
|
.unwrap_or(GenerateVideoTool::DEFAULT_VIDEO_DURATION_SECONDS),
|
|
resolution: args
|
|
.resolution
|
|
.unwrap_or_else(|| GenerateVideoTool::DEFAULT_VIDEO_RESOLUTION.to_string()),
|
|
mode: "std".to_string(),
|
|
sound: args
|
|
.sound
|
|
.unwrap_or_else(|| GenerateVideoTool::DEFAULT_VIDEO_SOUND.to_string()),
|
|
web_search_enabled: false,
|
|
reference_image_srcs,
|
|
reference_video_srcs: Vec::new(),
|
|
reference_audio_srcs: Vec::new(),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateVideoTool::NAME,
|
|
&title,
|
|
)),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
source_resource_id: None,
|
|
asset_kind: Some("video".to_string()),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title),
|
|
};
|
|
(
|
|
EDITOR_VIDEO_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成视频",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateSoundEffectTool::NAME => {
|
|
let args: GenerateSoundEffectToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
GenerateSoundEffectTool
|
|
.validate_args(&args)
|
|
.map_err(|error| {
|
|
editor_agent_bad_request(format!("invalid tool call args: {error}"))
|
|
})?;
|
|
let price = GenerateSoundEffectTool.pricing(&pricing, &args);
|
|
let title = args.prompt.clone();
|
|
let payload = EditorSoundEffectGenerateRequest {
|
|
prompt: args.prompt,
|
|
model: Some(args.model),
|
|
duration: args
|
|
.duration
|
|
.unwrap_or(GenerateSoundEffectTool::DEFAULT_DURATION),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateSoundEffectTool::NAME,
|
|
&title,
|
|
)),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title),
|
|
};
|
|
(
|
|
EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成音效",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateBackgroundMusicTool::NAME => {
|
|
let args: GenerateBackgroundMusicToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
GenerateBackgroundMusicTool
|
|
.validate_args(&args)
|
|
.map_err(|error| {
|
|
editor_agent_bad_request(format!("invalid tool call args: {error}"))
|
|
})?;
|
|
let price = GenerateBackgroundMusicTool.pricing(&pricing, &args);
|
|
let title = args.prompt.clone();
|
|
let payload = EditorBackgroundMusicGenerateRequest {
|
|
gpt_description_prompt: args.prompt,
|
|
make_instrumental: true,
|
|
project_id: Some(conversation.project_id.clone()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateBackgroundMusicTool::NAME,
|
|
"生成背景音乐",
|
|
)),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some("生成背景音乐".to_string()),
|
|
};
|
|
(
|
|
EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成背景音乐",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
_ => {
|
|
return Err(editor_agent_bad_request(format!(
|
|
"unsupported tool: {tool_name}"
|
|
)));
|
|
}
|
|
};
|
|
let payload = payload.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR)
|
|
.with_details(json!({"message": format!("failed to serialize tool payload: {error}")}))
|
|
})?;
|
|
let (job_id, dedupe_key) = editor_agent_tool_job_identity(
|
|
conversation.conversation_id.as_str(),
|
|
message_id,
|
|
tool_name.as_str(),
|
|
);
|
|
let job = enqueue_editor_generation_job_with_identity(
|
|
&state,
|
|
conversation.owner_user_id.as_str(),
|
|
job_kind,
|
|
conversation.project_id.clone(),
|
|
request_label,
|
|
u64::from(price_mud_points),
|
|
&payload,
|
|
job_id,
|
|
dedupe_key,
|
|
)
|
|
.await?;
|
|
let message = &mut document.messages[message_id];
|
|
let tool_call = message
|
|
.tool_call
|
|
.as_mut()
|
|
.ok_or_else(|| editor_agent_bad_request("message has no tool call"))?;
|
|
tool_call.external_job_id = Some(job.job_id);
|
|
tool_call.status = EditorAgentToolCallStatus::NotCompleted;
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
json!({ "ok": true }),
|
|
))
|
|
}
|
|
|
|
fn parse_confirm_tool_args<T: serde::de::DeserializeOwned>(value: &Value) -> Result<T, AppError> {
|
|
serde_json::from_value(value.clone())
|
|
.map_err(|error| editor_agent_bad_request(format!("invalid tool call args: {error}")))
|
|
}
|
|
|
|
fn resolve_editor_agent_image_ids(
|
|
image_ids: &[platform_editor_agent::agent::asset::ImageId],
|
|
context: &EditorToolContext,
|
|
) -> Result<Vec<String>, AppError> {
|
|
image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| editor_agent_bad_request(format!("image {image_id} not found")))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
fn editor_agent_tool_job_identity(
|
|
conversation_id: &str,
|
|
message_id: usize,
|
|
tool_name: &str,
|
|
) -> (String, String) {
|
|
let dedupe_key = format!("editor-agent:{conversation_id}:{message_id}:{tool_name}");
|
|
let digest = Sha256::digest(dedupe_key.as_bytes());
|
|
(format!("task-editor-agent-{digest:x}"), dedupe_key)
|
|
}
|
|
|
|
async fn load_editor_agent_project(
|
|
state: &AppState,
|
|
conversation: &EditorAgentConversationRecord,
|
|
) -> Result<spacetime_client::EditorProjectRecord, AppError> {
|
|
state
|
|
.spacetime_client()
|
|
.get_editor_project(EditorProjectGetRecordInput {
|
|
project_id: conversation.project_id.clone(),
|
|
owner_user_id: conversation.owner_user_id.clone(),
|
|
})
|
|
.await
|
|
.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": format!("project not found: {error}") }))
|
|
})
|
|
}
|