k88936
f774a791e8
工具消息改为保存外部任务 ID 与取消事实，不再复制执行状态。 确认请求以稳定身份提交外部任务，并由前端查询任务状态和结果。 同步更新画布 Agent 测试与执行契约文档。
1396 lines
54 KiB
Rust
1396 lines
54 KiB
Rust
use std::collections::HashMap;
|
|
|
|
use axum::extract::{Path, State};
|
|
use axum::{Extension, Json};
|
|
use module_editor_agent::agent::agent_builder::AgentBuilder;
|
|
use module_editor_agent::agent::error::PromptError;
|
|
use module_editor_agent::agent::memory::VecMemory;
|
|
use module_editor_agent::agent::run::PromptOutput;
|
|
use module_editor_agent::agent::tool::Tool;
|
|
use module_editor_agent::{
|
|
EDITOR_AGENT_CONVERSATION_ID_PREFIX, EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE,
|
|
derive_conversation_title, editor_agent_messages_object_key,
|
|
};
|
|
use platform_llm::LlmMessage;
|
|
use serde::Serialize;
|
|
use serde_json::{Value, json};
|
|
use sha2::{Digest, Sha256};
|
|
use shared_contracts::assets::{
|
|
EditorBackgroundMusicGenerateRequest, EditorSoundEffectGenerateRequest,
|
|
EditorVideoGenerateRequest,
|
|
};
|
|
use shared_contracts::editor_agent::{
|
|
CreateEditorAgentConversationRequest, EditorAgentConversationListResponse,
|
|
EditorAgentConversationMessagesDocument, EditorAgentConversationResponse,
|
|
EditorAgentConversationSummary, EditorAgentMessage, EditorAgentMessageRequest,
|
|
EditorAgentMessageResponse, EditorAgentMessageRole, EditorAgentToolCall,
|
|
EditorAgentToolCallDisplayArgs, EditorAgentToolCallDisplayExtras, EditorAgentToolCallImageArg,
|
|
EditorAgentToolCallImageRef, EditorAgentToolCallStringArg,
|
|
};
|
|
use spacetime_client::{
|
|
EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput,
|
|
EditorAgentConversationRecord, EditorAgentConversationTouchRecordInput,
|
|
EditorProjectGetRecordInput,
|
|
};
|
|
|
|
use crate::api_response::json_success_body;
|
|
use crate::auth::AuthenticatedAccessToken;
|
|
use crate::editor_agent::agent::LlmChatAgentBuilder;
|
|
use crate::editor_agent::editor_tools::common::{EditorAgentPricedTool, EditorToolContext};
|
|
use crate::editor_agent::editor_tools::edit_image::{EditImageTool, EditImageToolArgs};
|
|
use crate::editor_agent::editor_tools::generate_background_music::{
|
|
GenerateBackgroundMusicTool, GenerateBackgroundMusicToolArgs,
|
|
};
|
|
use crate::editor_agent::editor_tools::generate_character::GenerateCharacterTool;
|
|
use crate::editor_agent::editor_tools::generate_icon_spritesheet::{
|
|
GenerateIconSpritesheetTool, GenerateIconSpritesheetToolArgs,
|
|
};
|
|
use crate::editor_agent::editor_tools::generate_image::{GenerateImageTool, GenerateImageToolArgs};
|
|
use crate::editor_agent::editor_tools::generate_sound_effect::{
|
|
GenerateSoundEffectTool, GenerateSoundEffectToolArgs,
|
|
};
|
|
use crate::editor_agent::editor_tools::generate_ui_design::GenerateUiDesignTool;
|
|
use crate::editor_agent::editor_tools::generate_video::{GenerateVideoTool, GenerateVideoToolArgs};
|
|
use crate::editor_agent::utils::{
|
|
ImageId, ImageMetadata, IntoDataKey, IntoImageId, build_editor_agent_canvas_completion,
|
|
conversation_detail_from_record, conversation_summary_from_record, editor_agent_bad_request,
|
|
empty_messages_document, ensure_editor_project_access, normalize_editor_agent_attachments,
|
|
now_rfc3339, read_messages_document, require_editor_agent_sidebar_enabled,
|
|
write_messages_document,
|
|
};
|
|
use crate::editor_generation_config::EditorGenerationPricingConfig;
|
|
use crate::editor_generation_queue::{
|
|
EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND, EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND,
|
|
EDITOR_IMAGE_EDIT_JOB_KIND, EDITOR_IMAGE_GENERATION_JOB_KIND,
|
|
EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND, EDITOR_VIDEO_GENERATION_JOB_KIND,
|
|
enqueue_editor_generation_job_with_identity,
|
|
};
|
|
use crate::editor_project::{
|
|
EditorIconSpritesheetGenerationRequest, EditorImageEditRequest, EditorImageGenerationRequest,
|
|
};
|
|
use crate::editor_project::{current_utc_micros, map_editor_project_error};
|
|
use crate::http_error::AppError;
|
|
use crate::request_context::RequestContext;
|
|
use crate::state::AppState;
|
|
use shared_kernel::{build_prefixed_uuid_id, normalize_optional_string};
|
|
|
|
pub async fn editor_agent_message(
|
|
State(state): State<AppState>,
|
|
Path(conversation_id): Path<String>,
|
|
Extension(_request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
Json(payload): Json<EditorAgentMessageRequest>,
|
|
) -> Result<Json<EditorAgentMessageResponse>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
// Load conversation & attachments
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id.clone(), owner_user_id.clone())
|
|
.await
|
|
.map_err(|e| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": format!("conversation not found: {e}") }))
|
|
})?;
|
|
|
|
let attachments =
|
|
normalize_editor_agent_attachments(&state, &conversation, payload.attachments.as_slice())
|
|
.await?;
|
|
|
|
let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
|
|
conversation.conversation_id.as_str(),
|
|
);
|
|
let _conversation_lock_guard = conversation_lock.lock_owned().await;
|
|
|
|
let mut document: EditorAgentConversationMessagesDocument =
|
|
read_messages_document(&state, &conversation).await?;
|
|
|
|
// Determine initialization before attachment bookkeeping adds a system message.
|
|
let was_empty = document.messages.is_empty();
|
|
let now = now_rfc3339();
|
|
if !attachments.is_empty() {
|
|
let mut attachment_info = String::new();
|
|
attachment_info.push_str("user has just uploaded attachments of the order: ");
|
|
for a in &attachments {
|
|
attachment_info.push_str(&format!("{} ,", a.clone().into_image_id()))
|
|
}
|
|
document.messages.push(EditorAgentMessage {
|
|
id: 0,
|
|
role: EditorAgentMessageRole::System,
|
|
text: attachment_info,
|
|
attachments: Vec::new(),
|
|
tool_call: None,
|
|
created_at: now.clone(),
|
|
});
|
|
}
|
|
|
|
// Build conversation history as LlmMessage vec
|
|
let previous_messages: Vec<LlmMessage> = document
|
|
.messages
|
|
.iter()
|
|
.map(|msg| match msg.role {
|
|
EditorAgentMessageRole::User => LlmMessage::user(&msg.text),
|
|
EditorAgentMessageRole::Assistant => LlmMessage::assistant(&msg.text),
|
|
EditorAgentMessageRole::System => LlmMessage::system(&msg.text),
|
|
})
|
|
.collect();
|
|
|
|
// Save user message to document.
|
|
let user_message = EditorAgentMessage {
|
|
id: document.messages.len(),
|
|
role: EditorAgentMessageRole::User,
|
|
text: payload.text.trim().to_string(),
|
|
attachments,
|
|
tool_call: None,
|
|
created_at: now,
|
|
};
|
|
document.messages.push(user_message.clone());
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
|
|
// Persist and return the authoritative summary for every turn. Initialization sets the
|
|
// title from the first user prompt; a metadata write failure must fail the request.
|
|
let updated_conversation = state
|
|
.spacetime_client()
|
|
.touch_editor_agent_conversation(EditorAgentConversationTouchRecordInput {
|
|
conversation_id: conversation.conversation_id.clone(),
|
|
owner_user_id: conversation.owner_user_id.clone(),
|
|
title: was_empty.then(|| derive_conversation_title(user_message.text.as_str())),
|
|
updated_at_micros: current_utc_micros(),
|
|
})
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
let conversation_summary = conversation_summary_from_record(updated_conversation);
|
|
|
|
// Build tool context from document
|
|
let tool_context = build_tool_context(&document);
|
|
|
|
// Build and run agent
|
|
let llm_client = state.llm_client().ok_or_else(|| {
|
|
AppError::from_status(axum::http::StatusCode::SERVICE_UNAVAILABLE)
|
|
.with_details(json!({ "message": "LLM client not configured" }))
|
|
})?;
|
|
let llm_client = llm_client.clone();
|
|
let pricing = state.editor_generation_pricing().await.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({
|
|
"provider": "editor-generation-pricing",
|
|
"message": error.to_string(),
|
|
}))
|
|
})?;
|
|
|
|
let memory = VecMemory::new(previous_messages);
|
|
|
|
let mut agent = LlmChatAgentBuilder::new()
|
|
.with_client(llm_client)
|
|
.system_prompt(editor_agent_system_prompt())
|
|
.tool(EditImageTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateImageTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateCharacterTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateIconSpritesheetTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateSoundEffectTool)
|
|
.tool(GenerateBackgroundMusicTool)
|
|
.tool(GenerateVideoTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.tool(GenerateUiDesignTool {
|
|
context: tool_context.clone(),
|
|
})
|
|
.max_turns(3)
|
|
.memory(memory)
|
|
.build();
|
|
|
|
let agent_result = agent.prompt(LlmMessage::user(user_message.text)).await;
|
|
|
|
let assistant_now = now_rfc3339();
|
|
|
|
match build_delta_messages(
|
|
agent_result,
|
|
&assistant_now,
|
|
document.messages.len(),
|
|
&document,
|
|
&tool_context,
|
|
&pricing,
|
|
) {
|
|
Err(err) => Ok(Json(EditorAgentMessageResponse {
|
|
conversation: conversation_summary,
|
|
delta_messages: vec![],
|
|
error_message: Some(err.to_string()),
|
|
})),
|
|
Ok(delta_messages) => {
|
|
for msg in &delta_messages {
|
|
document.messages.push(msg.clone());
|
|
}
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
|
|
Ok(Json(EditorAgentMessageResponse {
|
|
conversation: conversation_summary,
|
|
delta_messages,
|
|
error_message: None,
|
|
}))
|
|
}
|
|
}
|
|
}
|
|
fn editor_agent_system_prompt() -> &'static str {
|
|
r#"
|
|
你是 Genarrative 图片画布 Agent，只负责帮助用户理解、规划和触发画布生成工具。
|
|
对话回复要简短，不能承诺免费生成；实际生成工具由后端按模型定价扣泥点。
|
|
"#
|
|
}
|
|
|
|
fn build_delta_messages(
|
|
result: Result<Vec<PromptOutput>, PromptError>,
|
|
created_at: &str,
|
|
messages_offset: usize,
|
|
document: &EditorAgentConversationMessagesDocument,
|
|
tool_context: &EditorToolContext,
|
|
pricing: &EditorGenerationPricingConfig,
|
|
) -> Result<Vec<EditorAgentMessage>, PromptError> {
|
|
let outputs = result?;
|
|
let mut messages = Vec::with_capacity(outputs.len());
|
|
|
|
for (i, out) in outputs.into_iter().enumerate() {
|
|
let absolute_idx = messages_offset + i;
|
|
match out {
|
|
PromptOutput::Text(text) => {
|
|
messages.push(EditorAgentMessage {
|
|
id: absolute_idx,
|
|
role: EditorAgentMessageRole::Assistant,
|
|
text,
|
|
attachments: Vec::new(),
|
|
tool_call: None,
|
|
created_at: created_at.to_string(),
|
|
});
|
|
}
|
|
PromptOutput::Tool(tco) => {
|
|
let display_args = build_tool_call_display_args(
|
|
tco.tool_call.name.as_str(),
|
|
&tco.tool_call.args,
|
|
document,
|
|
tool_context,
|
|
pricing,
|
|
)?;
|
|
messages.push(EditorAgentMessage {
|
|
id: absolute_idx,
|
|
role: EditorAgentMessageRole::System,
|
|
text: tco.message,
|
|
attachments: Vec::new(),
|
|
tool_call: Some(EditorAgentToolCall {
|
|
tool_name: tco.tool_call.name,
|
|
args: tco.tool_call.args,
|
|
display_args,
|
|
external_job_id: None,
|
|
cancelled_at: None,
|
|
images: Vec::new(),
|
|
videos: Vec::new(),
|
|
audios: Vec::new(),
|
|
error: None,
|
|
}),
|
|
created_at: created_at.to_string(),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(messages)
|
|
}
|
|
|
|
fn build_tool_call_display_args(
|
|
tool_name: &str,
|
|
args: &Value,
|
|
document: &EditorAgentConversationMessagesDocument,
|
|
tool_context: &EditorToolContext,
|
|
pricing: &EditorGenerationPricingConfig,
|
|
) -> Result<EditorAgentToolCallDisplayArgs, PromptError> {
|
|
let mut display_args = EditorAgentToolCallDisplayArgs::default();
|
|
|
|
let price_mud_points = match tool_name {
|
|
EditImageTool::NAME => {
|
|
let args: EditImageToolArgs = parse_display_tool_args(tool_name, args)?;
|
|
let price_mud_points = EditImageTool {
|
|
context: tool_context.clone(),
|
|
}
|
|
.pricing(pricing, &args);
|
|
push_string_display_arg(&mut display_args, "prompt", "修改要求", args.prompt);
|
|
push_image_display_arg(
|
|
&mut display_args,
|
|
document,
|
|
"object_image_id",
|
|
"目标图片",
|
|
std::slice::from_ref(&args.object_image_id),
|
|
)?;
|
|
push_image_display_arg(
|
|
&mut display_args,
|
|
document,
|
|
"reference_image_ids",
|
|
"参考图片",
|
|
args.reference_image_ids.as_slice(),
|
|
)?;
|
|
price_mud_points
|
|
}
|
|
GenerateImageTool::NAME | GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => {
|
|
let args: GenerateImageToolArgs = parse_display_tool_args(tool_name, args)?;
|
|
let price_mud_points = match tool_name {
|
|
GenerateImageTool::NAME => GenerateImageTool {
|
|
context: tool_context.clone(),
|
|
}
|
|
.pricing(pricing, &args),
|
|
GenerateCharacterTool::NAME => GenerateCharacterTool {
|
|
context: tool_context.clone(),
|
|
}
|
|
.pricing(pricing, &args),
|
|
GenerateUiDesignTool::NAME => GenerateUiDesignTool {
|
|
context: tool_context.clone(),
|
|
}
|
|
.pricing(pricing, &args),
|
|
_ => unreachable!("tool name was matched above"),
|
|
};
|
|
push_string_display_arg(&mut display_args, "prompt", "提示词", args.prompt);
|
|
push_optional_string_display_arg(
|
|
&mut display_args,
|
|
"aspect_ratio",
|
|
"画面比例",
|
|
args.aspect_ratio,
|
|
);
|
|
push_optional_string_display_arg(
|
|
&mut display_args,
|
|
"image_size",
|
|
"图片尺寸",
|
|
args.image_size,
|
|
);
|
|
push_image_display_arg(
|
|
&mut display_args,
|
|
document,
|
|
"reference_image_ids",
|
|
"参考图片",
|
|
args.reference_image_ids.as_slice(),
|
|
)?;
|
|
price_mud_points
|
|
}
|
|
GenerateIconSpritesheetTool::NAME => {
|
|
let args: GenerateIconSpritesheetToolArgs = parse_display_tool_args(tool_name, args)?;
|
|
let price_mud_points = GenerateIconSpritesheetTool {
|
|
context: tool_context.clone(),
|
|
}
|
|
.pricing(pricing, &args);
|
|
push_string_display_arg(
|
|
&mut display_args,
|
|
"icon_descriptions",
|
|
"图标描述",
|
|
args.icon_descriptions.join("\n"),
|
|
);
|
|
push_optional_string_display_arg(
|
|
&mut display_args,
|
|
"aspect_ratio",
|
|
"图集比例",
|
|
args.aspect_ratio,
|
|
);
|
|
push_optional_string_display_arg(
|
|
&mut display_args,
|
|
"image_size",
|
|
"图片尺寸",
|
|
args.image_size,
|
|
);
|
|
push_image_display_arg(
|
|
&mut display_args,
|
|
document,
|
|
"reference_image_id",
|
|
"规范参考图",
|
|
std::slice::from_ref(&args.reference_image_id),
|
|
)?;
|
|
push_image_display_arg(
|
|
&mut display_args,
|
|
document,
|
|
"reference_image_ids",
|
|
"额外参考图",
|
|
args.reference_image_ids.as_slice(),
|
|
)?;
|
|
price_mud_points
|
|
}
|
|
GenerateVideoTool::NAME => {
|
|
let args: GenerateVideoToolArgs = parse_display_tool_args(tool_name, args)?;
|
|
let price_mud_points = GenerateVideoTool {
|
|
context: tool_context.clone(),
|
|
}
|
|
.pricing(pricing, &args);
|
|
push_string_display_arg(&mut display_args, "prompt", "视频描述", args.prompt);
|
|
push_optional_string_display_arg(
|
|
&mut display_args,
|
|
"aspect_ratio",
|
|
"画面比例",
|
|
args.aspect_ratio,
|
|
);
|
|
if let Some(duration_seconds) = args.duration_seconds {
|
|
push_string_display_arg(
|
|
&mut display_args,
|
|
"duration_seconds",
|
|
"时长（秒）",
|
|
duration_seconds.to_string(),
|
|
);
|
|
}
|
|
push_optional_string_display_arg(&mut display_args, "model", "模型", args.model);
|
|
push_optional_string_display_arg(
|
|
&mut display_args,
|
|
"resolution",
|
|
"分辨率",
|
|
args.resolution,
|
|
);
|
|
push_optional_string_display_arg(&mut display_args, "sound", "声音", args.sound);
|
|
push_image_display_arg(
|
|
&mut display_args,
|
|
document,
|
|
"reference_image_ids",
|
|
"参考图片",
|
|
args.reference_image_ids.as_slice(),
|
|
)?;
|
|
price_mud_points
|
|
}
|
|
GenerateSoundEffectTool::NAME => {
|
|
let args: GenerateSoundEffectToolArgs = parse_display_tool_args(tool_name, args)?;
|
|
let price_mud_points = GenerateSoundEffectTool.pricing(pricing, &args);
|
|
push_string_display_arg(&mut display_args, "prompt", "音效描述", args.prompt);
|
|
if let Some(duration) = args.duration {
|
|
push_string_display_arg(
|
|
&mut display_args,
|
|
"duration",
|
|
"时长（秒）",
|
|
duration.to_string(),
|
|
);
|
|
}
|
|
push_optional_string_display_arg(&mut display_args, "model", "模型", args.model);
|
|
price_mud_points
|
|
}
|
|
GenerateBackgroundMusicTool::NAME => {
|
|
let args: GenerateBackgroundMusicToolArgs = parse_display_tool_args(tool_name, args)?;
|
|
let price_mud_points = GenerateBackgroundMusicTool.pricing(pricing, &args);
|
|
push_string_display_arg(&mut display_args, "prompt", "音乐描述", args.prompt);
|
|
push_string_display_arg(
|
|
&mut display_args,
|
|
"make_instrumental",
|
|
"纯音乐",
|
|
if args.make_instrumental { "是" } else { "否" },
|
|
);
|
|
price_mud_points
|
|
}
|
|
_ => {
|
|
return Err(PromptError::InternalError(format!(
|
|
"missing display argument mapping for tool {tool_name}"
|
|
)));
|
|
}
|
|
};
|
|
|
|
display_args.extras = EditorAgentToolCallDisplayExtras { price_mud_points };
|
|
|
|
Ok(display_args)
|
|
}
|
|
|
|
fn parse_display_tool_args<T>(tool_name: &str, args: &Value) -> Result<T, PromptError>
|
|
where
|
|
T: serde::de::DeserializeOwned,
|
|
{
|
|
serde_json::from_value(args.clone()).map_err(|error| {
|
|
PromptError::InternalError(format!(
|
|
"failed to build display arguments for {tool_name}: {error}"
|
|
))
|
|
})
|
|
}
|
|
|
|
fn push_string_display_arg(
|
|
display_args: &mut EditorAgentToolCallDisplayArgs,
|
|
name: &str,
|
|
label: &str,
|
|
value: impl Into<String>,
|
|
) {
|
|
let value = value.into();
|
|
if value.trim().is_empty() {
|
|
return;
|
|
}
|
|
display_args.string_args.push(EditorAgentToolCallStringArg {
|
|
name: name.to_string(),
|
|
label: label.to_string(),
|
|
value,
|
|
});
|
|
}
|
|
|
|
fn push_optional_string_display_arg(
|
|
display_args: &mut EditorAgentToolCallDisplayArgs,
|
|
name: &str,
|
|
label: &str,
|
|
value: Option<String>,
|
|
) {
|
|
if let Some(value) = value {
|
|
push_string_display_arg(display_args, name, label, value);
|
|
}
|
|
}
|
|
|
|
fn push_image_display_arg(
|
|
display_args: &mut EditorAgentToolCallDisplayArgs,
|
|
document: &EditorAgentConversationMessagesDocument,
|
|
name: &str,
|
|
label: &str,
|
|
image_ids: &[ImageId],
|
|
) -> Result<(), PromptError> {
|
|
if image_ids.is_empty() {
|
|
return Ok(());
|
|
}
|
|
|
|
let refs = image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
resolve_tool_call_image_ref(document, image_id).ok_or_else(|| {
|
|
PromptError::InternalError(format!(
|
|
"display image reference {} is missing from conversation context",
|
|
image_id.id
|
|
))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
|
|
display_args.image_args.push(EditorAgentToolCallImageArg {
|
|
name: name.to_string(),
|
|
label: label.to_string(),
|
|
refs,
|
|
});
|
|
Ok(())
|
|
}
|
|
|
|
fn resolve_tool_call_image_ref(
|
|
document: &EditorAgentConversationMessagesDocument,
|
|
image_id: &ImageId,
|
|
) -> Option<EditorAgentToolCallImageRef> {
|
|
for message in document.messages.iter().rev() {
|
|
for attachment in message.attachments.iter().rev() {
|
|
if attachment.clone().into_image_id() == *image_id {
|
|
return Some(EditorAgentToolCallImageRef {
|
|
image_id: image_id.id.clone(),
|
|
image_src: attachment.image_src.clone(),
|
|
object_key: attachment.object_key.clone(),
|
|
thumbnail_src: attachment.thumbnail_src.clone(),
|
|
label: attachment.label.clone(),
|
|
width: attachment.width,
|
|
height: attachment.height,
|
|
});
|
|
}
|
|
}
|
|
|
|
if let Some(tool_call) = &message.tool_call {
|
|
for image in tool_call.images.iter().rev() {
|
|
let data_key = image.clone().into_data_key();
|
|
if ImageId::from_data_key(data_key) == *image_id {
|
|
return Some(EditorAgentToolCallImageRef {
|
|
image_id: image_id.id.clone(),
|
|
image_src: image.image_src.clone(),
|
|
object_key: image.object_key.clone(),
|
|
thumbnail_src: image.thumbnail_src.clone(),
|
|
label: None,
|
|
width: image.width,
|
|
height: image.height,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
None
|
|
}
|
|
|
|
fn build_tool_context(document: &EditorAgentConversationMessagesDocument) -> EditorToolContext {
|
|
let mut images: HashMap<ImageId, ImageMetadata> = HashMap::new();
|
|
|
|
for msg in document.messages.iter().rev() {
|
|
if let Some(tc) = &msg.tool_call {
|
|
for img in &tc.images {
|
|
let data_key = img.clone().into_data_key();
|
|
let image_id = ImageId::from_data_key(&data_key);
|
|
let metadata = ImageMetadata {
|
|
tag: String::new(),
|
|
data_key,
|
|
};
|
|
images.insert(image_id, metadata);
|
|
}
|
|
}
|
|
for a in &msg.attachments {
|
|
let data_key = a.clone().into_data_key();
|
|
let image_id = ImageId::from_data_key(&data_key);
|
|
let metadata = ImageMetadata {
|
|
tag: String::new(),
|
|
data_key,
|
|
};
|
|
images.insert(image_id, metadata);
|
|
}
|
|
}
|
|
|
|
EditorToolContext { images }
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tool_context_tests {
|
|
use super::*;
|
|
use crate::editor_generation_config::load_editor_generation_pricing_from_paths;
|
|
use shared_contracts::editor_agent::{
|
|
EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION, EditorAgentAttachmentRef,
|
|
EditorAgentAttachmentSource, EditorAgentGeneratedImage, EditorAgentMessageRole,
|
|
};
|
|
|
|
#[test]
|
|
fn tool_context_maps_hashed_ids_back_to_real_data_keys() {
|
|
let data_key = "generated-images/proj-1/output.png";
|
|
let document = EditorAgentConversationMessagesDocument {
|
|
version: EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION,
|
|
conversation_id: "editor-agent-conversation-1".to_string(),
|
|
messages: vec![EditorAgentMessage {
|
|
id: 0,
|
|
role: EditorAgentMessageRole::System,
|
|
text: String::new(),
|
|
attachments: Vec::new(),
|
|
tool_call: Some(EditorAgentToolCall {
|
|
tool_name: "generate-image".to_string(),
|
|
args: json!({}),
|
|
display_args: EditorAgentToolCallDisplayArgs::default(),
|
|
external_job_id: Some("task-existing".to_string()),
|
|
cancelled_at: None,
|
|
images: vec![EditorAgentGeneratedImage {
|
|
resource_id: None,
|
|
object_key: Some(data_key.to_string()),
|
|
asset_object_id: None,
|
|
image_src: "/generated-images/proj-1/output.png".to_string(),
|
|
thumbnail_src: None,
|
|
width: None,
|
|
height: None,
|
|
}],
|
|
videos: Vec::new(),
|
|
audios: Vec::new(),
|
|
error: None,
|
|
}),
|
|
created_at: String::new(),
|
|
}],
|
|
};
|
|
|
|
let context = build_tool_context(&document);
|
|
let image_id = ImageId::from_data_key(data_key);
|
|
|
|
assert_eq!(context.image_data_key(&image_id), Some(data_key));
|
|
assert!(!image_id.id.contains(data_key));
|
|
|
|
let display_ref = resolve_tool_call_image_ref(&document, &image_id)
|
|
.expect("generated image should have a display reference");
|
|
assert_eq!(display_ref.image_src, "/generated-images/proj-1/output.png");
|
|
assert_eq!(display_ref.object_key.as_deref(), Some(data_key));
|
|
}
|
|
|
|
#[test]
|
|
fn tool_call_display_args_resolve_target_and_reference_image_previews() {
|
|
let target = EditorAgentAttachmentRef {
|
|
source: EditorAgentAttachmentSource::CanvasResource,
|
|
reference_id: "resource-target".to_string(),
|
|
object_key: Some("editor-projects/proj-1/target.png".to_string()),
|
|
image_src: "/target.png".to_string(),
|
|
thumbnail_src: Some("/target-thumbnail.png".to_string()),
|
|
label: Some("目标角色".to_string()),
|
|
width: Some(640),
|
|
height: Some(960),
|
|
};
|
|
let reference = EditorAgentAttachmentRef {
|
|
source: EditorAgentAttachmentSource::LibraryAsset,
|
|
reference_id: "asset-reference".to_string(),
|
|
object_key: Some("editor-assets/reference.png".to_string()),
|
|
image_src: "/reference.png".to_string(),
|
|
thumbnail_src: None,
|
|
label: Some("像素风参考".to_string()),
|
|
width: Some(512),
|
|
height: Some(512),
|
|
};
|
|
let target_id = target.clone().into_image_id();
|
|
let reference_id = reference.clone().into_image_id();
|
|
let document = EditorAgentConversationMessagesDocument {
|
|
version: EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION,
|
|
conversation_id: "editor-agent-conversation-1".to_string(),
|
|
messages: vec![EditorAgentMessage {
|
|
id: 0,
|
|
role: EditorAgentMessageRole::User,
|
|
text: "改成像素风".to_string(),
|
|
attachments: vec![target, reference],
|
|
tool_call: None,
|
|
created_at: String::new(),
|
|
}],
|
|
};
|
|
|
|
let tool_context = build_tool_context(&document);
|
|
let pricing = load_editor_generation_pricing_from_paths(None)
|
|
.expect("default editor pricing should load");
|
|
let display_args = build_tool_call_display_args(
|
|
EditImageTool::NAME,
|
|
&json!({
|
|
"object_image_id": target_id.id,
|
|
"reference_image_ids": [reference_id.id],
|
|
"prompt": "把角色换成像素风"
|
|
}),
|
|
&document,
|
|
&tool_context,
|
|
&pricing,
|
|
)
|
|
.expect("validated tool args should build a display projection");
|
|
|
|
assert_eq!(display_args.string_args.len(), 1);
|
|
assert_eq!(display_args.string_args[0].label, "修改要求");
|
|
assert_eq!(display_args.string_args[0].value, "把角色换成像素风");
|
|
assert_eq!(display_args.image_args.len(), 2);
|
|
assert_eq!(display_args.image_args[0].label, "目标图片");
|
|
assert_eq!(
|
|
display_args.image_args[0].refs[0].label.as_deref(),
|
|
Some("目标角色")
|
|
);
|
|
assert_eq!(display_args.image_args[1].label, "参考图片");
|
|
assert_eq!(
|
|
display_args.image_args[1].refs[0].label.as_deref(),
|
|
Some("像素风参考")
|
|
);
|
|
assert!(
|
|
display_args
|
|
.image_args
|
|
.iter()
|
|
.flat_map(|arg| &arg.refs)
|
|
.all(|image| image.image_id.starts_with("sha256:"))
|
|
);
|
|
assert_eq!(display_args.extras.price_mud_points, 3);
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
struct EditorAgentConversationDeleteResponse {
|
|
deleted_conversation_id: String,
|
|
conversation: EditorAgentConversationSummary,
|
|
}
|
|
|
|
pub async fn list_editor_agent_conversations(
|
|
State(state): State<AppState>,
|
|
Path(project_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
ensure_editor_project_access(&state, project_id.as_str(), owner_user_id.as_str()).await?;
|
|
let conversations = state
|
|
.spacetime_client()
|
|
.list_editor_agent_conversations(project_id, owner_user_id)
|
|
.await
|
|
.map_err(map_editor_project_error)?
|
|
.into_iter()
|
|
.map(conversation_summary_from_record)
|
|
.collect();
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationListResponse { conversations },
|
|
))
|
|
}
|
|
|
|
pub async fn create_editor_agent_conversation(
|
|
State(state): State<AppState>,
|
|
Path(project_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
Json(payload): Json<CreateEditorAgentConversationRequest>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
ensure_editor_project_access(&state, project_id.as_str(), owner_user_id.as_str()).await?;
|
|
|
|
let conversation_id = build_prefixed_uuid_id(EDITOR_AGENT_CONVERSATION_ID_PREFIX);
|
|
let messages_object_key = editor_agent_messages_object_key(conversation_id.as_str());
|
|
let title = normalize_optional_string(payload.title)
|
|
.unwrap_or_else(|| EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE.to_string());
|
|
let now_micros = current_utc_micros();
|
|
let seed_record = EditorAgentConversationRecord {
|
|
conversation_id: conversation_id.clone(),
|
|
project_id: project_id.clone(),
|
|
owner_user_id: owner_user_id.clone(),
|
|
title: title.clone(),
|
|
messages_object_key: messages_object_key.clone(),
|
|
deleted: false,
|
|
created_at: now_rfc3339(),
|
|
updated_at: now_rfc3339(),
|
|
updated_at_micros: now_micros,
|
|
};
|
|
write_messages_document(
|
|
&state,
|
|
&seed_record,
|
|
&empty_messages_document(conversation_id.as_str()),
|
|
)
|
|
.await?;
|
|
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.create_editor_agent_conversation(EditorAgentConversationCreateRecordInput {
|
|
conversation_id,
|
|
project_id,
|
|
owner_user_id,
|
|
title,
|
|
messages_object_key,
|
|
created_at_micros: now_micros,
|
|
})
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
let document = read_messages_document(&state, &conversation).await?;
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationResponse {
|
|
conversation: conversation_detail_from_record(conversation, document.messages),
|
|
},
|
|
))
|
|
}
|
|
|
|
pub async fn get_editor_agent_conversation(
|
|
State(state): State<AppState>,
|
|
Path(conversation_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id, owner_user_id)
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
let document = read_messages_document(&state, &conversation).await?;
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationResponse {
|
|
conversation: conversation_detail_from_record(conversation, document.messages),
|
|
},
|
|
))
|
|
}
|
|
|
|
pub async fn delete_editor_agent_conversation(
|
|
State(state): State<AppState>,
|
|
Path(conversation_id): Path<String>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.delete_editor_agent_conversation(EditorAgentConversationDeleteRecordInput {
|
|
conversation_id,
|
|
owner_user_id,
|
|
updated_at_micros: current_utc_micros(),
|
|
})
|
|
.await
|
|
.map_err(map_editor_project_error)?;
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
EditorAgentConversationDeleteResponse {
|
|
deleted_conversation_id: conversation.conversation_id.clone(),
|
|
conversation: conversation_summary_from_record(conversation),
|
|
},
|
|
))
|
|
}
|
|
|
|
pub async fn cancel_editor_agent_tool_call(
|
|
State(state): State<AppState>,
|
|
Path((conversation_id, message_id)): Path<(String, usize)>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id, owner_user_id)
|
|
.await
|
|
.map_err(|e| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": format!("conversation not found: {e}") }))
|
|
})?;
|
|
|
|
let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
|
|
conversation.conversation_id.as_str(),
|
|
);
|
|
let _conversation_lock_guard = conversation_lock.lock_owned().await;
|
|
|
|
let mut document: EditorAgentConversationMessagesDocument =
|
|
read_messages_document(&state, &conversation).await?;
|
|
|
|
// Validate message index
|
|
if message_id >= document.messages.len() {
|
|
return Err(AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": "message not found" })));
|
|
}
|
|
|
|
let msg = &mut document.messages[message_id];
|
|
|
|
// Validate role and tool_call
|
|
if msg.role != EditorAgentMessageRole::System {
|
|
return Err(editor_agent_bad_request("message is not a system message"));
|
|
}
|
|
let tc = msg
|
|
.tool_call
|
|
.as_mut()
|
|
.ok_or_else(|| editor_agent_bad_request("message has no tool call"))?;
|
|
if tc.external_job_id.is_some() || tc.cancelled_at.is_some() {
|
|
return Err(editor_agent_bad_request(
|
|
"tool call is no longer pending confirmation",
|
|
));
|
|
}
|
|
|
|
tc.cancelled_at = Some(now_rfc3339());
|
|
let arg_json = tc.args.to_string();
|
|
msg.text = format!(
|
|
"[tool_call:{tool_name}] args: {arg_json} output: 用户已取消该操作",
|
|
tool_name = tc.tool_name,
|
|
arg_json = arg_json,
|
|
);
|
|
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
&document.messages[message_id],
|
|
))
|
|
}
|
|
|
|
pub async fn confirm_editor_agent_tool_call(
|
|
State(state): State<AppState>,
|
|
Path((conversation_id, message_id)): Path<(String, usize)>,
|
|
Extension(request_context): Extension<RequestContext>,
|
|
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
|
) -> Result<Json<Value>, AppError> {
|
|
let owner_user_id = authenticated.claims().user_id().to_string();
|
|
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
|
let conversation = state
|
|
.spacetime_client()
|
|
.get_editor_agent_conversation(conversation_id, owner_user_id)
|
|
.await
|
|
.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND).with_details(json!({
|
|
"message": format!("conversation not found: {error}"),
|
|
}))
|
|
})?;
|
|
let conversation_lock = crate::editor_agent::utils::editor_agent_conversation_lock(
|
|
conversation.conversation_id.as_str(),
|
|
);
|
|
let _conversation_lock_guard = conversation_lock.lock_owned().await;
|
|
let mut document = read_messages_document(&state, &conversation).await?;
|
|
let message = document
|
|
.messages
|
|
.get(message_id)
|
|
.ok_or_else(|| AppError::from_status(axum::http::StatusCode::NOT_FOUND))?;
|
|
if message.role != EditorAgentMessageRole::System {
|
|
return Err(editor_agent_bad_request("message is not a system message"));
|
|
}
|
|
let tool_call = message
|
|
.tool_call
|
|
.as_ref()
|
|
.ok_or_else(|| editor_agent_bad_request("message has no tool call"))?;
|
|
if tool_call.cancelled_at.is_some() {
|
|
return Err(editor_agent_bad_request("tool call was cancelled"));
|
|
}
|
|
if tool_call.external_job_id.is_some() {
|
|
return Ok(json_success_body(Some(&request_context), message));
|
|
}
|
|
|
|
let tool_name = tool_call.tool_name.clone();
|
|
let tool_args = tool_call.args.clone();
|
|
let pricing = state.editor_generation_pricing().await.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({
|
|
"provider": "editor-generation-pricing",
|
|
"message": error.to_string(),
|
|
}))
|
|
})?;
|
|
let project = load_editor_agent_project(&state, &conversation).await?;
|
|
let context = build_tool_context(&document);
|
|
let generation_inputs = |title: &str, value: &str| {
|
|
Some(editor_agent_media_generation_inputs(
|
|
&conversation,
|
|
message_id,
|
|
title,
|
|
value,
|
|
))
|
|
};
|
|
|
|
let (job_kind, request_label, price_mud_points, payload) = match tool_name.as_str() {
|
|
GenerateImageTool::NAME | GenerateCharacterTool::NAME | GenerateUiDesignTool::NAME => {
|
|
let args: GenerateImageToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let tool = GenerateImageTool {
|
|
context: context.clone(),
|
|
};
|
|
tool.validate_args(&args).map_err(|error| {
|
|
editor_agent_bad_request(format!("invalid tool call args: {error}"))
|
|
})?;
|
|
let kind = match tool_name.as_str() {
|
|
GenerateCharacterTool::NAME => Some("character"),
|
|
GenerateUiDesignTool::NAME => Some("ui-design"),
|
|
_ => None,
|
|
};
|
|
let price = match kind {
|
|
Some("character") => GenerateCharacterTool {
|
|
context: context.clone(),
|
|
}
|
|
.pricing(&pricing, &args),
|
|
Some("ui-design") => GenerateUiDesignTool {
|
|
context: context.clone(),
|
|
}
|
|
.pricing(&pricing, &args),
|
|
_ => tool.pricing(&pricing, &args),
|
|
};
|
|
let title = args.prompt.clone();
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let payload = EditorImageGenerationRequest {
|
|
prompt: args.prompt,
|
|
size: None,
|
|
kind: kind.map(ToOwned::to_owned),
|
|
model: None,
|
|
screen_color: (kind == Some("character")).then(|| "auto".to_string()),
|
|
seg_model: (kind == Some("character")).then(|| "birefnet".to_string()),
|
|
aspect_ratio: args.aspect_ratio,
|
|
image_size: args.image_size,
|
|
reference_image_srcs: Some(reference_image_srcs),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
asset_kind: Some(kind.unwrap_or("editor_agent_generated_image").to_string()),
|
|
generation_inputs: editor_agent_generation_inputs(
|
|
&conversation,
|
|
message_id,
|
|
&GenerateImageToolArgs {
|
|
prompt: title.clone(),
|
|
reference_image_ids: Vec::new(),
|
|
aspect_ratio: None,
|
|
image_size: None,
|
|
},
|
|
),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title.clone()),
|
|
source_resource_id: None,
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project, &tool_name, &title,
|
|
)),
|
|
};
|
|
(
|
|
EDITOR_IMAGE_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成图片",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
EditImageTool::NAME => {
|
|
let args: EditImageToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
if args.prompt.trim().is_empty() {
|
|
return Err(editor_agent_bad_request("prompt not provided"));
|
|
}
|
|
let source_image_src = context
|
|
.image_data_key(&args.object_image_id)
|
|
.ok_or_else(|| editor_agent_bad_request("object image not found"))?
|
|
.to_string();
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let title = args.prompt.clone();
|
|
let price = EditImageTool {
|
|
context: context.clone(),
|
|
}
|
|
.pricing(&pricing, &args);
|
|
let payload = EditorImageEditRequest {
|
|
prompt: args.prompt,
|
|
source_image_src,
|
|
size: None,
|
|
model: None,
|
|
reference_image_srcs: Some(reference_image_srcs),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
asset_kind: Some("editor_agent_edit_image".to_string()),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title.clone()),
|
|
source_resource_id: None,
|
|
target_layer_id: None,
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
EditImageTool::NAME,
|
|
&title,
|
|
)),
|
|
};
|
|
(
|
|
EDITOR_IMAGE_EDIT_JOB_KIND,
|
|
"画布 Agent 修改图片",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateIconSpritesheetTool::NAME => {
|
|
let args: GenerateIconSpritesheetToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let tool = GenerateIconSpritesheetTool {
|
|
context: context.clone(),
|
|
};
|
|
let price = tool.pricing(&pricing, &args);
|
|
let reference_image_src = context
|
|
.image_data_key(&args.reference_image_id)
|
|
.ok_or_else(|| editor_agent_bad_request("reference image not found"))?
|
|
.to_string();
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let payload = EditorIconSpritesheetGenerationRequest {
|
|
reference_image_src,
|
|
reference_image_srcs: Some(reference_image_srcs),
|
|
icon_descriptions: args.icon_descriptions.clone(),
|
|
model: None,
|
|
screen_color: Some("auto".to_string()),
|
|
seg_model: Some("birefnet".to_string()),
|
|
aspect_ratio: args.aspect_ratio,
|
|
image_size: args.image_size,
|
|
project_id: Some(conversation.project_id.clone()),
|
|
generation_inputs: Some(
|
|
json!({"source":"editor-agent","conversationId":conversation.conversation_id,"toolCallMessageId":message_id}),
|
|
),
|
|
asset_folder_id: Some("project".to_string()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateIconSpritesheetTool::NAME,
|
|
"图标素材图集",
|
|
)),
|
|
};
|
|
(
|
|
EDITOR_ICON_SPRITESHEET_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成图标素材",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateVideoTool::NAME => {
|
|
let args: GenerateVideoToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let reference_image_srcs = args
|
|
.reference_image_ids
|
|
.iter()
|
|
.map(|image_id| {
|
|
context
|
|
.image_data_key(image_id)
|
|
.map(ToOwned::to_owned)
|
|
.ok_or_else(|| {
|
|
editor_agent_bad_request(format!("image {image_id} not found"))
|
|
})
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
let title = args.prompt.clone();
|
|
let price = GenerateVideoTool {
|
|
context: context.clone(),
|
|
}
|
|
.pricing(&pricing, &args);
|
|
let payload = EditorVideoGenerateRequest {
|
|
prompt: args.prompt,
|
|
model: args.model.unwrap_or_else(|| "seedance2.0-fast".to_string()),
|
|
aspect_ratio: args.aspect_ratio.unwrap_or_else(|| "16:9".to_string()),
|
|
duration_seconds: args.duration_seconds.unwrap_or(4),
|
|
resolution: args.resolution.unwrap_or_else(|| "720p".to_string()),
|
|
mode: "std".to_string(),
|
|
sound: args.sound.unwrap_or_else(|| "off".to_string()),
|
|
web_search_enabled: false,
|
|
reference_image_srcs,
|
|
reference_video_srcs: Vec::new(),
|
|
reference_audio_srcs: Vec::new(),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateVideoTool::NAME,
|
|
&title,
|
|
)),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
source_resource_id: None,
|
|
asset_kind: Some("video".to_string()),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title),
|
|
};
|
|
(
|
|
EDITOR_VIDEO_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成视频",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateSoundEffectTool::NAME => {
|
|
let args: GenerateSoundEffectToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let price = GenerateSoundEffectTool.pricing(&pricing, &args);
|
|
let title = args.prompt.clone();
|
|
let payload = EditorSoundEffectGenerateRequest {
|
|
prompt: args.prompt,
|
|
model: args.model,
|
|
duration: args.duration.unwrap_or(3),
|
|
project_id: Some(conversation.project_id.clone()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateSoundEffectTool::NAME,
|
|
&title,
|
|
)),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some(title),
|
|
};
|
|
(
|
|
EDITOR_SOUND_EFFECT_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成音效",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
GenerateBackgroundMusicTool::NAME => {
|
|
let args: GenerateBackgroundMusicToolArgs = parse_confirm_tool_args(&tool_args)?;
|
|
let price = GenerateBackgroundMusicTool.pricing(&pricing, &args);
|
|
let title = args.prompt.clone();
|
|
let payload = EditorBackgroundMusicGenerateRequest {
|
|
gpt_description_prompt: args.prompt,
|
|
make_instrumental: args.make_instrumental,
|
|
project_id: Some(conversation.project_id.clone()),
|
|
canvas_completion: Some(build_editor_agent_canvas_completion(
|
|
&project,
|
|
GenerateBackgroundMusicTool::NAME,
|
|
"生成背景音乐",
|
|
)),
|
|
generation_inputs: generation_inputs("用户指令", &title),
|
|
asset_folder_id: Some("project".to_string()),
|
|
asset_label: Some("生成背景音乐".to_string()),
|
|
};
|
|
(
|
|
EDITOR_BACKGROUND_MUSIC_GENERATION_JOB_KIND,
|
|
"画布 Agent 生成背景音乐",
|
|
price,
|
|
serde_json::to_value(payload),
|
|
)
|
|
}
|
|
_ => {
|
|
return Err(editor_agent_bad_request(format!(
|
|
"unsupported tool: {tool_name}"
|
|
)));
|
|
}
|
|
};
|
|
let payload = payload.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::INTERNAL_SERVER_ERROR)
|
|
.with_details(json!({"message": format!("failed to serialize tool payload: {error}")}))
|
|
})?;
|
|
let (job_id, dedupe_key) = editor_agent_tool_job_identity(
|
|
conversation.conversation_id.as_str(),
|
|
message_id,
|
|
tool_name.as_str(),
|
|
);
|
|
let job = enqueue_editor_generation_job_with_identity(
|
|
&state,
|
|
conversation.owner_user_id.as_str(),
|
|
job_kind,
|
|
conversation.project_id.clone(),
|
|
request_label,
|
|
u64::from(price_mud_points),
|
|
&payload,
|
|
job_id,
|
|
dedupe_key,
|
|
)
|
|
.await?;
|
|
let message = &mut document.messages[message_id];
|
|
let tool_call = message
|
|
.tool_call
|
|
.as_mut()
|
|
.ok_or_else(|| editor_agent_bad_request("message has no tool call"))?;
|
|
tool_call.external_job_id = Some(job.job_id);
|
|
write_messages_document(&state, &conversation, &document).await?;
|
|
Ok(json_success_body(
|
|
Some(&request_context),
|
|
&document.messages[message_id],
|
|
))
|
|
}
|
|
|
|
fn parse_confirm_tool_args<T: serde::de::DeserializeOwned>(value: &Value) -> Result<T, AppError> {
|
|
serde_json::from_value(value.clone())
|
|
.map_err(|error| editor_agent_bad_request(format!("invalid tool call args: {error}")))
|
|
}
|
|
|
|
fn editor_agent_tool_job_identity(
|
|
conversation_id: &str,
|
|
message_id: usize,
|
|
tool_name: &str,
|
|
) -> (String, String) {
|
|
let dedupe_key = format!("editor-agent:{conversation_id}:{message_id}:{tool_name}");
|
|
let digest = Sha256::digest(dedupe_key.as_bytes());
|
|
(format!("task-editor-agent-{digest:x}"), dedupe_key)
|
|
}
|
|
|
|
async fn load_editor_agent_project(
|
|
state: &AppState,
|
|
conversation: &EditorAgentConversationRecord,
|
|
) -> Result<spacetime_client::EditorProjectRecord, AppError> {
|
|
state
|
|
.spacetime_client()
|
|
.get_editor_project(EditorProjectGetRecordInput {
|
|
project_id: conversation.project_id.clone(),
|
|
owner_user_id: conversation.owner_user_id.clone(),
|
|
})
|
|
.await
|
|
.map_err(|error| {
|
|
AppError::from_status(axum::http::StatusCode::NOT_FOUND)
|
|
.with_details(json!({ "message": format!("project not found: {error}") }))
|
|
})
|
|
}
|
|
fn editor_agent_generation_inputs(
|
|
conversation: &EditorAgentConversationRecord,
|
|
message_id: usize,
|
|
args: &GenerateImageToolArgs,
|
|
) -> Option<Value> {
|
|
Some(json!({
|
|
"source": "editor-agent",
|
|
"conversationId": conversation.conversation_id,
|
|
"toolCallMessageId": message_id,
|
|
"fields": [{ "title": "用户指令", "value": args.prompt }],
|
|
}))
|
|
}
|
|
|
|
fn editor_agent_media_generation_inputs(
|
|
conversation: &EditorAgentConversationRecord,
|
|
message_id: usize,
|
|
title: &str,
|
|
value: &str,
|
|
) -> Value {
|
|
json!({
|
|
"source": "editor-agent",
|
|
"conversationId": conversation.conversation_id,
|
|
"toolCallMessageId": message_id,
|
|
"fields": [{ "title": title, "value": value }],
|
|
})
|
|
}
|