AIGameCreator App
4ef3dbd677
同步 master 的最新功能与修复 保留 AI 游戏创作客户端分支现有实现 # Conflicts: # docs/project-memory/shared-memory/decision-log.md # docs/project-memory/shared-memory/pitfalls.md # scripts/dev.mjs # server-rs/crates/api-server/src/editor_screen_background_decision.rs # server-rs/crates/api-server/src/modules/admin.rs # src/components/rpg-entry/RpgEntryHomeView.tsx
2815 lines
110 KiB
Rust
2815 lines
110 KiB
Rust
use std::{
|
||
collections::BTreeMap,
|
||
convert::Infallible,
|
||
sync::{Arc, Mutex, OnceLock},
|
||
};
|
||
|
||
use axum::{
|
||
Json,
|
||
extract::{Extension, Path, State},
|
||
http::StatusCode,
|
||
response::{
|
||
IntoResponse, Response,
|
||
sse::{Event, Sse},
|
||
},
|
||
};
|
||
use module_editor_agent::{
|
||
EDITOR_AGENT_CONVERSATION_ID_PREFIX, EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE,
|
||
EDITOR_AGENT_MESSAGE_ID_PREFIX, derive_conversation_title, editor_agent_messages_object_key,
|
||
validate_user_message,
|
||
};
|
||
use platform_llm::{LlmError, LlmErrorKind, LlmMessage, LlmMessageContentPart, LlmRunRequest};
|
||
use platform_oss::{
|
||
LegacyAssetPrefix, OssObjectAccess, OssPutObjectRequest, OssSignedGetObjectUrlRequest,
|
||
};
|
||
use serde::{Deserialize, Serialize};
|
||
use serde_json::{Value, json};
|
||
use shared_contracts::assets::{
|
||
EditorCanvasGenerationCompletionPayload, EditorCanvasGenerationPlaceholderPayload,
|
||
};
|
||
use shared_contracts::editor_agent::{
|
||
CreateEditorAgentConversationRequest, EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION,
|
||
EditorAgentAttachmentRef, EditorAgentAttachmentSource, EditorAgentConversationDetail,
|
||
EditorAgentConversationListResponse, EditorAgentConversationMessagesDocument,
|
||
EditorAgentConversationResponse, EditorAgentConversationSummary, EditorAgentDoneEvent,
|
||
EditorAgentErrorEvent, EditorAgentGeneratedImage, EditorAgentGenerationRecord,
|
||
EditorAgentGenerationResultEvent, EditorAgentGenerationStatus, EditorAgentMessage,
|
||
EditorAgentMessageDeltaEvent, EditorAgentMessageKind, EditorAgentMessageRole,
|
||
EditorAgentMessageStatus, EditorAgentStage, EditorAgentStageEvent, EditorAgentToolEvent,
|
||
EditorAgentToolName, StreamEditorAgentMessageRequest,
|
||
};
|
||
use shared_kernel::{build_prefixed_uuid_id, normalize_optional_string, normalize_required_string};
|
||
use spacetime_client::{
|
||
EditorAgentConversationCreateRecordInput, EditorAgentConversationDeleteRecordInput,
|
||
EditorAgentConversationRecord, EditorAgentConversationTouchRecordInput,
|
||
EditorAssetLibraryRecord, EditorAssetRecord, EditorProjectGetRecordInput, EditorProjectRecord,
|
||
EditorProjectResourceRecord,
|
||
};
|
||
|
||
use crate::{
|
||
api_response::json_success_body,
|
||
auth::AuthenticatedAccessToken,
|
||
editor_project::{
|
||
EditorGenerationCaller, EditorIconSpritesheetGenerationRequest, EditorImageEditRequest,
|
||
EditorImageGenerationRequest, current_utc_micros, edit_editor_image_for_owner,
|
||
generate_editor_icon_spritesheet_for_owner, generate_editor_image_for_owner,
|
||
map_editor_project_error,
|
||
},
|
||
http_error::AppError,
|
||
platform_errors::map_oss_error,
|
||
request_context::RequestContext,
|
||
state::AppState,
|
||
};
|
||
|
||
const EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES: usize = 2 * 1024 * 1024;
|
||
const EDITOR_AGENT_MESSAGES_READ_EXPIRE_SECONDS: u64 = 60;
|
||
const EDITOR_AGENT_LLM_ATTACHMENT_URL_EXPIRE_SECONDS: u64 = 300;
|
||
const EDITOR_AGENT_LLM_MAX_HISTORY_MESSAGES: usize = 12;
|
||
const EDITOR_AGENT_LLM_PLANNING_MODEL: &str = platform_agent::CREATIVE_AGENT_GPT5_MODEL;
|
||
const EDITOR_AGENT_LLM_MAX_OUTPUT_TOKENS: u32 = 1024;
|
||
const EDITOR_AGENT_LLM_REQUEST_TIMEOUT_MS: u64 = 60_000;
|
||
const EDITOR_AGENT_TOOL_CALL_ID_PREFIX: &str = "editor-agent-tool";
|
||
const EDITOR_AGENT_CANVAS_RESULT_GAP: f64 = 32.0;
|
||
type EditorAgentConversationLockMap = Mutex<BTreeMap<String, Arc<tokio::sync::Mutex<()>>>>;
|
||
static EDITOR_AGENT_CONVERSATION_LOCKS: OnceLock<EditorAgentConversationLockMap> = OnceLock::new();
|
||
|
||
#[derive(Debug, Serialize)]
|
||
#[serde(rename_all = "camelCase")]
|
||
struct EditorAgentConversationDeleteResponse {
|
||
deleted_conversation_id: String,
|
||
conversation: EditorAgentConversationSummary,
|
||
}
|
||
|
||
pub async fn list_editor_agent_conversations(
|
||
State(state): State<AppState>,
|
||
Path(project_id): Path<String>,
|
||
Extension(request_context): Extension<RequestContext>,
|
||
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
||
) -> Result<Json<Value>, AppError> {
|
||
let owner_user_id = authenticated.claims().user_id().to_string();
|
||
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
||
ensure_editor_project_access(&state, project_id.as_str(), owner_user_id.as_str()).await?;
|
||
let conversations = state
|
||
.spacetime_client()
|
||
.list_editor_agent_conversations(project_id, owner_user_id)
|
||
.await
|
||
.map_err(map_editor_project_error)?
|
||
.into_iter()
|
||
.map(conversation_summary_from_record)
|
||
.collect();
|
||
|
||
Ok(json_success_body(
|
||
Some(&request_context),
|
||
EditorAgentConversationListResponse { conversations },
|
||
))
|
||
}
|
||
|
||
pub async fn create_editor_agent_conversation(
|
||
State(state): State<AppState>,
|
||
Path(project_id): Path<String>,
|
||
Extension(request_context): Extension<RequestContext>,
|
||
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
||
Json(payload): Json<CreateEditorAgentConversationRequest>,
|
||
) -> Result<Json<Value>, AppError> {
|
||
let owner_user_id = authenticated.claims().user_id().to_string();
|
||
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
||
ensure_editor_project_access(&state, project_id.as_str(), owner_user_id.as_str()).await?;
|
||
|
||
let conversation_id = build_prefixed_uuid_id(EDITOR_AGENT_CONVERSATION_ID_PREFIX);
|
||
let messages_object_key = editor_agent_messages_object_key(conversation_id.as_str());
|
||
let title = normalize_optional_string(payload.title)
|
||
.unwrap_or_else(|| EDITOR_AGENT_DEFAULT_CONVERSATION_TITLE.to_string());
|
||
let now_micros = current_utc_micros();
|
||
let seed_record = EditorAgentConversationRecord {
|
||
conversation_id: conversation_id.clone(),
|
||
project_id: project_id.clone(),
|
||
owner_user_id: owner_user_id.clone(),
|
||
title: title.clone(),
|
||
messages_object_key: messages_object_key.clone(),
|
||
deleted: false,
|
||
created_at: now_rfc3339(),
|
||
updated_at: now_rfc3339(),
|
||
updated_at_micros: now_micros,
|
||
};
|
||
write_messages_document(
|
||
&state,
|
||
&seed_record,
|
||
&empty_messages_document(conversation_id.as_str()),
|
||
)
|
||
.await?;
|
||
|
||
let conversation = state
|
||
.spacetime_client()
|
||
.create_editor_agent_conversation(EditorAgentConversationCreateRecordInput {
|
||
conversation_id,
|
||
project_id,
|
||
owner_user_id,
|
||
title,
|
||
messages_object_key,
|
||
created_at_micros: now_micros,
|
||
})
|
||
.await
|
||
.map_err(map_editor_project_error)?;
|
||
let document = read_messages_document(&state, &conversation).await?;
|
||
|
||
Ok(json_success_body(
|
||
Some(&request_context),
|
||
EditorAgentConversationResponse {
|
||
conversation: conversation_detail_from_record(conversation, document.messages),
|
||
},
|
||
))
|
||
}
|
||
|
||
pub async fn get_editor_agent_conversation(
|
||
State(state): State<AppState>,
|
||
Path(conversation_id): Path<String>,
|
||
Extension(request_context): Extension<RequestContext>,
|
||
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
||
) -> Result<Json<Value>, AppError> {
|
||
let owner_user_id = authenticated.claims().user_id().to_string();
|
||
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
||
let conversation = state
|
||
.spacetime_client()
|
||
.get_editor_agent_conversation(conversation_id, owner_user_id)
|
||
.await
|
||
.map_err(map_editor_project_error)?;
|
||
let document = read_messages_document(&state, &conversation).await?;
|
||
|
||
Ok(json_success_body(
|
||
Some(&request_context),
|
||
EditorAgentConversationResponse {
|
||
conversation: conversation_detail_from_record(conversation, document.messages),
|
||
},
|
||
))
|
||
}
|
||
|
||
pub async fn delete_editor_agent_conversation(
|
||
State(state): State<AppState>,
|
||
Path(conversation_id): Path<String>,
|
||
Extension(request_context): Extension<RequestContext>,
|
||
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
||
) -> Result<Json<Value>, AppError> {
|
||
let owner_user_id = authenticated.claims().user_id().to_string();
|
||
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
||
let conversation = state
|
||
.spacetime_client()
|
||
.delete_editor_agent_conversation(EditorAgentConversationDeleteRecordInput {
|
||
conversation_id,
|
||
owner_user_id,
|
||
updated_at_micros: current_utc_micros(),
|
||
})
|
||
.await
|
||
.map_err(map_editor_project_error)?;
|
||
|
||
Ok(json_success_body(
|
||
Some(&request_context),
|
||
EditorAgentConversationDeleteResponse {
|
||
deleted_conversation_id: conversation.conversation_id.clone(),
|
||
conversation: conversation_summary_from_record(conversation),
|
||
},
|
||
))
|
||
}
|
||
|
||
pub async fn stream_editor_agent_message(
|
||
State(state): State<AppState>,
|
||
Path(conversation_id): Path<String>,
|
||
Extension(request_context): Extension<RequestContext>,
|
||
Extension(authenticated): Extension<AuthenticatedAccessToken>,
|
||
Json(payload): Json<StreamEditorAgentMessageRequest>,
|
||
) -> Result<Response, AppError> {
|
||
let owner_user_id = authenticated.claims().user_id().to_string();
|
||
require_editor_agent_sidebar_enabled(&state, owner_user_id.as_str()).await?;
|
||
let client_message_id = normalize_required_string(payload.client_message_id.as_str())
|
||
.ok_or_else(|| editor_agent_bad_request("clientMessageId is required"))?;
|
||
let attachment_reference_ids = payload
|
||
.attachments
|
||
.iter()
|
||
.map(|attachment| attachment.reference_id.clone())
|
||
.collect::<Vec<_>>();
|
||
validate_user_message(payload.text.as_str(), &attachment_reference_ids)
|
||
.map_err(|error| editor_agent_bad_request(error.to_string()))?;
|
||
|
||
let conversation = state
|
||
.spacetime_client()
|
||
.get_editor_agent_conversation(conversation_id, owner_user_id.clone())
|
||
.await
|
||
.map_err(map_editor_project_error)?;
|
||
let attachments =
|
||
normalize_editor_agent_attachments(&state, &conversation, payload.attachments.as_slice())
|
||
.await?;
|
||
let conversation_lock = editor_agent_conversation_lock(conversation.conversation_id.as_str());
|
||
let conversation_lock_guard = conversation_lock.lock_owned().await;
|
||
let mut document = read_messages_document(&state, &conversation).await?;
|
||
if document
|
||
.messages
|
||
.iter()
|
||
.any(|message| message.id == client_message_id)
|
||
{
|
||
return Err(
|
||
AppError::from_status(StatusCode::CONFLICT).with_details(json!({
|
||
"provider": "editor-agent",
|
||
"field": "clientMessageId",
|
||
"message": "clientMessageId already exists in this conversation",
|
||
})),
|
||
);
|
||
}
|
||
|
||
let was_empty = document.messages.is_empty();
|
||
let user_message = EditorAgentMessage {
|
||
id: client_message_id,
|
||
role: EditorAgentMessageRole::User,
|
||
kind: EditorAgentMessageKind::Chat,
|
||
text: payload.text.trim().to_string(),
|
||
attachments,
|
||
generations: Vec::new(),
|
||
status: EditorAgentMessageStatus::Completed,
|
||
created_at: now_rfc3339(),
|
||
};
|
||
document.messages.push(user_message.clone());
|
||
write_messages_document(&state, &conversation, &document).await?;
|
||
let latest_generation_reference = latest_editor_agent_generated_image_reference(&document);
|
||
|
||
let assistant_message_id = build_prefixed_uuid_id(EDITOR_AGENT_MESSAGE_ID_PREFIX);
|
||
let stream_state = state.clone();
|
||
let stream_conversation = conversation.clone();
|
||
let stream_title = was_empty.then(|| derive_conversation_title(user_message.text.as_str()));
|
||
let stream_assistant_message_id = assistant_message_id.clone();
|
||
let stream_user_message = user_message.clone();
|
||
let stream_latest_generation_reference = latest_generation_reference.clone();
|
||
let stream_request_context = request_context.clone();
|
||
let stream_conversation_lock_guard = conversation_lock_guard;
|
||
|
||
let stream = async_stream::stream! {
|
||
let _conversation_lock_guard = stream_conversation_lock_guard;
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"stage",
|
||
EditorAgentStageEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
stage: EditorAgentStage::Thinking,
|
||
},
|
||
));
|
||
|
||
let mut assistant_status = EditorAgentMessageStatus::Completed;
|
||
let mut generation_records = Vec::new();
|
||
let mut planning_failed = false;
|
||
|
||
let turn_plan = match plan_editor_agent_turn(
|
||
&stream_state,
|
||
&stream_conversation,
|
||
&document,
|
||
&stream_user_message,
|
||
stream_latest_generation_reference.as_ref(),
|
||
)
|
||
.await {
|
||
Ok(turn_plan) => turn_plan,
|
||
Err(error) => {
|
||
let app_error = error.into_app_error();
|
||
let error_message = app_error.body_text();
|
||
assistant_status = EditorAgentMessageStatus::Failed;
|
||
planning_failed = true;
|
||
EditorAgentTurnPlan {
|
||
reply_text: error_message,
|
||
tool_call: None,
|
||
}
|
||
}
|
||
};
|
||
let assistant_text = turn_plan.reply_text;
|
||
let assistant_kind = if assistant_status == EditorAgentMessageStatus::Failed {
|
||
EditorAgentMessageKind::Error
|
||
} else {
|
||
EditorAgentMessageKind::Chat
|
||
};
|
||
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"stage",
|
||
EditorAgentStageEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
stage: EditorAgentStage::Responding,
|
||
},
|
||
));
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"message_delta",
|
||
EditorAgentMessageDeltaEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
message_id: stream_assistant_message_id.clone(),
|
||
role: EditorAgentMessageRole::Assistant,
|
||
kind: assistant_kind,
|
||
text_delta: assistant_text.clone(),
|
||
},
|
||
));
|
||
if planning_failed {
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"stage",
|
||
EditorAgentStageEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
stage: EditorAgentStage::Failed,
|
||
},
|
||
));
|
||
}
|
||
|
||
if let Some(tool_call) = turn_plan.tool_call {
|
||
let tool_call_id = build_prefixed_uuid_id(EDITOR_AGENT_TOOL_CALL_ID_PREFIX);
|
||
let tool_summary = tool_call
|
||
.summary
|
||
.clone()
|
||
.or_else(|| Some(editor_agent_tool_default_summary(tool_call.tool_name)));
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"tool_started",
|
||
EditorAgentToolEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
message_id: stream_assistant_message_id.clone(),
|
||
tool_call_id: tool_call_id.clone(),
|
||
tool_name: tool_call.tool_name,
|
||
summary: tool_summary.clone(),
|
||
task_id: None,
|
||
model: tool_call.model.clone(),
|
||
status: Some(EditorAgentGenerationStatus::Generating),
|
||
error: None,
|
||
},
|
||
));
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"stage",
|
||
EditorAgentStageEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
stage: EditorAgentStage::Generating,
|
||
},
|
||
));
|
||
|
||
match execute_editor_agent_tool_call(
|
||
&stream_state,
|
||
&stream_request_context,
|
||
&stream_conversation,
|
||
&stream_user_message,
|
||
&tool_call_id,
|
||
stream_latest_generation_reference.as_ref(),
|
||
tool_call,
|
||
)
|
||
.await
|
||
{
|
||
Ok(execution) => {
|
||
generation_records.push(execution.record.clone());
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"generation_result",
|
||
EditorAgentGenerationResultEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
message_id: stream_assistant_message_id.clone(),
|
||
tool_call_id: tool_call_id.clone(),
|
||
tool_name: execution.record.tool_name,
|
||
model: execution.record.model.clone(),
|
||
images: execution.record.images.clone(),
|
||
},
|
||
));
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"tool_completed",
|
||
EditorAgentToolEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
message_id: stream_assistant_message_id.clone(),
|
||
tool_call_id,
|
||
tool_name: execution.record.tool_name,
|
||
summary: Some(execution.summary),
|
||
task_id: execution.record.task_id.clone(),
|
||
model: execution.record.model.clone(),
|
||
status: Some(EditorAgentGenerationStatus::Completed),
|
||
error: None,
|
||
},
|
||
));
|
||
}
|
||
Err(error) => {
|
||
let error_code = error.error.code().to_string();
|
||
let error_message = error.error.body_text();
|
||
let error_model = error.model.clone();
|
||
assistant_status = EditorAgentMessageStatus::Failed;
|
||
generation_records.push(EditorAgentGenerationRecord {
|
||
tool_call_id: tool_call_id.clone(),
|
||
tool_name: error.tool_name,
|
||
summary: tool_summary.clone(),
|
||
task_id: None,
|
||
status: EditorAgentGenerationStatus::Failed,
|
||
model: error_model.clone(),
|
||
images: Vec::new(),
|
||
error: Some(error_message.clone()),
|
||
});
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"tool_completed",
|
||
EditorAgentToolEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
message_id: stream_assistant_message_id.clone(),
|
||
tool_call_id,
|
||
tool_name: error.tool_name,
|
||
summary: tool_summary,
|
||
task_id: None,
|
||
model: error_model,
|
||
status: Some(EditorAgentGenerationStatus::Failed),
|
||
error: Some(error_message.clone()),
|
||
},
|
||
));
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"stage",
|
||
EditorAgentStageEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
stage: EditorAgentStage::Failed,
|
||
},
|
||
));
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"error",
|
||
EditorAgentErrorEvent {
|
||
conversation_id: Some(stream_conversation.conversation_id.clone()),
|
||
code: error_code,
|
||
message: error_message,
|
||
recoverable: true,
|
||
},
|
||
));
|
||
}
|
||
}
|
||
}
|
||
|
||
let write_result = async {
|
||
let mut next_document = read_messages_document(&stream_state, &stream_conversation).await?;
|
||
if !next_document
|
||
.messages
|
||
.iter()
|
||
.any(|message| message.id == stream_assistant_message_id)
|
||
{
|
||
next_document.messages.push(EditorAgentMessage {
|
||
id: stream_assistant_message_id.clone(),
|
||
role: EditorAgentMessageRole::Assistant,
|
||
kind: if assistant_status == EditorAgentMessageStatus::Failed {
|
||
EditorAgentMessageKind::Error
|
||
} else {
|
||
EditorAgentMessageKind::Chat
|
||
},
|
||
text: assistant_text,
|
||
attachments: Vec::new(),
|
||
generations: generation_records,
|
||
status: assistant_status,
|
||
created_at: now_rfc3339(),
|
||
});
|
||
}
|
||
write_messages_document(&stream_state, &stream_conversation, &next_document).await
|
||
}
|
||
.await;
|
||
match write_result {
|
||
Ok(()) => {
|
||
let mut done_title = None;
|
||
if let Some(title) = stream_title {
|
||
match stream_state.spacetime_client().touch_editor_agent_conversation(
|
||
EditorAgentConversationTouchRecordInput {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
owner_user_id: stream_conversation.owner_user_id.clone(),
|
||
title: Some(title.clone()),
|
||
updated_at_micros: current_utc_micros(),
|
||
},
|
||
).await {
|
||
Ok(updated) => done_title = Some(updated.title),
|
||
Err(error) => {
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"error",
|
||
EditorAgentErrorEvent {
|
||
conversation_id: Some(stream_conversation.conversation_id.clone()),
|
||
code: "SPACETIME_UPDATE_FAILED".to_string(),
|
||
message: error.to_string(),
|
||
recoverable: true,
|
||
},
|
||
));
|
||
}
|
||
}
|
||
} else {
|
||
match stream_state.spacetime_client().touch_editor_agent_conversation(
|
||
EditorAgentConversationTouchRecordInput {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
owner_user_id: stream_conversation.owner_user_id.clone(),
|
||
title: None,
|
||
updated_at_micros: current_utc_micros(),
|
||
},
|
||
).await {
|
||
Ok(_) => {}
|
||
Err(error) => {
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"error",
|
||
EditorAgentErrorEvent {
|
||
conversation_id: Some(stream_conversation.conversation_id.clone()),
|
||
code: "SPACETIME_UPDATE_FAILED".to_string(),
|
||
message: error.to_string(),
|
||
recoverable: true,
|
||
},
|
||
));
|
||
}
|
||
}
|
||
}
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"stage",
|
||
EditorAgentStageEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
stage: EditorAgentStage::Completed,
|
||
},
|
||
));
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"done",
|
||
EditorAgentDoneEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
title: done_title,
|
||
},
|
||
));
|
||
}
|
||
Err(error) => {
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"stage",
|
||
EditorAgentStageEvent {
|
||
conversation_id: stream_conversation.conversation_id.clone(),
|
||
stage: EditorAgentStage::Failed,
|
||
},
|
||
));
|
||
yield Ok::<Event, Infallible>(editor_agent_sse_json_event_or_error(
|
||
"error",
|
||
EditorAgentErrorEvent {
|
||
conversation_id: Some(stream_conversation.conversation_id.clone()),
|
||
code: error.code().to_string(),
|
||
message: error.body_text(),
|
||
recoverable: true,
|
||
},
|
||
));
|
||
}
|
||
}
|
||
};
|
||
|
||
Ok(Sse::new(stream).into_response())
|
||
}
|
||
|
||
fn editor_agent_conversation_lock(conversation_id: &str) -> Arc<tokio::sync::Mutex<()>> {
|
||
let locks = EDITOR_AGENT_CONVERSATION_LOCKS.get_or_init(|| Mutex::new(BTreeMap::new()));
|
||
let mut locks = locks
|
||
.lock()
|
||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||
locks
|
||
.entry(conversation_id.to_string())
|
||
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
|
||
.clone()
|
||
}
|
||
|
||
async fn ensure_editor_project_access(
|
||
state: &AppState,
|
||
project_id: &str,
|
||
owner_user_id: &str,
|
||
) -> Result<(), AppError> {
|
||
state
|
||
.spacetime_client()
|
||
.get_editor_project(EditorProjectGetRecordInput {
|
||
project_id: project_id.to_string(),
|
||
owner_user_id: owner_user_id.to_string(),
|
||
})
|
||
.await
|
||
.map(|_| ())
|
||
.map_err(map_editor_project_error)
|
||
}
|
||
|
||
async fn normalize_editor_agent_attachments(
|
||
state: &AppState,
|
||
conversation: &EditorAgentConversationRecord,
|
||
attachments: &[EditorAgentAttachmentRef],
|
||
) -> Result<Vec<EditorAgentAttachmentRef>, AppError> {
|
||
if attachments.is_empty() {
|
||
return Ok(Vec::new());
|
||
}
|
||
|
||
let needs_canvas_resources = attachments
|
||
.iter()
|
||
.any(|attachment| attachment.source == EditorAgentAttachmentSource::CanvasResource);
|
||
let needs_library_assets = attachments
|
||
.iter()
|
||
.any(|attachment| attachment.source == EditorAgentAttachmentSource::LibraryAsset);
|
||
|
||
let project = if needs_canvas_resources {
|
||
Some(
|
||
state
|
||
.spacetime_client()
|
||
.get_editor_project(EditorProjectGetRecordInput {
|
||
project_id: conversation.project_id.clone(),
|
||
owner_user_id: conversation.owner_user_id.clone(),
|
||
})
|
||
.await
|
||
.map_err(map_editor_project_error)?,
|
||
)
|
||
} else {
|
||
None
|
||
};
|
||
let library = if needs_library_assets {
|
||
Some(
|
||
state
|
||
.spacetime_client()
|
||
.get_editor_asset_library(conversation.owner_user_id.clone(), current_utc_micros())
|
||
.await
|
||
.map_err(map_editor_project_error)?,
|
||
)
|
||
} else {
|
||
None
|
||
};
|
||
|
||
attachments
|
||
.iter()
|
||
.map(|attachment| {
|
||
normalize_editor_agent_attachment(
|
||
conversation,
|
||
project.as_ref(),
|
||
library.as_ref(),
|
||
attachment,
|
||
)
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
fn normalize_editor_agent_attachment(
|
||
conversation: &EditorAgentConversationRecord,
|
||
project: Option<&EditorProjectRecord>,
|
||
library: Option<&EditorAssetLibraryRecord>,
|
||
attachment: &EditorAgentAttachmentRef,
|
||
) -> Result<EditorAgentAttachmentRef, AppError> {
|
||
let reference_id = normalize_required_string(attachment.reference_id.as_str())
|
||
.ok_or_else(|| editor_agent_bad_request("attachment.referenceId is required"))?;
|
||
match attachment.source {
|
||
EditorAgentAttachmentSource::CanvasResource => {
|
||
let project = project.ok_or_else(|| {
|
||
editor_agent_bad_request("canvas resource attachment project context missing")
|
||
})?;
|
||
let resource = project
|
||
.resources
|
||
.iter()
|
||
.find(|resource| resource.resource_id == reference_id)
|
||
.ok_or_else(|| {
|
||
editor_agent_bad_request(format!(
|
||
"canvas resource attachment not found in current project: {reference_id}"
|
||
))
|
||
})?;
|
||
normalize_canvas_resource_attachment(conversation, attachment, resource)
|
||
}
|
||
EditorAgentAttachmentSource::LibraryAsset => {
|
||
let library = library.ok_or_else(|| {
|
||
editor_agent_bad_request("library asset attachment context missing")
|
||
})?;
|
||
let asset = library
|
||
.assets
|
||
.iter()
|
||
.find(|asset| asset.asset_id == reference_id)
|
||
.ok_or_else(|| {
|
||
editor_agent_bad_request(format!(
|
||
"library asset attachment not found for current user: {reference_id}"
|
||
))
|
||
})?;
|
||
normalize_library_asset_attachment(attachment, asset)
|
||
}
|
||
}
|
||
}
|
||
|
||
fn normalize_canvas_resource_attachment(
|
||
conversation: &EditorAgentConversationRecord,
|
||
attachment: &EditorAgentAttachmentRef,
|
||
resource: &EditorProjectResourceRecord,
|
||
) -> Result<EditorAgentAttachmentRef, AppError> {
|
||
if resource.project_id != conversation.project_id
|
||
|| resource.owner_user_id != conversation.owner_user_id
|
||
{
|
||
return Err(editor_agent_bad_request(
|
||
"canvas resource attachment does not belong to this conversation project",
|
||
));
|
||
}
|
||
validate_attachment_object_key(
|
||
attachment.object_key.as_deref(),
|
||
resource.object_key.as_deref(),
|
||
resource.resource_id.as_str(),
|
||
)?;
|
||
|
||
Ok(EditorAgentAttachmentRef {
|
||
source: EditorAgentAttachmentSource::CanvasResource,
|
||
reference_id: resource.resource_id.clone(),
|
||
object_key: resource.object_key.clone(),
|
||
image_src: resource.image_src.clone(),
|
||
thumbnail_src: None,
|
||
label: normalize_optional_string(attachment.label.clone()),
|
||
width: Some(resource.width),
|
||
height: Some(resource.height),
|
||
})
|
||
}
|
||
|
||
fn normalize_library_asset_attachment(
|
||
attachment: &EditorAgentAttachmentRef,
|
||
asset: &EditorAssetRecord,
|
||
) -> Result<EditorAgentAttachmentRef, AppError> {
|
||
validate_attachment_object_key(
|
||
attachment.object_key.as_deref(),
|
||
asset.object_key.as_deref(),
|
||
asset.asset_id.as_str(),
|
||
)?;
|
||
|
||
Ok(EditorAgentAttachmentRef {
|
||
source: EditorAgentAttachmentSource::LibraryAsset,
|
||
reference_id: asset.asset_id.clone(),
|
||
object_key: asset.object_key.clone(),
|
||
image_src: asset.image_src.clone(),
|
||
thumbnail_src: asset.thumbnail_src.clone(),
|
||
label: normalize_optional_string(attachment.label.clone())
|
||
.or_else(|| Some(asset.label.clone())),
|
||
width: Some(asset.width),
|
||
height: Some(asset.height),
|
||
})
|
||
}
|
||
|
||
fn validate_attachment_object_key(
|
||
submitted_object_key: Option<&str>,
|
||
stored_object_key: Option<&str>,
|
||
reference_id: &str,
|
||
) -> Result<(), AppError> {
|
||
let Some(submitted_object_key) = submitted_object_key.and_then(normalize_required_string)
|
||
else {
|
||
return Ok(());
|
||
};
|
||
let Some(stored_object_key) = stored_object_key.and_then(normalize_required_string) else {
|
||
return Err(editor_agent_bad_request(format!(
|
||
"attachment objectKey is not available for reference: {reference_id}"
|
||
)));
|
||
};
|
||
if submitted_object_key != stored_object_key {
|
||
return Err(editor_agent_bad_request(format!(
|
||
"attachment objectKey does not match reference: {reference_id}"
|
||
)));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
fn conversation_summary_from_record(
|
||
conversation: EditorAgentConversationRecord,
|
||
) -> EditorAgentConversationSummary {
|
||
EditorAgentConversationSummary {
|
||
conversation_id: conversation.conversation_id,
|
||
project_id: conversation.project_id,
|
||
title: conversation.title,
|
||
created_at: conversation.created_at,
|
||
updated_at: conversation.updated_at,
|
||
}
|
||
}
|
||
|
||
fn conversation_detail_from_record(
|
||
conversation: EditorAgentConversationRecord,
|
||
messages: Vec<EditorAgentMessage>,
|
||
) -> EditorAgentConversationDetail {
|
||
EditorAgentConversationDetail {
|
||
conversation_id: conversation.conversation_id,
|
||
project_id: conversation.project_id,
|
||
title: conversation.title,
|
||
created_at: conversation.created_at,
|
||
updated_at: conversation.updated_at,
|
||
messages,
|
||
}
|
||
}
|
||
|
||
fn empty_messages_document(conversation_id: &str) -> EditorAgentConversationMessagesDocument {
|
||
EditorAgentConversationMessagesDocument {
|
||
version: EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION,
|
||
conversation_id: conversation_id.to_string(),
|
||
messages: Vec::new(),
|
||
}
|
||
}
|
||
|
||
async fn read_messages_document(
|
||
state: &AppState,
|
||
conversation: &EditorAgentConversationRecord,
|
||
) -> Result<EditorAgentConversationMessagesDocument, AppError> {
|
||
let oss_client = state
|
||
.oss_client()
|
||
.ok_or_else(editor_agent_oss_unavailable)?;
|
||
let signed = oss_client
|
||
.sign_internal_get_object_url(OssSignedGetObjectUrlRequest {
|
||
object_key: conversation.messages_object_key.clone(),
|
||
expire_seconds: Some(EDITOR_AGENT_MESSAGES_READ_EXPIRE_SECONDS),
|
||
})
|
||
.map_err(|error| map_oss_error(error, "aliyun-oss"))?;
|
||
let response = reqwest::Client::new()
|
||
.get(signed.signed_url.as_str())
|
||
.send()
|
||
.await
|
||
.map_err(|error| editor_agent_oss_read_error(error.to_string()))?;
|
||
if response.status() == reqwest::StatusCode::NOT_FOUND {
|
||
return Ok(empty_messages_document(
|
||
conversation.conversation_id.as_str(),
|
||
));
|
||
}
|
||
if !response.status().is_success() {
|
||
return Err(editor_agent_oss_read_error(format!(
|
||
"OSS returned non-success status {}",
|
||
response.status().as_u16()
|
||
)));
|
||
}
|
||
if response
|
||
.content_length()
|
||
.is_some_and(|size| size > EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES as u64)
|
||
{
|
||
return Err(editor_agent_messages_document_too_large());
|
||
}
|
||
let bytes = response
|
||
.bytes()
|
||
.await
|
||
.map_err(|error| editor_agent_oss_read_error(error.to_string()))?;
|
||
if bytes.is_empty() {
|
||
return Ok(empty_messages_document(
|
||
conversation.conversation_id.as_str(),
|
||
));
|
||
}
|
||
if bytes.len() > EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES {
|
||
return Err(editor_agent_messages_document_too_large());
|
||
}
|
||
let document: EditorAgentConversationMessagesDocument = serde_json::from_slice(&bytes)
|
||
.map_err(|error| {
|
||
editor_agent_oss_read_error(format!("message document JSON invalid: {error}"))
|
||
})?;
|
||
if document.conversation_id != conversation.conversation_id {
|
||
return Err(
|
||
AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({
|
||
"provider": "editor-agent",
|
||
"message": "message document conversationId does not match metadata",
|
||
"conversationId": conversation.conversation_id,
|
||
"documentConversationId": document.conversation_id,
|
||
})),
|
||
);
|
||
}
|
||
Ok(document)
|
||
}
|
||
|
||
async fn write_messages_document(
|
||
state: &AppState,
|
||
conversation: &EditorAgentConversationRecord,
|
||
document: &EditorAgentConversationMessagesDocument,
|
||
) -> Result<(), AppError> {
|
||
if document.conversation_id != conversation.conversation_id {
|
||
return Err(
|
||
AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({
|
||
"provider": "editor-agent",
|
||
"message": "message document conversationId does not match metadata",
|
||
})),
|
||
);
|
||
}
|
||
let body = serde_json::to_vec(document).map_err(|error| {
|
||
AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({
|
||
"provider": "editor-agent",
|
||
"message": format!("failed to serialize message document: {error}"),
|
||
}))
|
||
})?;
|
||
if body.len() > EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES {
|
||
return Err(editor_agent_messages_document_too_large());
|
||
}
|
||
|
||
let oss_client = state
|
||
.oss_client()
|
||
.ok_or_else(editor_agent_oss_unavailable)?;
|
||
let put_result = oss_client
|
||
.put_object(
|
||
&reqwest::Client::new(),
|
||
OssPutObjectRequest {
|
||
prefix: LegacyAssetPrefix::EditorAgent,
|
||
path_segments: Vec::new(),
|
||
file_name: format!("{}.json", conversation.conversation_id),
|
||
content_type: Some("application/json; charset=utf-8".to_string()),
|
||
access: OssObjectAccess::Private,
|
||
metadata: BTreeMap::from([
|
||
(
|
||
"conversation-id".to_string(),
|
||
conversation.conversation_id.clone(),
|
||
),
|
||
("project-id".to_string(), conversation.project_id.clone()),
|
||
(
|
||
"owner-user-id".to_string(),
|
||
conversation.owner_user_id.clone(),
|
||
),
|
||
]),
|
||
body,
|
||
},
|
||
)
|
||
.await
|
||
.map_err(|error| map_oss_error(error, "aliyun-oss"))?;
|
||
if put_result.object_key != conversation.messages_object_key {
|
||
return Err(
|
||
AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({
|
||
"provider": "editor-agent",
|
||
"message": "OSS object key mismatch while writing message document",
|
||
"expectedObjectKey": conversation.messages_object_key,
|
||
"actualObjectKey": put_result.object_key,
|
||
})),
|
||
);
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq)]
|
||
struct EditorAgentTurnPlan {
|
||
reply_text: String,
|
||
tool_call: Option<EditorAgentToolCallPlan>,
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq)]
|
||
struct EditorAgentToolCallPlan {
|
||
tool_name: EditorAgentToolName,
|
||
prompt: String,
|
||
summary: Option<String>,
|
||
model: Option<String>,
|
||
aspect_ratio: Option<String>,
|
||
image_size: Option<String>,
|
||
icon_descriptions: Vec<String>,
|
||
}
|
||
|
||
#[derive(Debug, Deserialize)]
|
||
#[serde(rename_all = "camelCase")]
|
||
struct EditorAgentRawTurnPlan {
|
||
reply_text: Option<String>,
|
||
tool_call: Option<EditorAgentRawToolCallPlan>,
|
||
}
|
||
|
||
#[derive(Debug, Deserialize)]
|
||
#[serde(rename_all = "camelCase")]
|
||
struct EditorAgentRawToolCallPlan {
|
||
tool_name: EditorAgentToolName,
|
||
prompt: Option<String>,
|
||
summary: Option<String>,
|
||
model: Option<String>,
|
||
aspect_ratio: Option<String>,
|
||
image_size: Option<String>,
|
||
icon_descriptions: Option<Vec<String>>,
|
||
}
|
||
|
||
#[derive(Debug)]
|
||
struct EditorAgentToolExecution {
|
||
record: EditorAgentGenerationRecord,
|
||
summary: String,
|
||
}
|
||
|
||
#[derive(Clone, Debug)]
|
||
struct EditorAgentGeneratedImageReference {
|
||
source_message_id: String,
|
||
tool_call_id: String,
|
||
tool_name: EditorAgentToolName,
|
||
summary: Option<String>,
|
||
model: Option<String>,
|
||
resource_id: Option<String>,
|
||
object_key: Option<String>,
|
||
asset_object_id: Option<String>,
|
||
image_src: String,
|
||
thumbnail_src: Option<String>,
|
||
width: Option<u32>,
|
||
height: Option<u32>,
|
||
}
|
||
|
||
#[derive(Debug)]
|
||
struct EditorAgentToolExecutionError {
|
||
tool_name: EditorAgentToolName,
|
||
model: Option<String>,
|
||
error: AppError,
|
||
}
|
||
|
||
#[derive(Debug)]
|
||
enum EditorAgentPlanningError {
|
||
LlmUnavailable,
|
||
LlmRequestFailed(LlmError),
|
||
InvalidLlmResponse,
|
||
}
|
||
|
||
impl EditorAgentPlanningError {
|
||
fn into_app_error(self) -> AppError {
|
||
match self {
|
||
Self::LlmUnavailable => AppError::from_status(StatusCode::SERVICE_UNAVAILABLE)
|
||
.with_details(json!({
|
||
"provider": "editor-agent-llm",
|
||
"message": "画布 Agent 的 LLM 未配置，无法处理这句话。",
|
||
})),
|
||
Self::LlmRequestFailed(error) => {
|
||
let status = match error.kind() {
|
||
LlmErrorKind::Timeout => StatusCode::GATEWAY_TIMEOUT,
|
||
LlmErrorKind::InvalidConfig | LlmErrorKind::InvalidRequest => {
|
||
StatusCode::SERVICE_UNAVAILABLE
|
||
}
|
||
LlmErrorKind::Connectivity
|
||
| LlmErrorKind::Upstream
|
||
| LlmErrorKind::StreamUnavailable
|
||
| LlmErrorKind::EmptyResponse
|
||
| LlmErrorKind::Transport
|
||
| LlmErrorKind::Deserialize => StatusCode::BAD_GATEWAY,
|
||
};
|
||
AppError::from_status(status).with_details(json!({
|
||
"provider": "editor-agent-llm",
|
||
"message": format!("画布 Agent 调用 LLM 失败：{error}"),
|
||
}))
|
||
}
|
||
Self::InvalidLlmResponse => AppError::from_status(StatusCode::BAD_GATEWAY)
|
||
.with_details(json!({
|
||
"provider": "editor-agent-llm",
|
||
"message": "画布 Agent 的 LLM 返回格式错误，未能解析回复。",
|
||
})),
|
||
}
|
||
}
|
||
}
|
||
|
||
async fn plan_editor_agent_turn(
|
||
state: &AppState,
|
||
conversation: &EditorAgentConversationRecord,
|
||
document: &EditorAgentConversationMessagesDocument,
|
||
user_message: &EditorAgentMessage,
|
||
latest_generation_reference: Option<&EditorAgentGeneratedImageReference>,
|
||
) -> Result<EditorAgentTurnPlan, EditorAgentPlanningError> {
|
||
let Some(llm_client) = state.creative_agent_gpt5_client() else {
|
||
return Err(EditorAgentPlanningError::LlmUnavailable);
|
||
};
|
||
|
||
let request = build_editor_agent_llm_request(
|
||
state,
|
||
conversation,
|
||
document,
|
||
user_message,
|
||
latest_generation_reference,
|
||
)
|
||
.await;
|
||
let response = llm_client
|
||
.run(request)
|
||
.await
|
||
.map_err(EditorAgentPlanningError::LlmRequestFailed)?;
|
||
parse_editor_agent_turn_plan(response.text.as_str())
|
||
.map(|plan| {
|
||
apply_previous_generation_edit_default(plan, user_message, latest_generation_reference)
|
||
})
|
||
.ok_or(EditorAgentPlanningError::InvalidLlmResponse)
|
||
}
|
||
|
||
async fn build_editor_agent_llm_request(
|
||
state: &AppState,
|
||
conversation: &EditorAgentConversationRecord,
|
||
document: &EditorAgentConversationMessagesDocument,
|
||
user_message: &EditorAgentMessage,
|
||
latest_generation_reference: Option<&EditorAgentGeneratedImageReference>,
|
||
) -> LlmRunRequest {
|
||
let mut user_parts = vec![LlmMessageContentPart::InputText {
|
||
text: build_editor_agent_llm_user_prompt(
|
||
conversation,
|
||
document,
|
||
user_message,
|
||
latest_generation_reference,
|
||
),
|
||
}];
|
||
for attachment in &user_message.attachments {
|
||
if let Some(image_url) = sign_editor_agent_attachment_image_url(state, attachment) {
|
||
user_parts.push(LlmMessageContentPart::InputImage { image_url });
|
||
}
|
||
}
|
||
if user_message.attachments.is_empty()
|
||
&& editor_agent_text_mentions_previous_generation(user_message.text.as_str())
|
||
{
|
||
if let Some(reference) = latest_generation_reference {
|
||
if let Some(image_url) = sign_editor_agent_generated_image_url(state, reference) {
|
||
user_parts.push(LlmMessageContentPart::InputImage { image_url });
|
||
}
|
||
}
|
||
}
|
||
|
||
LlmRunRequest::new(vec![
|
||
LlmMessage::system(editor_agent_llm_system_prompt()),
|
||
LlmMessage::user_multimodal(user_parts),
|
||
])
|
||
.with_model(EDITOR_AGENT_LLM_PLANNING_MODEL)
|
||
.with_openai_chat()
|
||
.with_max_output_tokens(EDITOR_AGENT_LLM_MAX_OUTPUT_TOKENS)
|
||
.with_request_timeout_ms(EDITOR_AGENT_LLM_REQUEST_TIMEOUT_MS)
|
||
}
|
||
|
||
fn editor_agent_llm_system_prompt() -> String {
|
||
[
|
||
"你是 Genarrative 图片画布 Agent，只负责帮助用户理解、规划和触发画布生成工具。",
|
||
"必须只输出一个 JSON 对象，不要输出 Markdown、解释或额外文本。",
|
||
"JSON 结构：{\"replyText\":\"给用户看的中文回复\",\"toolCall\":null 或 {\"toolName\":\"generate_image|edit_image|generate_character|generate_icon_spritesheet|generate_ui_design\",\"prompt\":\"生成或修改提示词\",\"summary\":\"短动作摘要\",\"model\":null,\"aspectRatio\":null,\"imageSize\":\"1K\",\"iconDescriptions\":[\"图标描述\"]}}。",
|
||
"只有用户明确要求生成、重绘、修改、画 UI、出角色或图标时才给 toolCall；普通咨询、评价、解释时 toolCall 为 null。",
|
||
"工具说明：",
|
||
"generate_image：从文字生成一张全新图片，适合新场景、新物体、新插画、新背景、规范图/视觉规范图/风格规范图/素材规范展板；不要用于修改上一张图或附件图。prompt 必须写完整画面、主体、风格、构图和背景。",
|
||
"edit_image：修改已有图片，适合换衣服、改颜色、替换背景、局部重绘、保持主体/构图/姿势不变的编辑请求；必须使用 latestAttachments 或 latestGeneratedImage 作为源图。",
|
||
"generate_character：生成新的角色形象、人物立绘或角色设定图；如果用户是在改上一张角色图的服装、颜色、表情、姿势或背景，应改用 edit_image。",
|
||
"generate_icon_spritesheet：生成一组图标素材或图标图集，适合用户明确要多个 icon / 图标 / spritesheet；必须有 latestAttachments 作为图标规范或风格参考，并填写 iconDescriptions。",
|
||
"generate_ui_design：生成一张完整 UI 设计图或界面稿，适合 HUD、弹窗、面板、按钮组合和整页界面；不要用于提取图标、拆素材或修改上一张图。",
|
||
"规范图要求：用户要求生成规范图/视觉规范图/风格规范图/素材规范展板时，当前使用 generate_image；prompt 必须明确这是规范展板，并写入统一视角、线条粗细、描边、填充风格、材质、阴影、圆角、状态层级、色卡/色号、尺寸标注和排版层级。",
|
||
"角色规范图/角色美术视觉规范设定图属于规范展板时使用 generate_image，prompt 要包含头身比例、标准立绘、动作帧样例、服饰配饰分层和专属角色色卡；只有用户要生成单个全新角色形象/立绘/普通角色设定图时才使用 generate_character。",
|
||
"图标规范图/图标视觉规范展板属于规范展板时使用 generate_image；只有用户明确要生成多个图标成品、图标素材图集或 spritesheet，并提供图标规范/风格参考附件时才使用 generate_icon_spritesheet。",
|
||
"UI 规范图/组件规范展板如果是规范展板而非完整可用界面稿，也使用 generate_image；完整界面稿/HUD/弹窗/面板才使用 generate_ui_design。",
|
||
"工具选择优先级：明确修改/指代已有图 => edit_image；规范图/视觉规范图/风格规范图/素材规范展板 => generate_image；明确新角色且不是规范展板或修改已有图 => generate_character；多个图标成品/图标图集 => generate_icon_spritesheet；完整界面稿 => generate_ui_design；其他全新图片 => generate_image。",
|
||
"edit_image 必须依赖 latestAttachments 或 latestGeneratedImage；generate_icon_spritesheet 必须依赖 latestAttachments。",
|
||
"当 latestGeneratedImage 存在，且用户说“这张/刚才那个/上一张/把衣服换成/改成/换成/修改上一张图”等指代或修改上一轮结果的话，必须选择 edit_image，prompt 保留用户修改要求，不要要求用户重新选择参考图，也不要降级成 generate_image。",
|
||
"没有 latestAttachments 且没有 latestGeneratedImage 时，不要调用 edit_image 或 generate_icon_spritesheet，应在 replyText 中提示先选择参考图。",
|
||
"generate_ui_design 表示生成一张 UI 设计图，不是提取 UI 素材。",
|
||
"对话回复要简短，不能承诺免费生成；生成工具由后端按模型定价扣泥点。",
|
||
]
|
||
.join("\n")
|
||
}
|
||
|
||
fn build_editor_agent_llm_user_prompt(
|
||
conversation: &EditorAgentConversationRecord,
|
||
document: &EditorAgentConversationMessagesDocument,
|
||
user_message: &EditorAgentMessage,
|
||
latest_generation_reference: Option<&EditorAgentGeneratedImageReference>,
|
||
) -> String {
|
||
let history = document
|
||
.messages
|
||
.iter()
|
||
.rev()
|
||
.take(EDITOR_AGENT_LLM_MAX_HISTORY_MESSAGES)
|
||
.collect::<Vec<_>>()
|
||
.into_iter()
|
||
.rev()
|
||
.map(|message| {
|
||
json!({
|
||
"role": message.role,
|
||
"kind": message.kind,
|
||
"text": message.text,
|
||
"attachmentCount": message.attachments.len(),
|
||
"generationCount": message.generations.len(),
|
||
"generations": message
|
||
.generations
|
||
.iter()
|
||
.map(editor_agent_generation_context_json)
|
||
.collect::<Vec<_>>(),
|
||
})
|
||
})
|
||
.collect::<Vec<_>>();
|
||
let attachments = user_message
|
||
.attachments
|
||
.iter()
|
||
.map(|attachment| {
|
||
json!({
|
||
"source": attachment.source,
|
||
"referenceId": attachment.reference_id,
|
||
"label": attachment.label,
|
||
"width": attachment.width,
|
||
"height": attachment.height,
|
||
"hasObjectKey": attachment.object_key.as_ref().is_some_and(|value| !value.trim().is_empty()),
|
||
})
|
||
})
|
||
.collect::<Vec<_>>();
|
||
|
||
json!({
|
||
"conversationId": conversation.conversation_id,
|
||
"projectId": conversation.project_id,
|
||
"latestUserText": user_message.text,
|
||
"latestAttachments": attachments,
|
||
"latestGeneratedImage": latest_generation_reference
|
||
.map(editor_agent_generated_image_reference_context_json),
|
||
"recentMessages": history,
|
||
})
|
||
.to_string()
|
||
}
|
||
|
||
fn editor_agent_generation_context_json(generation: &EditorAgentGenerationRecord) -> Value {
|
||
json!({
|
||
"toolCallId": generation.tool_call_id,
|
||
"toolName": generation.tool_name,
|
||
"summary": generation.summary,
|
||
"status": generation.status,
|
||
"model": generation.model,
|
||
"images": generation.images.iter().take(4).map(|image| json!({
|
||
"resourceId": image.resource_id,
|
||
"objectKey": image.object_key,
|
||
"assetObjectId": image.asset_object_id,
|
||
"imageSrc": image.image_src,
|
||
"thumbnailSrc": image.thumbnail_src,
|
||
"width": image.width,
|
||
"height": image.height,
|
||
"hasObjectKey": image.object_key.as_ref().is_some_and(|value| !value.trim().is_empty()),
|
||
})).collect::<Vec<_>>(),
|
||
})
|
||
}
|
||
|
||
fn editor_agent_generated_image_reference_context_json(
|
||
reference: &EditorAgentGeneratedImageReference,
|
||
) -> Value {
|
||
json!({
|
||
"sourceMessageId": reference.source_message_id,
|
||
"toolCallId": reference.tool_call_id,
|
||
"toolName": reference.tool_name,
|
||
"summary": reference.summary,
|
||
"model": reference.model,
|
||
"resourceId": reference.resource_id,
|
||
"objectKey": reference.object_key,
|
||
"assetObjectId": reference.asset_object_id,
|
||
"imageSrc": reference.image_src,
|
||
"thumbnailSrc": reference.thumbnail_src,
|
||
"width": reference.width,
|
||
"height": reference.height,
|
||
"hasObjectKey": reference.object_key.as_ref().is_some_and(|value| !value.trim().is_empty()),
|
||
})
|
||
}
|
||
|
||
fn sign_editor_agent_attachment_image_url(
|
||
state: &AppState,
|
||
attachment: &EditorAgentAttachmentRef,
|
||
) -> Option<String> {
|
||
let object_key = attachment
|
||
.object_key
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())?;
|
||
state
|
||
.oss_client()?
|
||
.sign_get_object_url(OssSignedGetObjectUrlRequest {
|
||
object_key: object_key.trim_start_matches('/').to_string(),
|
||
expire_seconds: Some(EDITOR_AGENT_LLM_ATTACHMENT_URL_EXPIRE_SECONDS),
|
||
})
|
||
.ok()
|
||
.map(|signed| signed.signed_url)
|
||
}
|
||
|
||
fn sign_editor_agent_generated_image_url(
|
||
state: &AppState,
|
||
reference: &EditorAgentGeneratedImageReference,
|
||
) -> Option<String> {
|
||
let object_key = reference
|
||
.object_key
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())?;
|
||
state
|
||
.oss_client()?
|
||
.sign_get_object_url(OssSignedGetObjectUrlRequest {
|
||
object_key: object_key.trim_start_matches('/').to_string(),
|
||
expire_seconds: Some(EDITOR_AGENT_LLM_ATTACHMENT_URL_EXPIRE_SECONDS),
|
||
})
|
||
.ok()
|
||
.map(|signed| signed.signed_url)
|
||
}
|
||
|
||
fn latest_editor_agent_generated_image_reference(
|
||
document: &EditorAgentConversationMessagesDocument,
|
||
) -> Option<EditorAgentGeneratedImageReference> {
|
||
document.messages.iter().rev().find_map(|message| {
|
||
message
|
||
.generations
|
||
.iter()
|
||
.rev()
|
||
.filter(|generation| generation.status == EditorAgentGenerationStatus::Completed)
|
||
.find_map(|generation| {
|
||
generation.images.iter().rev().find_map(|image| {
|
||
let has_reference_source = image
|
||
.object_key
|
||
.as_deref()
|
||
.is_some_and(|value| !value.trim().is_empty())
|
||
|| image
|
||
.resource_id
|
||
.as_deref()
|
||
.is_some_and(|value| !value.trim().is_empty())
|
||
|| !image.image_src.trim().is_empty();
|
||
has_reference_source.then(|| EditorAgentGeneratedImageReference {
|
||
source_message_id: message.id.clone(),
|
||
tool_call_id: generation.tool_call_id.clone(),
|
||
tool_name: generation.tool_name,
|
||
summary: generation.summary.clone(),
|
||
model: generation.model.clone(),
|
||
resource_id: image.resource_id.clone(),
|
||
object_key: image.object_key.clone(),
|
||
asset_object_id: image.asset_object_id.clone(),
|
||
image_src: image.image_src.clone(),
|
||
thumbnail_src: image.thumbnail_src.clone(),
|
||
width: image.width,
|
||
height: image.height,
|
||
})
|
||
})
|
||
})
|
||
})
|
||
}
|
||
|
||
fn editor_agent_generated_reference_source(
|
||
reference: &EditorAgentGeneratedImageReference,
|
||
) -> Option<String> {
|
||
reference
|
||
.object_key
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
.map(|value| value.trim_start_matches('/').to_string())
|
||
.or_else(|| {
|
||
reference
|
||
.resource_id
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
.map(ToOwned::to_owned)
|
||
})
|
||
.or_else(|| {
|
||
reference
|
||
.image_src
|
||
.trim()
|
||
.strip_prefix('/')
|
||
.map(str::to_string)
|
||
.filter(|value| !value.is_empty())
|
||
})
|
||
}
|
||
|
||
fn editor_agent_text_mentions_previous_generation(text: &str) -> bool {
|
||
let normalized = text.trim().to_lowercase();
|
||
contains_any(
|
||
normalized.as_str(),
|
||
&[
|
||
"这张",
|
||
"这个图",
|
||
"这幅",
|
||
"这张图",
|
||
"刚才",
|
||
"刚刚",
|
||
"刚才那个",
|
||
"刚才那张",
|
||
"上一张",
|
||
"上张",
|
||
"上一幅",
|
||
"上一个",
|
||
"前一张",
|
||
"上次生成",
|
||
"刚生成",
|
||
"latest image",
|
||
"previous image",
|
||
"last image",
|
||
],
|
||
)
|
||
}
|
||
|
||
fn editor_agent_text_requests_previous_generation_edit(text: &str) -> bool {
|
||
let normalized = text.trim().to_lowercase();
|
||
if editor_agent_text_mentions_previous_generation(normalized.as_str())
|
||
&& contains_any(
|
||
normalized.as_str(),
|
||
&[
|
||
"修改", "改成", "换成", "替换", "重绘", "编辑", "优化", "去掉", "加上", "增加",
|
||
"变成", "调整", "换一", "换掉", "remove", "change", "edit", "redraw",
|
||
],
|
||
)
|
||
{
|
||
return true;
|
||
}
|
||
|
||
let has_visual_target = contains_any(
|
||
normalized.as_str(),
|
||
&[
|
||
"衣服", "服装", "发型", "头发", "背景", "颜色", "色调", "表情", "姿势", "主体", "风格",
|
||
"脸", "眼睛", "帽子", "鞋", "裙子", "外套",
|
||
],
|
||
);
|
||
let has_edit_verb = contains_any(
|
||
normalized.as_str(),
|
||
&[
|
||
"换", "改", "变", "加", "去", "删", "替换", "调整", "change", "edit", "remove",
|
||
],
|
||
);
|
||
has_visual_target && has_edit_verb
|
||
}
|
||
|
||
fn apply_previous_generation_edit_default(
|
||
mut plan: EditorAgentTurnPlan,
|
||
user_message: &EditorAgentMessage,
|
||
latest_generation_reference: Option<&EditorAgentGeneratedImageReference>,
|
||
) -> EditorAgentTurnPlan {
|
||
if latest_generation_reference.is_none()
|
||
|| !user_message.attachments.is_empty()
|
||
|| !editor_agent_text_requests_previous_generation_edit(user_message.text.as_str())
|
||
{
|
||
return plan;
|
||
}
|
||
|
||
let should_force_edit = plan
|
||
.tool_call
|
||
.as_ref()
|
||
.is_none_or(|tool_call| tool_call.tool_name != EditorAgentToolName::EditImage);
|
||
if !should_force_edit {
|
||
return plan;
|
||
}
|
||
|
||
let original_tool = plan.tool_call.take();
|
||
let prompt = original_tool
|
||
.as_ref()
|
||
.map(|tool_call| tool_call.prompt.clone())
|
||
.unwrap_or_else(|| user_message.text.trim().to_string());
|
||
let summary = original_tool
|
||
.as_ref()
|
||
.and_then(|tool_call| tool_call.summary.clone())
|
||
.or_else(|| Some("修改上一张图".to_string()));
|
||
let model = original_tool
|
||
.as_ref()
|
||
.and_then(|tool_call| tool_call.model.clone());
|
||
let image_size = original_tool
|
||
.as_ref()
|
||
.and_then(|tool_call| tool_call.image_size.clone())
|
||
.or_else(|| Some("1K".to_string()));
|
||
|
||
plan.reply_text = "我会基于上一张生成图按你的要求修改。".to_string();
|
||
plan.tool_call = Some(EditorAgentToolCallPlan {
|
||
tool_name: EditorAgentToolName::EditImage,
|
||
prompt,
|
||
summary,
|
||
model,
|
||
aspect_ratio: None,
|
||
image_size,
|
||
icon_descriptions: Vec::new(),
|
||
});
|
||
plan
|
||
}
|
||
|
||
fn parse_editor_agent_turn_plan(raw_text: &str) -> Option<EditorAgentTurnPlan> {
|
||
let json_text = extract_editor_agent_json_object(raw_text)?;
|
||
let raw: EditorAgentRawTurnPlan = serde_json::from_str(json_text).ok()?;
|
||
let tool_call = match raw.tool_call {
|
||
Some(tool) => {
|
||
let prompt = normalize_optional_string(tool.prompt)?;
|
||
Some(EditorAgentToolCallPlan {
|
||
tool_name: tool.tool_name,
|
||
prompt,
|
||
summary: normalize_optional_string(tool.summary),
|
||
model: normalize_optional_string(tool.model),
|
||
aspect_ratio: normalize_optional_string(tool.aspect_ratio),
|
||
image_size: normalize_optional_string(tool.image_size),
|
||
icon_descriptions: tool
|
||
.icon_descriptions
|
||
.unwrap_or_default()
|
||
.into_iter()
|
||
.filter_map(|description| normalize_optional_string(Some(description)))
|
||
.collect(),
|
||
})
|
||
}
|
||
None => None,
|
||
};
|
||
let reply_text = normalize_editor_agent_reply_text(raw.reply_text, tool_call.as_ref())?;
|
||
|
||
Some(EditorAgentTurnPlan {
|
||
reply_text,
|
||
tool_call,
|
||
})
|
||
}
|
||
|
||
fn extract_editor_agent_json_object(raw_text: &str) -> Option<&str> {
|
||
let trimmed = raw_text.trim();
|
||
let without_fence = if let Some(rest) = trimmed.strip_prefix("```json") {
|
||
rest.strip_suffix("```").unwrap_or(rest).trim()
|
||
} else if let Some(rest) = trimmed.strip_prefix("```") {
|
||
rest.strip_suffix("```").unwrap_or(rest).trim()
|
||
} else {
|
||
trimmed
|
||
};
|
||
let start = without_fence.find('{')?;
|
||
let end = without_fence.rfind('}')?;
|
||
(start <= end).then_some(&without_fence[start..=end])
|
||
}
|
||
|
||
fn normalize_editor_agent_reply_text(
|
||
reply_text: Option<String>,
|
||
tool_call: Option<&EditorAgentToolCallPlan>,
|
||
) -> Option<String> {
|
||
let reply_text = normalize_optional_string(reply_text)?;
|
||
if !editor_agent_reply_text_looks_structural(reply_text.as_str()) {
|
||
return Some(reply_text);
|
||
}
|
||
tool_call.map(editor_agent_tool_call_reply_text)
|
||
}
|
||
|
||
fn editor_agent_reply_text_looks_structural(text: &str) -> bool {
|
||
let trimmed = text.trim();
|
||
if trimmed.is_empty() {
|
||
return true;
|
||
}
|
||
let lower = trimmed.to_ascii_lowercase();
|
||
if matches!(lower.as_str(), "null" | "undefined") {
|
||
return true;
|
||
}
|
||
trimmed.len() <= 2
|
||
&& trimmed
|
||
.chars()
|
||
.all(|character| matches!(character, '{' | '}' | '[' | ']' | '"' | ':' | ','))
|
||
}
|
||
|
||
fn editor_agent_tool_call_reply_text(tool_call: &EditorAgentToolCallPlan) -> String {
|
||
match tool_call.tool_name {
|
||
EditorAgentToolName::GenerateImage => "我来生成图片。".to_string(),
|
||
EditorAgentToolName::EditImage => "我来修改图片。".to_string(),
|
||
EditorAgentToolName::GenerateCharacter => "我来生成角色形象。".to_string(),
|
||
EditorAgentToolName::GenerateIconSpritesheet => "我来生成图标素材。".to_string(),
|
||
EditorAgentToolName::GenerateUiDesign => "我来生成 UI 设计图。".to_string(),
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
fn heuristic_editor_agent_turn_plan(
|
||
text: &str,
|
||
attachments: &[EditorAgentAttachmentRef],
|
||
) -> EditorAgentTurnPlan {
|
||
let trimmed = text.trim();
|
||
let normalized = trimmed.to_lowercase();
|
||
let wants_generation = contains_any(
|
||
normalized.as_str(),
|
||
&[
|
||
"生成", "出图", "绘制", "设计", "画一", "画个", "画张", "generate", "create",
|
||
],
|
||
);
|
||
let wants_edit = contains_any(
|
||
normalized.as_str(),
|
||
&[
|
||
"修改", "改成", "重绘", "换成", "编辑", "优化", "edit", "redraw",
|
||
],
|
||
);
|
||
let wants_character = contains_any(
|
||
normalized.as_str(),
|
||
&["角色", "人物", "立绘", "形象", "character"],
|
||
);
|
||
let wants_icons = contains_any(
|
||
normalized.as_str(),
|
||
&["图标", "icon", "spritesheet", "素材图集"],
|
||
);
|
||
let wants_ui = contains_any(
|
||
normalized.as_str(),
|
||
&["ui", "界面", "hud", "按钮", "面板", "弹窗"],
|
||
);
|
||
|
||
let tool_name = if wants_icons && !attachments.is_empty() {
|
||
Some(EditorAgentToolName::GenerateIconSpritesheet)
|
||
} else if wants_edit && !attachments.is_empty() {
|
||
Some(EditorAgentToolName::EditImage)
|
||
} else if wants_character && wants_generation {
|
||
Some(EditorAgentToolName::GenerateCharacter)
|
||
} else if wants_ui && wants_generation {
|
||
Some(EditorAgentToolName::GenerateUiDesign)
|
||
} else if wants_generation {
|
||
Some(EditorAgentToolName::GenerateImage)
|
||
} else {
|
||
None
|
||
};
|
||
|
||
let reply_text = match tool_name {
|
||
Some(EditorAgentToolName::EditImage) => "我来按你的要求修改这张图。".to_string(),
|
||
Some(EditorAgentToolName::GenerateCharacter) => "我来生成角色形象。".to_string(),
|
||
Some(EditorAgentToolName::GenerateIconSpritesheet) => "我来生成图标素材图集。".to_string(),
|
||
Some(EditorAgentToolName::GenerateUiDesign) => "我来生成 UI 设计图。".to_string(),
|
||
Some(EditorAgentToolName::GenerateImage) => "我来生成图片。".to_string(),
|
||
None => build_echo_assistant_text(trimmed, attachments.len()),
|
||
};
|
||
let tool_call = tool_name.map(|tool_name| EditorAgentToolCallPlan {
|
||
tool_name,
|
||
prompt: if trimmed.is_empty() {
|
||
"根据附件生成适合当前画布的图片".to_string()
|
||
} else {
|
||
trimmed.to_string()
|
||
},
|
||
summary: Some(editor_agent_tool_default_summary(tool_name)),
|
||
model: None,
|
||
aspect_ratio: None,
|
||
image_size: Some("1K".to_string()),
|
||
icon_descriptions: derive_icon_descriptions(trimmed),
|
||
});
|
||
|
||
EditorAgentTurnPlan {
|
||
reply_text,
|
||
tool_call,
|
||
}
|
||
}
|
||
|
||
fn contains_any(text: &str, needles: &[&str]) -> bool {
|
||
needles.iter().any(|needle| text.contains(needle))
|
||
}
|
||
|
||
fn derive_icon_descriptions(prompt: &str) -> Vec<String> {
|
||
let normalized = prompt.trim();
|
||
if normalized.is_empty() {
|
||
return vec![
|
||
"主按钮图标".to_string(),
|
||
"奖励图标".to_string(),
|
||
"关闭图标".to_string(),
|
||
];
|
||
}
|
||
normalized
|
||
.split(['、', ',', '，', '\n', ';', '；'])
|
||
.map(str::trim)
|
||
.filter(|item| !item.is_empty())
|
||
.take(8)
|
||
.map(ToOwned::to_owned)
|
||
.collect::<Vec<_>>()
|
||
.into_iter()
|
||
.filter(|item| item.chars().count() <= 80)
|
||
.collect::<Vec<_>>()
|
||
.pipe_non_empty_or_else(|| vec![normalized.to_string()])
|
||
}
|
||
|
||
trait VecPipeNonEmpty<T> {
|
||
fn pipe_non_empty_or_else<F>(self, fallback: F) -> Self
|
||
where
|
||
F: FnOnce() -> Self;
|
||
}
|
||
|
||
impl<T> VecPipeNonEmpty<T> for Vec<T> {
|
||
fn pipe_non_empty_or_else<F>(self, fallback: F) -> Self
|
||
where
|
||
F: FnOnce() -> Self,
|
||
{
|
||
if self.is_empty() { fallback() } else { self }
|
||
}
|
||
}
|
||
|
||
async fn execute_editor_agent_tool_call(
|
||
state: &AppState,
|
||
request_context: &RequestContext,
|
||
conversation: &EditorAgentConversationRecord,
|
||
user_message: &EditorAgentMessage,
|
||
tool_call_id: &str,
|
||
latest_generation_reference: Option<&EditorAgentGeneratedImageReference>,
|
||
tool_call: EditorAgentToolCallPlan,
|
||
) -> Result<EditorAgentToolExecution, EditorAgentToolExecutionError> {
|
||
let project = state
|
||
.spacetime_client()
|
||
.get_editor_project(EditorProjectGetRecordInput {
|
||
project_id: conversation.project_id.clone(),
|
||
owner_user_id: conversation.owner_user_id.clone(),
|
||
})
|
||
.await
|
||
.map_err(map_editor_project_error)
|
||
.map_err(|error| EditorAgentToolExecutionError {
|
||
tool_name: tool_call.tool_name,
|
||
model: tool_call.model.clone(),
|
||
error,
|
||
})?;
|
||
let completion =
|
||
build_editor_agent_canvas_completion(&project, tool_call.tool_name, &tool_call.prompt);
|
||
let caller = EditorGenerationCaller {
|
||
owner_user_id: conversation.owner_user_id.clone(),
|
||
audit_subject_user_id: Some(conversation.owner_user_id.clone()),
|
||
audit_project_id: Some(conversation.project_id.clone()),
|
||
};
|
||
let tool_request_context = editor_agent_tool_request_context(request_context, tool_call_id);
|
||
let attachment_sources = editor_agent_attachment_sources(user_message.attachments.as_slice());
|
||
let previous_generation_source = (tool_call.tool_name == EditorAgentToolName::EditImage
|
||
&& user_message.attachments.is_empty())
|
||
.then(|| latest_generation_reference.and_then(editor_agent_generated_reference_source))
|
||
.flatten();
|
||
let tool_reference_sources = previous_generation_source
|
||
.clone()
|
||
.map(|source| vec![source])
|
||
.unwrap_or_else(|| attachment_sources.clone());
|
||
let generation_reference_context = editor_agent_tool_generation_reference_context(
|
||
user_message.attachments.as_slice(),
|
||
latest_generation_reference,
|
||
previous_generation_source.as_deref(),
|
||
);
|
||
let generation_inputs = Some(json!({
|
||
"source": "editor-agent",
|
||
"conversationId": conversation.conversation_id,
|
||
"messageId": user_message.id,
|
||
"toolCallId": tool_call_id,
|
||
"fields": [{ "title": "用户指令", "value": tool_call.prompt }],
|
||
"references": generation_reference_context,
|
||
}));
|
||
|
||
let tool_name = tool_call.tool_name;
|
||
let tool_model = tool_call.model.clone();
|
||
let response = match tool_name {
|
||
EditorAgentToolName::GenerateImage
|
||
| EditorAgentToolName::GenerateCharacter
|
||
| EditorAgentToolName::GenerateUiDesign => {
|
||
let kind = match tool_name {
|
||
EditorAgentToolName::GenerateCharacter => Some("character".to_string()),
|
||
EditorAgentToolName::GenerateUiDesign => Some("ui-design".to_string()),
|
||
_ => None,
|
||
};
|
||
generate_editor_image_for_owner(
|
||
state,
|
||
&tool_request_context,
|
||
caller,
|
||
EditorImageGenerationRequest {
|
||
prompt: tool_call.prompt.clone(),
|
||
size: None,
|
||
kind,
|
||
model: tool_call.model.clone(),
|
||
screen_color: Some("auto".to_string()),
|
||
seg_model: Some("birefnet".to_string()),
|
||
aspect_ratio: tool_call.aspect_ratio.clone(),
|
||
image_size: tool_call.image_size.clone(),
|
||
reference_image_srcs: Some(tool_reference_sources.clone()),
|
||
project_id: Some(conversation.project_id.clone()),
|
||
asset_kind: Some(editor_agent_tool_asset_kind(tool_name).to_string()),
|
||
generation_inputs,
|
||
asset_folder_id: Some(editor_agent_default_asset_folder_id()),
|
||
asset_label: tool_call.summary.clone(),
|
||
source_resource_id: editor_agent_tool_source_resource_id(
|
||
user_message.attachments.as_slice(),
|
||
latest_generation_reference,
|
||
previous_generation_source.as_deref(),
|
||
),
|
||
canvas_completion: Some(completion),
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
EditorAgentToolName::EditImage => {
|
||
let (source_image_src, reference_image_srcs) = split_editor_agent_edit_sources(
|
||
tool_reference_sources.clone(),
|
||
)
|
||
.ok_or_else(|| EditorAgentToolExecutionError {
|
||
tool_name,
|
||
model: tool_model.clone(),
|
||
error: editor_agent_bad_request("修改图片需要先选择一张画布或素材库图片附件"),
|
||
})?;
|
||
edit_editor_image_for_owner(
|
||
state,
|
||
&tool_request_context,
|
||
caller,
|
||
EditorImageEditRequest {
|
||
prompt: tool_call.prompt.clone(),
|
||
source_image_src,
|
||
size: None,
|
||
model: tool_call.model.clone(),
|
||
reference_image_srcs: Some(reference_image_srcs),
|
||
project_id: Some(conversation.project_id.clone()),
|
||
asset_kind: Some(editor_agent_tool_asset_kind(tool_name).to_string()),
|
||
generation_inputs,
|
||
asset_folder_id: Some(editor_agent_default_asset_folder_id()),
|
||
asset_label: tool_call.summary.clone(),
|
||
source_resource_id: editor_agent_tool_source_resource_id(
|
||
user_message.attachments.as_slice(),
|
||
latest_generation_reference,
|
||
previous_generation_source.as_deref(),
|
||
),
|
||
target_layer_id: None,
|
||
canvas_completion: Some(completion),
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
EditorAgentToolName::GenerateIconSpritesheet => {
|
||
let (reference_image_src, extra_references) =
|
||
split_editor_agent_edit_sources(attachment_sources).ok_or_else(|| {
|
||
EditorAgentToolExecutionError {
|
||
tool_name,
|
||
model: tool_model.clone(),
|
||
error: editor_agent_bad_request("生成图标素材需要先选择一张图标规范参考图"),
|
||
}
|
||
})?;
|
||
let icon_descriptions = if tool_call.icon_descriptions.is_empty() {
|
||
derive_icon_descriptions(tool_call.prompt.as_str())
|
||
} else {
|
||
tool_call.icon_descriptions.clone()
|
||
};
|
||
generate_editor_icon_spritesheet_for_owner(
|
||
state,
|
||
&tool_request_context,
|
||
caller,
|
||
EditorIconSpritesheetGenerationRequest {
|
||
reference_image_src,
|
||
reference_image_srcs: Some(extra_references),
|
||
icon_descriptions,
|
||
model: tool_call.model.clone(),
|
||
screen_color: Some("auto".to_string()),
|
||
seg_model: Some("birefnet".to_string()),
|
||
aspect_ratio: tool_call.aspect_ratio.clone(),
|
||
image_size: tool_call.image_size.clone(),
|
||
project_id: Some(conversation.project_id.clone()),
|
||
generation_inputs,
|
||
asset_folder_id: Some(editor_agent_default_asset_folder_id()),
|
||
asset_label: None,
|
||
canvas_completion: Some(completion),
|
||
},
|
||
)
|
||
.await
|
||
}
|
||
}
|
||
.map_err(|error| EditorAgentToolExecutionError {
|
||
tool_name,
|
||
model: tool_model.clone(),
|
||
error,
|
||
})?;
|
||
|
||
Ok(build_editor_agent_tool_execution(
|
||
tool_call_id,
|
||
tool_name,
|
||
tool_call.summary,
|
||
response.0,
|
||
))
|
||
}
|
||
|
||
fn editor_agent_tool_request_context(base: &RequestContext, tool_call_id: &str) -> RequestContext {
|
||
RequestContext::new(
|
||
format!("{}:{tool_call_id}", base.request_id()),
|
||
format!("{} editor-agent-tool", base.operation()),
|
||
std::time::Duration::ZERO,
|
||
false,
|
||
)
|
||
}
|
||
|
||
fn build_editor_agent_tool_execution(
|
||
tool_call_id: &str,
|
||
tool_name: EditorAgentToolName,
|
||
summary: Option<String>,
|
||
response_body: Value,
|
||
) -> EditorAgentToolExecution {
|
||
let data = response_body
|
||
.get("data")
|
||
.filter(|_| response_body.get("ok").and_then(Value::as_bool) == Some(true))
|
||
.unwrap_or(&response_body);
|
||
let images = editor_agent_generated_images_from_response(tool_name, data);
|
||
let task_id = data
|
||
.get("taskId")
|
||
.and_then(Value::as_str)
|
||
.map(str::to_string)
|
||
.or_else(|| {
|
||
images
|
||
.iter()
|
||
.find_map(|image| image.resource_id.as_ref().map(|_| ()))
|
||
.and_then(|_| {
|
||
data.get("taskId")
|
||
.and_then(Value::as_str)
|
||
.map(str::to_string)
|
||
})
|
||
});
|
||
let model = data
|
||
.get("model")
|
||
.and_then(Value::as_str)
|
||
.map(str::to_string);
|
||
let resolved_summary = summary.unwrap_or_else(|| editor_agent_tool_default_summary(tool_name));
|
||
|
||
EditorAgentToolExecution {
|
||
record: EditorAgentGenerationRecord {
|
||
tool_call_id: tool_call_id.to_string(),
|
||
tool_name,
|
||
summary: Some(resolved_summary.clone()),
|
||
task_id,
|
||
status: EditorAgentGenerationStatus::Completed,
|
||
model,
|
||
images,
|
||
error: None,
|
||
},
|
||
summary: resolved_summary,
|
||
}
|
||
}
|
||
|
||
fn editor_agent_generated_images_from_response(
|
||
tool_name: EditorAgentToolName,
|
||
data: &Value,
|
||
) -> Vec<EditorAgentGeneratedImage> {
|
||
if tool_name == EditorAgentToolName::GenerateIconSpritesheet {
|
||
let mut images = Vec::new();
|
||
if let Some(resource) = data.get("spritesheetResource") {
|
||
images.push(editor_agent_generated_image_from_resource(
|
||
resource,
|
||
data.get("spritesheetImageSrc").and_then(Value::as_str),
|
||
data.get("spritesheetWidth").and_then(Value::as_u64),
|
||
data.get("spritesheetHeight").and_then(Value::as_u64),
|
||
));
|
||
}
|
||
if let Some(icon_images) = data.get("iconImageSrcs").and_then(Value::as_array) {
|
||
for icon in icon_images {
|
||
if let Some(resource) = icon.get("resource") {
|
||
images.push(editor_agent_generated_image_from_resource(
|
||
resource,
|
||
icon.get("imageSrc").and_then(Value::as_str),
|
||
icon.get("width").and_then(Value::as_u64),
|
||
icon.get("height").and_then(Value::as_u64),
|
||
));
|
||
}
|
||
}
|
||
}
|
||
return images;
|
||
}
|
||
|
||
data.get("resource")
|
||
.map(|resource| {
|
||
vec![editor_agent_generated_image_from_resource(
|
||
resource,
|
||
data.get("imageSrc").and_then(Value::as_str),
|
||
data.get("width").and_then(Value::as_u64),
|
||
data.get("height").and_then(Value::as_u64),
|
||
)]
|
||
})
|
||
.unwrap_or_else(|| {
|
||
data.get("imageSrc")
|
||
.and_then(Value::as_str)
|
||
.map(|image_src| {
|
||
vec![EditorAgentGeneratedImage {
|
||
resource_id: None,
|
||
object_key: None,
|
||
asset_object_id: None,
|
||
image_src: image_src.to_string(),
|
||
thumbnail_src: None,
|
||
width: data
|
||
.get("width")
|
||
.and_then(Value::as_u64)
|
||
.and_then(|value| u32::try_from(value).ok()),
|
||
height: data
|
||
.get("height")
|
||
.and_then(Value::as_u64)
|
||
.and_then(|value| u32::try_from(value).ok()),
|
||
}]
|
||
})
|
||
.unwrap_or_default()
|
||
})
|
||
}
|
||
|
||
fn editor_agent_generated_image_from_resource(
|
||
resource: &Value,
|
||
fallback_image_src: Option<&str>,
|
||
fallback_width: Option<u64>,
|
||
fallback_height: Option<u64>,
|
||
) -> EditorAgentGeneratedImage {
|
||
EditorAgentGeneratedImage {
|
||
resource_id: resource
|
||
.get("resourceId")
|
||
.and_then(Value::as_str)
|
||
.map(str::to_string),
|
||
object_key: resource
|
||
.get("objectKey")
|
||
.and_then(Value::as_str)
|
||
.map(str::to_string),
|
||
asset_object_id: resource
|
||
.get("assetObjectId")
|
||
.and_then(Value::as_str)
|
||
.map(str::to_string),
|
||
image_src: resource
|
||
.get("imageSrc")
|
||
.and_then(Value::as_str)
|
||
.or(fallback_image_src)
|
||
.unwrap_or("")
|
||
.to_string(),
|
||
thumbnail_src: resource
|
||
.get("thumbnailSrc")
|
||
.and_then(Value::as_str)
|
||
.map(str::to_string),
|
||
width: resource
|
||
.get("width")
|
||
.and_then(Value::as_u64)
|
||
.or(fallback_width)
|
||
.and_then(|value| u32::try_from(value).ok()),
|
||
height: resource
|
||
.get("height")
|
||
.and_then(Value::as_u64)
|
||
.or(fallback_height)
|
||
.and_then(|value| u32::try_from(value).ok()),
|
||
}
|
||
}
|
||
|
||
fn editor_agent_attachment_sources(attachments: &[EditorAgentAttachmentRef]) -> Vec<String> {
|
||
attachments
|
||
.iter()
|
||
.filter_map(|attachment| {
|
||
attachment
|
||
.object_key
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
.map(|value| value.trim_start_matches('/').to_string())
|
||
.or_else(|| normalize_required_string(attachment.reference_id.as_str()))
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
fn editor_agent_tool_generation_reference_context(
|
||
attachments: &[EditorAgentAttachmentRef],
|
||
latest_generation_reference: Option<&EditorAgentGeneratedImageReference>,
|
||
previous_generation_source: Option<&str>,
|
||
) -> Vec<Value> {
|
||
if previous_generation_source.is_some() {
|
||
return latest_generation_reference
|
||
.map(|reference| {
|
||
vec![json!({
|
||
"title": reference
|
||
.summary
|
||
.clone()
|
||
.unwrap_or_else(|| "上一张生成图".to_string()),
|
||
"label": reference.summary,
|
||
"refType": "previous-generation",
|
||
"refId": reference.resource_id,
|
||
"resourceId": reference.resource_id,
|
||
"objectKey": reference.object_key,
|
||
"assetObjectId": reference.asset_object_id,
|
||
"toolCallId": reference.tool_call_id,
|
||
"toolName": reference.tool_name,
|
||
"sourceMessageId": reference.source_message_id,
|
||
"source": previous_generation_source,
|
||
"implicit": true,
|
||
})]
|
||
})
|
||
.unwrap_or_default();
|
||
}
|
||
|
||
attachments
|
||
.iter()
|
||
.map(|attachment| {
|
||
json!({
|
||
"title": attachment.label.clone().unwrap_or_else(|| "对话附件".to_string()),
|
||
"label": attachment.label,
|
||
"refType": match attachment.source {
|
||
EditorAgentAttachmentSource::CanvasResource => "project-resource",
|
||
EditorAgentAttachmentSource::LibraryAsset => "asset",
|
||
},
|
||
"refId": attachment.reference_id,
|
||
"resourceId": (attachment.source == EditorAgentAttachmentSource::CanvasResource)
|
||
.then(|| attachment.reference_id.clone()),
|
||
"objectKey": attachment.object_key,
|
||
})
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
fn editor_agent_tool_source_resource_id(
|
||
attachments: &[EditorAgentAttachmentRef],
|
||
latest_generation_reference: Option<&EditorAgentGeneratedImageReference>,
|
||
previous_generation_source: Option<&str>,
|
||
) -> Option<String> {
|
||
if previous_generation_source.is_some() {
|
||
return latest_generation_reference.and_then(|reference| reference.resource_id.clone());
|
||
}
|
||
attachments
|
||
.first()
|
||
.map(|attachment| attachment.reference_id.clone())
|
||
}
|
||
|
||
fn split_editor_agent_edit_sources(sources: Vec<String>) -> Option<(String, Vec<String>)> {
|
||
let mut iter = sources.into_iter();
|
||
let first = iter.next()?;
|
||
Some((first, iter.collect()))
|
||
}
|
||
|
||
fn editor_agent_tool_asset_kind(tool_name: EditorAgentToolName) -> &'static str {
|
||
match tool_name {
|
||
EditorAgentToolName::GenerateCharacter => "character",
|
||
EditorAgentToolName::GenerateUiDesign => "ui-design",
|
||
EditorAgentToolName::EditImage => "editor_agent_edit_image",
|
||
EditorAgentToolName::GenerateIconSpritesheet => "icon-spritesheet",
|
||
EditorAgentToolName::GenerateImage => "editor_agent_generated_image",
|
||
}
|
||
}
|
||
|
||
fn editor_agent_default_asset_folder_id() -> String {
|
||
"project".to_string()
|
||
}
|
||
|
||
fn editor_agent_tool_default_summary(tool_name: EditorAgentToolName) -> String {
|
||
match tool_name {
|
||
EditorAgentToolName::GenerateImage => "生成图片",
|
||
EditorAgentToolName::EditImage => "修改图片",
|
||
EditorAgentToolName::GenerateCharacter => "生成角色形象",
|
||
EditorAgentToolName::GenerateIconSpritesheet => "生成图标素材",
|
||
EditorAgentToolName::GenerateUiDesign => "生成 UI 设计图",
|
||
}
|
||
.to_string()
|
||
}
|
||
|
||
fn build_editor_agent_canvas_completion(
|
||
project: &EditorProjectRecord,
|
||
tool_name: EditorAgentToolName,
|
||
title: &str,
|
||
) -> EditorCanvasGenerationCompletionPayload {
|
||
let (width, height) = editor_agent_tool_display_size(tool_name);
|
||
let (x, y) = next_editor_agent_canvas_position(project.layers.clone(), width, height);
|
||
EditorCanvasGenerationCompletionPayload {
|
||
dialog_id: None,
|
||
title: normalize_optional_string(Some(title.to_string()))
|
||
.unwrap_or_else(|| editor_agent_tool_default_summary(tool_name)),
|
||
placeholder: EditorCanvasGenerationPlaceholderPayload {
|
||
x,
|
||
y,
|
||
width,
|
||
height,
|
||
original_width: width,
|
||
original_height: height,
|
||
},
|
||
}
|
||
}
|
||
|
||
fn editor_agent_tool_display_size(tool_name: EditorAgentToolName) -> (f64, f64) {
|
||
match tool_name {
|
||
EditorAgentToolName::GenerateUiDesign => (640.0, 360.0),
|
||
EditorAgentToolName::GenerateCharacter => (512.0, 768.0),
|
||
_ => (512.0, 512.0),
|
||
}
|
||
}
|
||
|
||
fn next_editor_agent_canvas_position(layers: Value, _width: f64, _height: f64) -> (f64, f64) {
|
||
let mut max_right: Option<f64> = None;
|
||
let mut min_y: Option<f64> = None;
|
||
if let Value::Array(items) = layers {
|
||
for item in items {
|
||
if item.get("itemType").and_then(Value::as_str) == Some("generation-dialog") {
|
||
continue;
|
||
}
|
||
let Some(x) = item.get("x").and_then(Value::as_f64) else {
|
||
continue;
|
||
};
|
||
let Some(y) = item.get("y").and_then(Value::as_f64) else {
|
||
continue;
|
||
};
|
||
let width = item.get("width").and_then(Value::as_f64).unwrap_or(0.0);
|
||
if !x.is_finite() || !y.is_finite() || !width.is_finite() {
|
||
continue;
|
||
}
|
||
max_right = Some(max_right.map_or(x + width, |value| value.max(x + width)));
|
||
min_y = Some(min_y.map_or(y, |value| value.min(y)));
|
||
}
|
||
}
|
||
(
|
||
max_right.map_or(0.0, |right| right + EDITOR_AGENT_CANVAS_RESULT_GAP),
|
||
min_y.unwrap_or(0.0),
|
||
)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
fn build_echo_assistant_text(text: &str, attachment_count: usize) -> String {
|
||
let trimmed = text.trim();
|
||
if trimmed.is_empty() {
|
||
return format!("已收到 {attachment_count} 个附件。");
|
||
}
|
||
format!("收到：{trimmed}")
|
||
}
|
||
|
||
fn now_rfc3339() -> String {
|
||
shared_kernel::format_rfc3339(time::OffsetDateTime::now_utc())
|
||
.unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_string())
|
||
}
|
||
|
||
async fn require_editor_agent_sidebar_enabled(
|
||
state: &AppState,
|
||
owner_user_id: &str,
|
||
) -> Result<(), AppError> {
|
||
match state
|
||
.is_image_editor_agent_sidebar_enabled_for_user(Some(owner_user_id))
|
||
.await
|
||
{
|
||
Ok(true) => Ok(()),
|
||
Ok(false) => Err(editor_agent_sidebar_unavailable()),
|
||
Err(error) => Err(AppError::from_status(StatusCode::BAD_GATEWAY)
|
||
.with_message("读取画布 Agent 灰度配置失败")
|
||
.with_details(json!({
|
||
"provider": "spacetimedb",
|
||
"message": error.to_string(),
|
||
}))),
|
||
}
|
||
}
|
||
|
||
fn editor_agent_sidebar_unavailable() -> AppError {
|
||
AppError::from_status(StatusCode::SERVICE_UNAVAILABLE)
|
||
.with_message("画布 Agent 暂不可用")
|
||
.with_details(json!({
|
||
"provider": "editor-agent",
|
||
"reason": "image_editor_agent_sidebar_disabled",
|
||
"gateKey": module_runtime::IMAGE_EDITOR_AGENT_SIDEBAR_GATE_KEY,
|
||
}))
|
||
}
|
||
|
||
fn editor_agent_bad_request(message: impl Into<String>) -> AppError {
|
||
AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({
|
||
"provider": "editor-agent",
|
||
"message": message.into(),
|
||
}))
|
||
}
|
||
|
||
fn editor_agent_oss_unavailable() -> AppError {
|
||
AppError::from_status(StatusCode::SERVICE_UNAVAILABLE).with_details(json!({
|
||
"provider": "aliyun-oss",
|
||
"reason": "OSS is not configured for editor agent conversations",
|
||
}))
|
||
}
|
||
|
||
fn editor_agent_oss_read_error(message: impl Into<String>) -> AppError {
|
||
AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({
|
||
"provider": "aliyun-oss",
|
||
"message": message.into(),
|
||
}))
|
||
}
|
||
|
||
fn editor_agent_messages_document_too_large() -> AppError {
|
||
AppError::from_status(StatusCode::PAYLOAD_TOO_LARGE).with_details(json!({
|
||
"provider": "editor-agent",
|
||
"message": "message document is too large",
|
||
"maxBytes": EDITOR_AGENT_MESSAGES_DOCUMENT_MAX_BYTES,
|
||
}))
|
||
}
|
||
|
||
fn editor_agent_sse_json_event_or_error<T>(event_name: &str, payload: T) -> Event
|
||
where
|
||
T: Serialize,
|
||
{
|
||
match serde_json::to_string(&payload) {
|
||
Ok(data) => Event::default().event(event_name).data(data),
|
||
Err(error) => Event::default().event("error").data(
|
||
serde_json::to_string(&EditorAgentErrorEvent {
|
||
conversation_id: None,
|
||
code: "SERIALIZE_EVENT_FAILED".to_string(),
|
||
message: error.to_string(),
|
||
recoverable: false,
|
||
})
|
||
.unwrap_or_else(|_| "{}".to_string()),
|
||
),
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use crate::AppConfig;
|
||
use platform_llm::LlmApiKind;
|
||
|
||
#[test]
|
||
fn echo_assistant_text_uses_attachment_fallback() {
|
||
assert_eq!(
|
||
build_echo_assistant_text(" 画一棵树 ", 0),
|
||
"收到：画一棵树"
|
||
);
|
||
assert_eq!(build_echo_assistant_text(" ", 2), "已收到 2 个附件。");
|
||
}
|
||
|
||
#[test]
|
||
fn empty_document_uses_contract_version() {
|
||
let document = empty_messages_document("editor-agent-conv-1");
|
||
assert_eq!(document.version, EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION);
|
||
assert_eq!(document.conversation_id, "editor-agent-conv-1");
|
||
assert!(document.messages.is_empty());
|
||
}
|
||
|
||
#[test]
|
||
fn canvas_attachment_normalization_uses_project_resource_truth() {
|
||
let conversation = test_conversation_record();
|
||
let attachment = EditorAgentAttachmentRef {
|
||
source: EditorAgentAttachmentSource::CanvasResource,
|
||
reference_id: " resource-1 ".to_string(),
|
||
object_key: Some("generated-editor-assets/resource-1.png".to_string()),
|
||
image_src: "https://client.example/stale.png".to_string(),
|
||
thumbnail_src: Some("https://client.example/thumb.png".to_string()),
|
||
label: Some(" 参考图 ".to_string()),
|
||
width: Some(1),
|
||
height: Some(1),
|
||
};
|
||
let resource = test_project_resource_record();
|
||
|
||
let normalized =
|
||
normalize_canvas_resource_attachment(&conversation, &attachment, &resource)
|
||
.expect("attachment should normalize");
|
||
|
||
assert_eq!(normalized.reference_id, "resource-1");
|
||
assert_eq!(
|
||
normalized.object_key.as_deref(),
|
||
Some("generated-editor-assets/resource-1.png")
|
||
);
|
||
assert_eq!(normalized.image_src, "/api/assets/read/resource-1.png");
|
||
assert_eq!(normalized.thumbnail_src, None);
|
||
assert_eq!(normalized.label.as_deref(), Some("参考图"));
|
||
assert_eq!(normalized.width, Some(512));
|
||
assert_eq!(normalized.height, Some(256));
|
||
}
|
||
|
||
#[test]
|
||
fn attachment_normalization_rejects_mismatched_object_key() {
|
||
let conversation = test_conversation_record();
|
||
let attachment = EditorAgentAttachmentRef {
|
||
source: EditorAgentAttachmentSource::CanvasResource,
|
||
reference_id: "resource-1".to_string(),
|
||
object_key: Some("generated-editor-assets/other.png".to_string()),
|
||
image_src: "/api/assets/read/resource-1.png".to_string(),
|
||
thumbnail_src: None,
|
||
label: None,
|
||
width: None,
|
||
height: None,
|
||
};
|
||
let resource = test_project_resource_record();
|
||
|
||
assert!(
|
||
normalize_canvas_resource_attachment(&conversation, &attachment, &resource).is_err()
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn library_attachment_normalization_uses_asset_label_fallback() {
|
||
let attachment = EditorAgentAttachmentRef {
|
||
source: EditorAgentAttachmentSource::LibraryAsset,
|
||
reference_id: "asset-1".to_string(),
|
||
object_key: None,
|
||
image_src: "https://client.example/stale.png".to_string(),
|
||
thumbnail_src: None,
|
||
label: None,
|
||
width: None,
|
||
height: None,
|
||
};
|
||
let asset = EditorAssetRecord {
|
||
asset_id: "asset-1".to_string(),
|
||
folder_id: "project".to_string(),
|
||
label: "素材库图".to_string(),
|
||
asset_object_id: Some("asset-object-1".to_string()),
|
||
image_src: "/api/assets/read/asset-1.png".to_string(),
|
||
object_key: Some("generated-editor-assets/asset-1.png".to_string()),
|
||
width: 128,
|
||
height: 128,
|
||
source_type: "uploaded".to_string(),
|
||
prompt: None,
|
||
actual_prompt: None,
|
||
model: None,
|
||
provider: None,
|
||
task_id: None,
|
||
asset_kind: None,
|
||
generation_inputs: None,
|
||
source_resource_id: None,
|
||
public_showcase_enabled: None,
|
||
thumbnail_src: Some("/api/assets/read/asset-1-thumb.png".to_string()),
|
||
generation_cost_mud_points: 0,
|
||
showcase_id: None,
|
||
showcase_review_status: None,
|
||
showcase_display_enabled: None,
|
||
showcase_like_count: None,
|
||
created_at: "2026-07-03T00:00:00Z".to_string(),
|
||
updated_at: "2026-07-03T00:00:00Z".to_string(),
|
||
};
|
||
|
||
let normalized = normalize_library_asset_attachment(&attachment, &asset)
|
||
.expect("library attachment should normalize");
|
||
|
||
assert_eq!(normalized.label.as_deref(), Some("素材库图"));
|
||
assert_eq!(
|
||
normalized.thumbnail_src.as_deref(),
|
||
Some("/api/assets/read/asset-1-thumb.png")
|
||
);
|
||
assert_eq!(normalized.width, Some(128));
|
||
assert_eq!(normalized.height, Some(128));
|
||
}
|
||
|
||
#[test]
|
||
fn generated_image_from_resource_keeps_private_asset_keys() {
|
||
let image = editor_agent_generated_image_from_resource(
|
||
&json!({
|
||
"resourceId": "resource-1",
|
||
"imageSrc": "/generated-editor-assets/result.png",
|
||
"thumbnailSrc": "/generated-editor-assets/result-thumb.png",
|
||
"objectKey": "generated-editor-assets/result.png",
|
||
"assetObjectId": "asset-object-result",
|
||
"width": 1024,
|
||
"height": 768
|
||
}),
|
||
None,
|
||
None,
|
||
None,
|
||
);
|
||
|
||
assert_eq!(image.resource_id.as_deref(), Some("resource-1"));
|
||
assert_eq!(
|
||
image.object_key.as_deref(),
|
||
Some("generated-editor-assets/result.png")
|
||
);
|
||
assert_eq!(
|
||
image.asset_object_id.as_deref(),
|
||
Some("asset-object-result")
|
||
);
|
||
assert_eq!(image.width, Some(1024));
|
||
assert_eq!(image.height, Some(768));
|
||
}
|
||
|
||
#[test]
|
||
fn editor_agent_generations_target_default_asset_library_folder() {
|
||
assert_eq!(editor_agent_default_asset_folder_id(), "project");
|
||
}
|
||
|
||
#[test]
|
||
fn editor_agent_turn_plan_parses_fenced_json_tool_call() {
|
||
let plan = parse_editor_agent_turn_plan(
|
||
r#"```json
|
||
{
|
||
"replyText": "我来生成一张像素风角色图。",
|
||
"toolCall": {
|
||
"toolName": "generate_character",
|
||
"prompt": "像素风银发游侠",
|
||
"summary": "生成像素角色",
|
||
"imageSize": "1K"
|
||
}
|
||
}
|
||
```"#,
|
||
)
|
||
.expect("fenced JSON should parse");
|
||
|
||
assert_eq!(plan.reply_text, "我来生成一张像素风角色图。");
|
||
let tool_call = plan.tool_call.expect("tool call should exist");
|
||
assert_eq!(tool_call.tool_name, EditorAgentToolName::GenerateCharacter);
|
||
assert_eq!(tool_call.prompt, "像素风银发游侠");
|
||
assert_eq!(tool_call.summary.as_deref(), Some("生成像素角色"));
|
||
assert_eq!(tool_call.image_size.as_deref(), Some("1K"));
|
||
}
|
||
|
||
#[test]
|
||
fn editor_agent_turn_plan_rejects_invalid_llm_response() {
|
||
assert!(parse_editor_agent_turn_plan("收到：画一棵树").is_none());
|
||
assert!(
|
||
parse_editor_agent_turn_plan(
|
||
r#"{"toolCall":{"toolName":"generate_image","prompt":"画一棵树"}}"#,
|
||
)
|
||
.is_none()
|
||
);
|
||
assert!(
|
||
parse_editor_agent_turn_plan(
|
||
r#"{"replyText":"我来生成图片。","toolCall":{"toolName":"generate_image"}}"#,
|
||
)
|
||
.is_none()
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn editor_agent_turn_plan_replaces_structural_reply_fragment() {
|
||
let plan = parse_editor_agent_turn_plan(
|
||
r#"{"replyText":"{","toolCall":{"toolName":"generate_image","prompt":"生成一张角色规范图","summary":"生成角色规范图"}}"#,
|
||
)
|
||
.expect("valid tool call should parse");
|
||
|
||
assert_eq!(plan.reply_text, "我来生成图片。");
|
||
assert_eq!(
|
||
plan.tool_call.as_ref().map(|tool_call| tool_call.tool_name),
|
||
Some(EditorAgentToolName::GenerateImage)
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn editor_agent_llm_system_prompt_describes_each_tool() {
|
||
let prompt = editor_agent_llm_system_prompt();
|
||
|
||
for expected in [
|
||
"generate_image：从文字生成一张全新图片",
|
||
"edit_image：修改已有图片",
|
||
"generate_character：生成新的角色形象",
|
||
"generate_icon_spritesheet：生成一组图标素材或图标图集",
|
||
"generate_ui_design：生成一张完整 UI 设计图或界面稿",
|
||
"规范图/视觉规范图/风格规范图/素材规范展板",
|
||
"当前使用 generate_image",
|
||
"角色规范图/角色美术视觉规范设定图",
|
||
"图标规范图/图标视觉规范展板",
|
||
"统一视角、线条粗细",
|
||
"色卡/色号",
|
||
"工具选择优先级",
|
||
"明确修改/指代已有图 => edit_image",
|
||
"规范图/视觉规范图/风格规范图/素材规范展板 => generate_image",
|
||
] {
|
||
assert!(
|
||
prompt.contains(expected),
|
||
"system prompt should contain {expected}"
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn latest_generation_reference_enters_llm_prompt() {
|
||
let conversation = test_conversation_record();
|
||
let user_message = test_user_message("把刚才那个衣服换成蓝色");
|
||
let mut document = empty_messages_document(conversation.conversation_id.as_str());
|
||
document.messages.push(EditorAgentMessage {
|
||
id: "editor-agent-message-assistant-1".to_string(),
|
||
role: EditorAgentMessageRole::Assistant,
|
||
kind: EditorAgentMessageKind::Chat,
|
||
text: "已生成角色图。".to_string(),
|
||
attachments: Vec::new(),
|
||
generations: vec![test_completed_generation_record()],
|
||
status: EditorAgentMessageStatus::Completed,
|
||
created_at: "2026-07-05T00:00:00Z".to_string(),
|
||
});
|
||
document.messages.push(user_message.clone());
|
||
|
||
let reference = latest_editor_agent_generated_image_reference(&document)
|
||
.expect("latest generated image should be available");
|
||
let prompt = build_editor_agent_llm_user_prompt(
|
||
&conversation,
|
||
&document,
|
||
&user_message,
|
||
Some(&reference),
|
||
);
|
||
let prompt_json: Value = serde_json::from_str(prompt.as_str()).expect("prompt is JSON");
|
||
|
||
assert_eq!(
|
||
prompt_json["latestGeneratedImage"]["toolName"],
|
||
json!("generate_image")
|
||
);
|
||
assert_eq!(
|
||
prompt_json["latestGeneratedImage"]["summary"],
|
||
json!("生成红衣角色")
|
||
);
|
||
assert_eq!(
|
||
prompt_json["latestGeneratedImage"]["resourceId"],
|
||
json!("resource-generated-1")
|
||
);
|
||
assert_eq!(
|
||
prompt_json["latestGeneratedImage"]["objectKey"],
|
||
json!("generated-editor-assets/result.png")
|
||
);
|
||
assert_eq!(
|
||
prompt_json["recentMessages"][0]["generations"][0]["images"][0]["objectKey"],
|
||
json!("generated-editor-assets/result.png")
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn previous_generation_edit_default_coerces_generate_image_to_edit_image() {
|
||
let user_message = test_user_message("把衣服换成蓝色");
|
||
let reference = test_generated_image_reference();
|
||
let plan = EditorAgentTurnPlan {
|
||
reply_text: "我来生成图片。".to_string(),
|
||
tool_call: Some(EditorAgentToolCallPlan {
|
||
tool_name: EditorAgentToolName::GenerateImage,
|
||
prompt: "把衣服换成蓝色".to_string(),
|
||
summary: Some("生成蓝色衣服".to_string()),
|
||
model: None,
|
||
aspect_ratio: Some("1:1".to_string()),
|
||
image_size: Some("1K".to_string()),
|
||
icon_descriptions: Vec::new(),
|
||
}),
|
||
};
|
||
|
||
let adjusted =
|
||
apply_previous_generation_edit_default(plan, &user_message, Some(&reference));
|
||
let tool_call = adjusted.tool_call.expect("tool should be forced");
|
||
|
||
assert_eq!(tool_call.tool_name, EditorAgentToolName::EditImage);
|
||
assert_eq!(tool_call.prompt, "把衣服换成蓝色");
|
||
assert_eq!(adjusted.reply_text, "我会基于上一张生成图按你的要求修改。");
|
||
}
|
||
|
||
#[test]
|
||
fn previous_generation_reference_becomes_implicit_edit_source() {
|
||
let reference = test_generated_image_reference();
|
||
|
||
let sources = editor_agent_generated_reference_source(&reference);
|
||
let context = editor_agent_tool_generation_reference_context(
|
||
&[],
|
||
Some(&reference),
|
||
sources.as_deref(),
|
||
);
|
||
|
||
assert_eq!(
|
||
sources.as_deref(),
|
||
Some("generated-editor-assets/result.png")
|
||
);
|
||
assert_eq!(context[0]["refType"], json!("previous-generation"));
|
||
assert_eq!(context[0]["resourceId"], json!("resource-generated-1"));
|
||
assert_eq!(
|
||
context[0]["objectKey"],
|
||
json!("generated-editor-assets/result.png")
|
||
);
|
||
assert_eq!(context[0]["implicit"], json!(true));
|
||
}
|
||
|
||
#[test]
|
||
fn editor_agent_tool_execution_persists_summary() {
|
||
let execution = build_editor_agent_tool_execution(
|
||
"editor-agent-tool-1",
|
||
EditorAgentToolName::EditImage,
|
||
Some("换蓝色衣服".to_string()),
|
||
json!({
|
||
"ok": true,
|
||
"data": {
|
||
"model": "gpt-image-2",
|
||
"resource": {
|
||
"resourceId": "resource-generated-2",
|
||
"imageSrc": "/generated-editor-assets/blue.png",
|
||
"objectKey": "generated-editor-assets/blue.png",
|
||
"assetObjectId": "asset-object-blue",
|
||
"width": 1024,
|
||
"height": 1024
|
||
}
|
||
}
|
||
}),
|
||
);
|
||
|
||
assert_eq!(execution.summary, "换蓝色衣服");
|
||
assert_eq!(execution.record.summary.as_deref(), Some("换蓝色衣服"));
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn editor_agent_llm_request_uses_vector_engine_chat_model() {
|
||
let conversation = test_conversation_record();
|
||
let document = empty_messages_document(conversation.conversation_id.as_str());
|
||
let user_message = test_user_message("帮我看看这张图还能怎么改");
|
||
let state = AppState::new(AppConfig::default()).expect("state should build");
|
||
|
||
let request =
|
||
build_editor_agent_llm_request(&state, &conversation, &document, &user_message, None)
|
||
.await;
|
||
|
||
assert_eq!(request.api_kind, LlmApiKind::OpenAiChat);
|
||
assert_eq!(
|
||
request.model.as_deref(),
|
||
Some(platform_agent::CREATIVE_AGENT_GPT5_MODEL)
|
||
);
|
||
assert_eq!(
|
||
request.max_output_tokens,
|
||
Some(EDITOR_AGENT_LLM_MAX_OUTPUT_TOKENS)
|
||
);
|
||
assert_eq!(
|
||
request.request_timeout_ms,
|
||
Some(EDITOR_AGENT_LLM_REQUEST_TIMEOUT_MS)
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn editor_agent_heuristic_routes_tools_from_user_text() {
|
||
let character_plan =
|
||
heuristic_editor_agent_turn_plan("帮我生成一个角色形象：猫耳骑士", &[]);
|
||
assert_eq!(
|
||
character_plan
|
||
.tool_call
|
||
.as_ref()
|
||
.expect("character tool expected")
|
||
.tool_name,
|
||
EditorAgentToolName::GenerateCharacter
|
||
);
|
||
|
||
let edit_plan = heuristic_editor_agent_turn_plan(
|
||
"把附件这张图改成赛博朋克风格",
|
||
&[EditorAgentAttachmentRef {
|
||
source: EditorAgentAttachmentSource::CanvasResource,
|
||
reference_id: "resource-1".to_string(),
|
||
object_key: Some("generated-editor-assets/resource-1.png".to_string()),
|
||
image_src: "/generated-editor-assets/resource-1.png".to_string(),
|
||
thumbnail_src: None,
|
||
label: None,
|
||
width: Some(512),
|
||
height: Some(512),
|
||
}],
|
||
);
|
||
assert_eq!(
|
||
edit_plan
|
||
.tool_call
|
||
.as_ref()
|
||
.expect("edit tool expected")
|
||
.tool_name,
|
||
EditorAgentToolName::EditImage
|
||
);
|
||
|
||
let chat_plan = heuristic_editor_agent_turn_plan("这张图适合做头像吗？", &[]);
|
||
assert!(chat_plan.tool_call.is_none());
|
||
}
|
||
|
||
#[test]
|
||
fn editor_agent_canvas_completion_places_result_after_existing_layers() {
|
||
let project = EditorProjectRecord {
|
||
project_id: "project-1".to_string(),
|
||
owner_user_id: "user-1".to_string(),
|
||
title: "画布".to_string(),
|
||
canvas: spacetime_client::EditorCanvasRecord {
|
||
canvas_id: "canvas-1".to_string(),
|
||
project_id: "project-1".to_string(),
|
||
title: "画布".to_string(),
|
||
viewport: spacetime_client::EditorCanvasViewportRecord {
|
||
x: 0.0,
|
||
y: 0.0,
|
||
scale: 1.0,
|
||
},
|
||
layers: json!([]),
|
||
created_at: "2026-07-03T00:00:00Z".to_string(),
|
||
updated_at: "2026-07-03T00:00:00Z".to_string(),
|
||
},
|
||
viewport: spacetime_client::EditorCanvasViewportRecord {
|
||
x: 0.0,
|
||
y: 0.0,
|
||
scale: 1.0,
|
||
},
|
||
layers: json!([
|
||
{
|
||
"layerId": "layer-1",
|
||
"resourceId": "resource-1",
|
||
"x": 10.0,
|
||
"y": 20.0,
|
||
"width": 180.0,
|
||
"height": 120.0
|
||
}
|
||
]),
|
||
resources: Vec::new(),
|
||
created_at: "2026-07-03T00:00:00Z".to_string(),
|
||
updated_at: "2026-07-03T00:00:00Z".to_string(),
|
||
};
|
||
|
||
let completion = build_editor_agent_canvas_completion(
|
||
&project,
|
||
EditorAgentToolName::GenerateUiDesign,
|
||
"UI 设计图",
|
||
);
|
||
|
||
assert_eq!(completion.dialog_id, None);
|
||
assert_eq!(completion.title, "UI 设计图");
|
||
assert_eq!(completion.placeholder.x, 222.0);
|
||
assert_eq!(completion.placeholder.y, 20.0);
|
||
assert_eq!(completion.placeholder.width, 640.0);
|
||
assert_eq!(completion.placeholder.height, 360.0);
|
||
}
|
||
|
||
fn test_conversation_record() -> EditorAgentConversationRecord {
|
||
EditorAgentConversationRecord {
|
||
conversation_id: "editor-agent-conv-1".to_string(),
|
||
project_id: "project-1".to_string(),
|
||
owner_user_id: "user-1".to_string(),
|
||
title: "新对话".to_string(),
|
||
messages_object_key: "editor-agent/editor-agent-conv-1.json".to_string(),
|
||
deleted: false,
|
||
created_at: "2026-07-03T00:00:00Z".to_string(),
|
||
updated_at: "2026-07-03T00:00:00Z".to_string(),
|
||
updated_at_micros: 1,
|
||
}
|
||
}
|
||
|
||
fn test_project_resource_record() -> EditorProjectResourceRecord {
|
||
EditorProjectResourceRecord {
|
||
resource_id: "resource-1".to_string(),
|
||
project_id: "project-1".to_string(),
|
||
owner_user_id: "user-1".to_string(),
|
||
asset_object_id: Some("asset-object-1".to_string()),
|
||
image_src: "/api/assets/read/resource-1.png".to_string(),
|
||
object_key: Some("generated-editor-assets/resource-1.png".to_string()),
|
||
width: 512,
|
||
height: 256,
|
||
source_type: "generated".to_string(),
|
||
prompt: Some("森林".to_string()),
|
||
actual_prompt: None,
|
||
model: Some("gpt-image-2".to_string()),
|
||
provider: Some("VectorEngine".to_string()),
|
||
task_id: Some("task-1".to_string()),
|
||
source_resource_id: None,
|
||
asset_kind: Some("editor_generated_image".to_string()),
|
||
generation_inputs: None,
|
||
public_showcase_enabled: false,
|
||
created_at: "2026-07-03T00:00:00Z".to_string(),
|
||
updated_at: "2026-07-03T00:00:00Z".to_string(),
|
||
}
|
||
}
|
||
|
||
fn test_user_message(text: &str) -> EditorAgentMessage {
|
||
EditorAgentMessage {
|
||
id: "editor-agent-message-user-1".to_string(),
|
||
role: EditorAgentMessageRole::User,
|
||
kind: EditorAgentMessageKind::Chat,
|
||
text: text.to_string(),
|
||
attachments: Vec::new(),
|
||
generations: Vec::new(),
|
||
status: EditorAgentMessageStatus::Completed,
|
||
created_at: "2026-07-05T00:00:00Z".to_string(),
|
||
}
|
||
}
|
||
|
||
fn test_completed_generation_record() -> EditorAgentGenerationRecord {
|
||
EditorAgentGenerationRecord {
|
||
tool_call_id: "editor-agent-tool-generated-1".to_string(),
|
||
tool_name: EditorAgentToolName::GenerateImage,
|
||
summary: Some("生成红衣角色".to_string()),
|
||
task_id: Some("task-generated-1".to_string()),
|
||
status: EditorAgentGenerationStatus::Completed,
|
||
model: Some("gpt-image-2".to_string()),
|
||
images: vec![EditorAgentGeneratedImage {
|
||
resource_id: Some("resource-generated-1".to_string()),
|
||
object_key: Some("generated-editor-assets/result.png".to_string()),
|
||
asset_object_id: Some("asset-object-result".to_string()),
|
||
image_src: "/generated-editor-assets/result.png".to_string(),
|
||
thumbnail_src: Some("/generated-editor-assets/result-thumb.png".to_string()),
|
||
width: Some(1024),
|
||
height: Some(1024),
|
||
}],
|
||
error: None,
|
||
}
|
||
}
|
||
|
||
fn test_generated_image_reference() -> EditorAgentGeneratedImageReference {
|
||
let mut document = empty_messages_document("editor-agent-conv-1");
|
||
document.messages.push(EditorAgentMessage {
|
||
id: "editor-agent-message-assistant-1".to_string(),
|
||
role: EditorAgentMessageRole::Assistant,
|
||
kind: EditorAgentMessageKind::Chat,
|
||
text: "已生成角色图。".to_string(),
|
||
attachments: Vec::new(),
|
||
generations: vec![test_completed_generation_record()],
|
||
status: EditorAgentMessageStatus::Completed,
|
||
created_at: "2026-07-05T00:00:00Z".to_string(),
|
||
});
|
||
latest_editor_agent_generated_image_reference(&document)
|
||
.expect("latest generated image should be available")
|
||
}
|
||
}
|