diff --git a/packages/shared/src/contracts/editorAgent.ts b/packages/shared/src/contracts/editorAgent.ts index 732319684..e7412842d 100644 --- a/packages/shared/src/contracts/editorAgent.ts +++ b/packages/shared/src/contracts/editorAgent.ts @@ -37,12 +37,32 @@ export interface EditorAgentGeneratedImage { height?: number | null; } +export interface EditorAgentGeneratedVideo { + resourceId?: string | null; + objectKey?: string | null; + assetObjectId?: string | null; + videoSrc: string; + thumbnailSrc?: string | null; + width?: number | null; + height?: number | null; +} + +export interface EditorAgentGeneratedAudio { + resourceId?: string | null; + objectKey?: string | null; + assetObjectId?: string | null; + audioSrc: string; +} + export interface EditorAgentToolCall { toolName: string; summary: string; status: EditorAgentToolCallStatus; args: unknown; images: EditorAgentGeneratedImage[]; + // Older persisted conversation documents do not contain these media fields. + videos?: EditorAgentGeneratedVideo[]; + audios?: EditorAgentGeneratedAudio[]; error?: string | null; } diff --git a/server-rs/crates/api-server/src/editor_agent/api.rs b/server-rs/crates/api-server/src/editor_agent/api.rs index 81d83d46b..9840ef1f7 100644 --- a/server-rs/crates/api-server/src/editor_agent/api.rs +++ b/server-rs/crates/api-server/src/editor_agent/api.rs @@ -17,8 +17,8 @@ use serde_json::{Value, json}; use shared_contracts::editor_agent::{ CreateEditorAgentConversationRequest, EditorAgentConversationListResponse, EditorAgentConversationMessagesDocument, EditorAgentConversationResponse, - EditorAgentConversationSummary, EditorAgentGeneratedImage, EditorAgentMessage, - EditorAgentMessageRequest, EditorAgentMessageResponse, EditorAgentMessageRole, + EditorAgentConversationSummary, EditorAgentGeneratedAudio, EditorAgentGeneratedImage, + EditorAgentGeneratedVideo, EditorAgentMessage,EditorAgentMessageRequest, EditorAgentMessageResponse, EditorAgentMessageRole, EditorAgentToolCall, EditorAgentToolCallStatus, }; use spacetime_client::{ @@ -251,6 +251,8 @@ fn build_delta_messages( status: EditorAgentToolCallStatus::PendingConfirmation, args: tco.tool_call.args, images: Vec::new(), + videos: Vec::new(), + audios: Vec::new(), error: None, }), created_at: created_at.to_string(), @@ -574,21 +576,20 @@ pub async fn confirm_editor_agent_tool_call( ), ) .await?; - complete_editor_agent_tool_call( + complete_editor_agent_media_tool_call( &state, &request_context, &conversation, &mut document, message_id, tool.format_execute_message(&args, &result), - vec![editor_agent_generated_media( + Vec::new(), + Vec::new(), + vec![editor_agent_generated_audio( result.resource.as_ref(), result.object_key, result.asset_object_id, result.audio_src, - None, - result.width, - result.height, )], ) .await @@ -622,21 +623,20 @@ pub async fn confirm_editor_agent_tool_call( ), ) .await?; - complete_editor_agent_tool_call( + complete_editor_agent_media_tool_call( &state, &request_context, &conversation, &mut document, message_id, tool.format_execute_message(&args, &result), - vec![editor_agent_generated_media( + Vec::new(), + Vec::new(), + vec![editor_agent_generated_audio( result.resource.as_ref(), result.object_key, result.asset_object_id, result.audio_src, - None, - result.width, - result.height, )], ) .await @@ -668,14 +668,15 @@ pub async fn confirm_editor_agent_tool_call( build_editor_agent_canvas_completion(&project, GenerateVideoTool::NAME, &title), ) .await?; - complete_editor_agent_tool_call( + complete_editor_agent_media_tool_call( &state, &request_context, &conversation, &mut document, message_id, tool.format_execute_message(&args, &result), - vec![editor_agent_generated_media( + Vec::new(), + vec![editor_agent_generated_video( result.resource.as_ref(), result.object_key, result.asset_object_id, @@ -684,6 +685,7 @@ pub async fn confirm_editor_agent_tool_call( result.width, result.height, )], + Vec::new(), ) .await } @@ -1010,29 +1012,46 @@ fn editor_agent_media_generation_inputs( }) } -fn editor_agent_generated_media( +fn editor_agent_generated_video( resource: Option<&Value>, object_key: Option, asset_object_id: Option, - image_src: String, + video_src: String, thumbnail_src: Option, width: u32, height: u32, -) -> EditorAgentGeneratedImage { - EditorAgentGeneratedImage { +) -> EditorAgentGeneratedVideo { + EditorAgentGeneratedVideo { resource_id: resource .and_then(|value| value.get("resourceId")) .and_then(Value::as_str) .map(ToOwned::to_owned), object_key, asset_object_id, - image_src, + video_src, thumbnail_src, width: Some(width), height: Some(height), } } +fn editor_agent_generated_audio( + resource: Option<&Value>, + object_key: Option, + asset_object_id: Option, + audio_src: String, +) -> EditorAgentGeneratedAudio { + EditorAgentGeneratedAudio { + resource_id: resource + .and_then(|value| value.get("resourceId")) + .and_then(Value::as_str) + .map(ToOwned::to_owned), + object_key, + asset_object_id, + audio_src, + } +} + async fn complete_editor_agent_image_tool_call( state: &AppState, request_context: &RequestContext, @@ -1089,3 +1108,29 @@ async fn complete_editor_agent_tool_call( &document.messages[message_id], )) } + +async fn complete_editor_agent_media_tool_call( + state: &AppState, + request_context: &RequestContext, + conversation: &EditorAgentConversationRecord, + document: &mut EditorAgentConversationMessagesDocument, + message_id: usize, + formatted_text: String, + images: Vec, + videos: Vec, + audios: Vec, +) -> Result, AppError> { + let message = &mut document.messages[message_id]; + message.text = formatted_text; + if let Some(tool_call) = &mut message.tool_call { + tool_call.status = EditorAgentToolCallStatus::Completed; + tool_call.images = images; + tool_call.videos = videos; + tool_call.audios = audios; + } + write_messages_document(state, conversation, document).await?; + Ok(json_success_body( + Some(request_context), + &document.messages[message_id], + )) +} diff --git a/server-rs/crates/shared-contracts/src/editor_agent.rs b/server-rs/crates/shared-contracts/src/editor_agent.rs index b24ab82ef..bda696606 100644 --- a/server-rs/crates/shared-contracts/src/editor_agent.rs +++ b/server-rs/crates/shared-contracts/src/editor_agent.rs @@ -13,10 +13,9 @@ pub const EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION: u32 = 1; pub enum EditorAgentMessageRole { User, Assistant, - System + System, } - #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum EditorAgentAttachmentSource { @@ -70,6 +69,36 @@ pub struct EditorAgentGeneratedImage { pub height: Option, } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct EditorAgentGeneratedVideo { + #[serde(default)] + pub resource_id: Option, + #[serde(default)] + pub object_key: Option, + #[serde(default)] + pub asset_object_id: Option, + pub video_src: String, + #[serde(default)] + pub thumbnail_src: Option, + #[serde(default)] + pub width: Option, + #[serde(default)] + pub height: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct EditorAgentGeneratedAudio { + #[serde(default)] + pub resource_id: Option, + #[serde(default)] + pub object_key: Option, + #[serde(default)] + pub asset_object_id: Option, + pub audio_src: String, +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct EditorAgentToolCall { @@ -81,6 +110,10 @@ pub struct EditorAgentToolCall { #[serde(default)] pub images: Vec, #[serde(default)] + pub videos: Vec, + #[serde(default)] + pub audios: Vec, + #[serde(default)] pub error: Option, } @@ -159,3 +192,49 @@ pub struct EditorAgentMessageResponse { pub delta_messages: Vec, pub error_message: Option, } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn tool_call_media_results_use_camel_case_and_default_for_existing_documents() { + let legacy: EditorAgentToolCall = serde_json::from_value(json!({ + "toolName": "generate-video", + "status": "completed", + "args": {}, + "images": [] + })) + .expect("existing message documents should remain readable"); + assert!(legacy.videos.is_empty()); + assert!(legacy.audios.is_empty()); + + let tool_call = EditorAgentToolCall { + tool_name: "generate-video".to_string(), + summary: String::new(), + status: EditorAgentToolCallStatus::Completed, + args: json!({}), + images: Vec::new(), + videos: vec![EditorAgentGeneratedVideo { + resource_id: Some("resource-video-1".to_string()), + object_key: Some("generated/video.mp4".to_string()), + asset_object_id: Some("asset-video-1".to_string()), + video_src: "/generated-video.mp4".to_string(), + thumbnail_src: Some("/generated-video.png".to_string()), + width: Some(1280), + height: Some(720), + }], + audios: vec![EditorAgentGeneratedAudio { + resource_id: Some("resource-audio-1".to_string()), + object_key: Some("generated/audio.mp3".to_string()), + asset_object_id: Some("asset-audio-1".to_string()), + audio_src: "/generated-audio.mp3".to_string(), + }], + error: None, + }; + let payload = serde_json::to_value(tool_call).expect("tool call should serialize"); + assert_eq!(payload["videos"][0]["videoSrc"], "/generated-video.mp4"); + assert_eq!(payload["audios"][0]["audioSrc"], "/generated-audio.mp3"); + } +} diff --git a/src/components/ResolvedAssetAudio.tsx b/src/components/ResolvedAssetAudio.tsx new file mode 100644 index 000000000..1cac1c542 --- /dev/null +++ b/src/components/ResolvedAssetAudio.tsx @@ -0,0 +1,33 @@ +import type { AudioHTMLAttributes } from 'react'; + +import { useResolvedAssetReadUrl } from '../hooks/useResolvedAssetReadUrl'; + +type ResolvedAssetAudioProps = Omit< + AudioHTMLAttributes, + 'src' +> & { + src?: string | null; + objectKey?: string | null; + fallbackSrc?: string | null; + refreshKey?: string | number | null; +}; + +export function ResolvedAssetAudio({ + src, + objectKey, + fallbackSrc, + refreshKey, + ...rest +}: ResolvedAssetAudioProps) { + const { resolvedUrl } = useResolvedAssetReadUrl(src, { + objectKey, + refreshKey, + }); + const finalSrc = resolvedUrl || fallbackSrc?.trim() || ''; + + if (!finalSrc) { + return null; + } + + return