GenarrativeAI/Genarrative
11
0
Fork 0
Code Issues 19 Pull Requests 4 Actions 1 Packages Projects 3 Releases Wiki Activity

extend media display

Browse Source
This commit is contained in:
王德宇
2026-07-10 17:26:44 +08:00
parent 0911c825d4
commit a3f7332dfe
6 changed files with 254 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/shared/src/contracts/editorAgent.ts
+20
View File
@@ -37,12 +37,32 @@ export interface EditorAgentGeneratedImage {
height?: number | null;
}
export interface EditorAgentGeneratedVideo {
resourceId?: string | null;
objectKey?: string | null;
assetObjectId?: string | null;
videoSrc: string;
thumbnailSrc?: string | null;
width?: number | null;
height?: number | null;
}
export interface EditorAgentGeneratedAudio {
resourceId?: string | null;
objectKey?: string | null;
assetObjectId?: string | null;
audioSrc: string;
}
export interface EditorAgentToolCall {
toolName: string;
summary: string;
status: EditorAgentToolCallStatus;
args: unknown;
images: EditorAgentGeneratedImage[];
// Older persisted conversation documents do not contain these media fields.
videos?: EditorAgentGeneratedVideo[];
audios?: EditorAgentGeneratedAudio[];
error?: string | null;
}
server-rs/crates/api-server/src/editor_agent/api.rs
+64 -19
View File
@@ -17,8 +17,8 @@ use serde_json::{Value, json};
use shared_contracts::editor_agent::{
CreateEditorAgentConversationRequest, EditorAgentConversationListResponse,
EditorAgentConversationMessagesDocument, EditorAgentConversationResponse,
EditorAgentConversationSummary, EditorAgentGeneratedImage, EditorAgentMessage,
EditorAgentMessageRequest, EditorAgentMessageResponse, EditorAgentMessageRole,
EditorAgentConversationSummary, EditorAgentGeneratedAudio, EditorAgentGeneratedImage,
EditorAgentGeneratedVideo, EditorAgentMessage,EditorAgentMessageRequest, EditorAgentMessageResponse, EditorAgentMessageRole,
EditorAgentToolCall, EditorAgentToolCallStatus,
};
use spacetime_client::{
@@ -251,6 +251,8 @@ fn build_delta_messages(
status: EditorAgentToolCallStatus::PendingConfirmation,
args: tco.tool_call.args,
images: Vec::new(),
videos: Vec::new(),
audios: Vec::new(),
error: None,
}),
created_at: created_at.to_string(),
@@ -574,21 +576,20 @@ pub async fn confirm_editor_agent_tool_call(
),
)
.await?;
complete_editor_agent_tool_call(
complete_editor_agent_media_tool_call(
&state,
&request_context,
&conversation,
&mut document,
message_id,
tool.format_execute_message(&args, &result),
vec![editor_agent_generated_media(
Vec::new(),
Vec::new(),
vec![editor_agent_generated_audio(
result.resource.as_ref(),
result.object_key,
result.asset_object_id,
result.audio_src,
None,
result.width,
result.height,
)],
)
.await
@@ -622,21 +623,20 @@ pub async fn confirm_editor_agent_tool_call(
),
)
.await?;
complete_editor_agent_tool_call(
complete_editor_agent_media_tool_call(
&state,
&request_context,
&conversation,
&mut document,
message_id,
tool.format_execute_message(&args, &result),
vec![editor_agent_generated_media(
Vec::new(),
Vec::new(),
vec![editor_agent_generated_audio(
result.resource.as_ref(),
result.object_key,
result.asset_object_id,
result.audio_src,
None,
result.width,
result.height,
)],
)
.await
@@ -668,14 +668,15 @@ pub async fn confirm_editor_agent_tool_call(
build_editor_agent_canvas_completion(&project, GenerateVideoTool::NAME, &title),
)
.await?;
complete_editor_agent_tool_call(
complete_editor_agent_media_tool_call(
&state,
&request_context,
&conversation,
&mut document,
message_id,
tool.format_execute_message(&args, &result),
vec![editor_agent_generated_media(
Vec::new(),
vec![editor_agent_generated_video(
result.resource.as_ref(),
result.object_key,
result.asset_object_id,
@@ -684,6 +685,7 @@ pub async fn confirm_editor_agent_tool_call(
result.width,
result.height,
)],
Vec::new(),
)
.await
}
@@ -1010,29 +1012,46 @@ fn editor_agent_media_generation_inputs(
})
}
fn editor_agent_generated_media(
fn editor_agent_generated_video(
resource: Option<&Value>,
object_key: Option<String>,
asset_object_id: Option<String>,
image_src: String,
video_src: String,
thumbnail_src: Option<String>,
width: u32,
height: u32,
) -> EditorAgentGeneratedImage {
EditorAgentGeneratedImage {
) -> EditorAgentGeneratedVideo {
EditorAgentGeneratedVideo {
resource_id: resource
.and_then(|value| value.get("resourceId"))
.and_then(Value::as_str)
.map(ToOwned::to_owned),
object_key,
asset_object_id,
image_src,
video_src,
thumbnail_src,
width: Some(width),
height: Some(height),
}
}
fn editor_agent_generated_audio(
resource: Option<&Value>,
object_key: Option<String>,
asset_object_id: Option<String>,
audio_src: String,
) -> EditorAgentGeneratedAudio {
EditorAgentGeneratedAudio {
resource_id: resource
.and_then(|value| value.get("resourceId"))
.and_then(Value::as_str)
.map(ToOwned::to_owned),
object_key,
asset_object_id,
audio_src,
}
}
async fn complete_editor_agent_image_tool_call(
state: &AppState,
request_context: &RequestContext,
@@ -1089,3 +1108,29 @@ async fn complete_editor_agent_tool_call(
&document.messages[message_id],
))
}
async fn complete_editor_agent_media_tool_call(
state: &AppState,
request_context: &RequestContext,
conversation: &EditorAgentConversationRecord,
document: &mut EditorAgentConversationMessagesDocument,
message_id: usize,
formatted_text: String,
images: Vec<EditorAgentGeneratedImage>,
videos: Vec<EditorAgentGeneratedVideo>,
audios: Vec<EditorAgentGeneratedAudio>,
) -> Result<Json<Value>, AppError> {
let message = &mut document.messages[message_id];
message.text = formatted_text;
if let Some(tool_call) = &mut message.tool_call {
tool_call.status = EditorAgentToolCallStatus::Completed;
tool_call.images = images;
tool_call.videos = videos;
tool_call.audios = audios;
}
write_messages_document(state, conversation, document).await?;
Ok(json_success_body(
Some(request_context),
&document.messages[message_id],
))
}
server-rs/crates/shared-contracts/src/editor_agent.rs
+81 -2
View File
@@ -13,10 +13,9 @@ pub const EDITOR_AGENT_MESSAGES_DOCUMENT_VERSION: u32 = 1;
pub enum EditorAgentMessageRole {
User,
Assistant,
System
System,
}
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum EditorAgentAttachmentSource {
@@ -70,6 +69,36 @@ pub struct EditorAgentGeneratedImage {
pub height: Option<u32>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct EditorAgentGeneratedVideo {
#[serde(default)]
pub resource_id: Option<String>,
#[serde(default)]
pub object_key: Option<String>,
#[serde(default)]
pub asset_object_id: Option<String>,
pub video_src: String,
#[serde(default)]
pub thumbnail_src: Option<String>,
#[serde(default)]
pub width: Option<u32>,
#[serde(default)]
pub height: Option<u32>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct EditorAgentGeneratedAudio {
#[serde(default)]
pub resource_id: Option<String>,
#[serde(default)]
pub object_key: Option<String>,
#[serde(default)]
pub asset_object_id: Option<String>,
pub audio_src: String,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct EditorAgentToolCall {
@@ -81,6 +110,10 @@ pub struct EditorAgentToolCall {
#[serde(default)]
pub images: Vec<EditorAgentGeneratedImage>,
#[serde(default)]
pub videos: Vec<EditorAgentGeneratedVideo>,
#[serde(default)]
pub audios: Vec<EditorAgentGeneratedAudio>,
#[serde(default)]
pub error: Option<String>,
}
@@ -159,3 +192,49 @@ pub struct EditorAgentMessageResponse {
pub delta_messages: Vec<EditorAgentMessage>,
pub error_message: Option<String>,
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn tool_call_media_results_use_camel_case_and_default_for_existing_documents() {
let legacy: EditorAgentToolCall = serde_json::from_value(json!({
"toolName": "generate-video",
"status": "completed",
"args": {},
"images": []
}))
.expect("existing message documents should remain readable");
assert!(legacy.videos.is_empty());
assert!(legacy.audios.is_empty());
let tool_call = EditorAgentToolCall {
tool_name: "generate-video".to_string(),
summary: String::new(),
status: EditorAgentToolCallStatus::Completed,
args: json!({}),
images: Vec::new(),
videos: vec![EditorAgentGeneratedVideo {
resource_id: Some("resource-video-1".to_string()),
object_key: Some("generated/video.mp4".to_string()),
asset_object_id: Some("asset-video-1".to_string()),
video_src: "/generated-video.mp4".to_string(),
thumbnail_src: Some("/generated-video.png".to_string()),
width: Some(1280),
height: Some(720),
}],
audios: vec![EditorAgentGeneratedAudio {
resource_id: Some("resource-audio-1".to_string()),
object_key: Some("generated/audio.mp3".to_string()),
asset_object_id: Some("asset-audio-1".to_string()),
audio_src: "/generated-audio.mp3".to_string(),
}],
error: None,
};
let payload = serde_json::to_value(tool_call).expect("tool call should serialize");
assert_eq!(payload["videos"][0]["videoSrc"], "/generated-video.mp4");
assert_eq!(payload["audios"][0]["audioSrc"], "/generated-audio.mp3");
}
}
src/components/ResolvedAssetAudio.tsx
+33
View File
@@ -0,0 +1,33 @@
import type { AudioHTMLAttributes } from 'react';
import { useResolvedAssetReadUrl } from '../hooks/useResolvedAssetReadUrl';
type ResolvedAssetAudioProps = Omit<
AudioHTMLAttributes<HTMLAudioElement>,
'src'
> & {
src?: string | null;
objectKey?: string | null;
fallbackSrc?: string | null;
refreshKey?: string | number | null;
};
export function ResolvedAssetAudio({
src,
objectKey,
fallbackSrc,
refreshKey,
...rest
}: ResolvedAssetAudioProps) {
const { resolvedUrl } = useResolvedAssetReadUrl(src, {
objectKey,
refreshKey,
});
const finalSrc = resolvedUrl || fallbackSrc?.trim() || '';
if (!finalSrc) {
return null;
}
return <audio {...rest} src={finalSrc} />;
}
src/components/image-editor/EditorAgentConversation/MessageBubble.tsx
+53 -1
View File
@@ -1,4 +1,4 @@
import { Check, Image as ImageIcon, Loader2, X } from 'lucide-react';
import { Check, Image as ImageIcon, Loader2, Volume2, X } from 'lucide-react';
import type {
EditorAgentAttachmentRef,
@@ -8,7 +8,9 @@ import type {
import { attachmentKey } from '@/src/components/image-editor/EditorAgentConversation/common.ts';
import { PendingToolCall } from '@/src/components/image-editor/EditorAgentConversation/PendingToolCall.tsx';
import { editorAgentToolLabel } from '@/src/components/image-editor/EditorAgentConversation/toolCallPresentation.ts';
import { ResolvedAssetAudio } from '@/src/components/ResolvedAssetAudio.tsx';
import { ResolvedAssetImage } from '@/src/components/ResolvedAssetImage.tsx';
import { ResolvedAssetVideo } from '@/src/components/ResolvedAssetVideo.tsx';
function messageRoleLabel(role: EditorAgentMessage['role']) {
if (role === 'user') {
@@ -55,6 +57,8 @@ export function AttachmentChip({
}
function ToolCallView({ toolCall }: { toolCall: EditorAgentToolCall }) {
const videos = toolCall.videos ?? [];
const audios = toolCall.audios ?? [];
const isExecuting = toolCall.status === 'executing';
const statusLabel =
toolCall.status === 'completed'
@@ -102,6 +106,54 @@ function ToolCallView({ toolCall }: { toolCall: EditorAgentToolCall }) {
))}
</div>
) : null}
{videos.length ? (
<div className="mt-2 grid grid-cols-1 gap-2">
{videos.map((video, index) => (
<div
key={`${toolCall.toolName}-${video.resourceId ?? video.objectKey ?? index}`}
className="overflow-hidden rounded-xl border border-slate-200 bg-slate-100"
>
<ResolvedAssetVideo
src={video.videoSrc}
objectKey={video.objectKey}
fallbackSrc={video.thumbnailSrc}
refreshKey={
video.resourceId ?? video.objectKey ?? toolCall.toolName
}
controls
playsInline
preload="metadata"
className="max-h-56 w-full bg-black object-contain"
/>
</div>
))}
</div>
) : null}
{audios.length ? (
<div className="mt-2 space-y-2">
{audios.map((audio, index) => (
<div
key={`${toolCall.toolName}-${audio.resourceId ?? audio.objectKey ?? index}`}
className="flex items-center gap-2 rounded-xl border border-slate-200 bg-slate-50 p-2"
>
<Volume2
className="h-4 w-4 shrink-0 text-slate-500"
aria-hidden="true"
/>
<ResolvedAssetAudio
src={audio.audioSrc}
objectKey={audio.objectKey}
refreshKey={
audio.resourceId ?? audio.objectKey ?? toolCall.toolName
}
controls
preload="metadata"
className="min-w-0 flex-1"
/>
</div>
))}
</div>
) : null}
</div>
);
}
src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.ts
+3 -1
View File
@@ -298,7 +298,9 @@ export function useEditorAgentConversation({
nextMessages.some(
(message) =>
message.toolCall?.status === 'completed' &&
message.toolCall.images.length > 0,
(message.toolCall.images.length > 0 ||
(message.toolCall.videos?.length ?? 0) > 0 ||
(message.toolCall.audios?.length ?? 0) > 0),
)
) {
onCanvasRefreshRequested?.();