GenarrativeAI/Genarrative
11
0
Fork 0
Code Issues 14 Pull Requests 3 Actions Packages Projects 3 Releases Wiki Activity

prepare info display to user for review

Browse Source
This commit is contained in:
王德宇
2026-07-10 20:46:27 +08:00
parent ffde522b1e
commit 88368cd900
10 changed files with 751 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/shared/src/contracts/editorAgent.ts
+33 -1
View File
@@ -54,11 +54,43 @@ export interface EditorAgentGeneratedAudio {
audioSrc: string;
}
export interface EditorAgentToolCallStringArg {
name: string;
label: string;
value: string;
}
export interface EditorAgentToolCallImageRef {
imageId: string;
imageSrc: string;
objectKey?: string | null;
thumbnailSrc?: string | null;
label?: string | null;
width?: number | null;
height?: number | null;
}
export interface EditorAgentToolCallImageArg {
name: string;
label: string;
refs: EditorAgentToolCallImageRef[];
}
export interface EditorAgentToolCallDisplayExtras {
priceMudPoints: number;
}
export interface EditorAgentToolCallDisplayArgs {
stringArgs: EditorAgentToolCallStringArg[];
imageArgs: EditorAgentToolCallImageArg[];
extras: EditorAgentToolCallDisplayExtras;
}
export interface EditorAgentToolCall {
toolName: string;
summary: string;
status: EditorAgentToolCallStatus;
args: unknown;
displayArgs: EditorAgentToolCallDisplayArgs;
images: EditorAgentGeneratedImage[];
// Older persisted conversation documents do not contain these media fields.
videos?: EditorAgentGeneratedVideo[];
server-rs/crates/api-server/src/editor_agent/api.rs
+420 -12
View File
File diff suppressed because it is too large Load Diff
server-rs/crates/api-server/src/editor_agent/editor_tools/common.rs
-2
View File
@@ -27,12 +27,10 @@ impl EditorToolContext {
///
/// 通用 `Tool` 仍只负责参数校验；价格依赖 api-server 的运行时配置，不能下沉到
/// `module-editor-agent`。实际执行和扣费仍由既有生成 BFF 负责。
#[allow(dead_code)]
pub(crate) trait EditorAgentPricedTool: Tool {
fn pricing(&self, pricing: &EditorGenerationPricingConfig, args: &<Self as Tool>::Args) -> u32;
}
#[allow(dead_code)]
pub(crate) fn editor_agent_image_mud_points(
pricing: &EditorGenerationPricingConfig,
kind: Option<&str>,
server-rs/crates/api-server/src/editor_agent/editor_tools/generate_video.rs
-3
View File
@@ -24,11 +24,8 @@ pub struct GenerateVideoTool {
pub context: crate::editor_agent::editor_tools::common::EditorToolContext,
}
#[allow(dead_code)]
const DEFAULT_VIDEO_MODEL: &str = "seedance2.0-fast";
#[allow(dead_code)]
const DEFAULT_VIDEO_RESOLUTION: &str = "720p";
#[allow(dead_code)]
const DEFAULT_VIDEO_DURATION_SECONDS: u32 = 4;
#[derive(Debug, Clone)]
server-rs/crates/shared-contracts/src/editor_agent.rs
+117 -38
View File
@@ -99,14 +99,60 @@ pub struct EditorAgentGeneratedAudio {
pub audio_src: String,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct EditorAgentToolCallDisplayArgs {
pub string_args: Vec<EditorAgentToolCallStringArg>,
pub image_args: Vec<EditorAgentToolCallImageArg>,
pub extras: EditorAgentToolCallDisplayExtras,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct EditorAgentToolCallDisplayExtras {
pub price_mud_points: u32,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct EditorAgentToolCallStringArg {
pub name: String,
pub label: String,
pub value: String,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct EditorAgentToolCallImageArg {
pub name: String,
pub label: String,
pub refs: Vec<EditorAgentToolCallImageRef>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct EditorAgentToolCallImageRef {
pub image_id: String,
pub image_src: String,
#[serde(default)]
pub object_key: Option<String>,
#[serde(default)]
pub thumbnail_src: Option<String>,
#[serde(default)]
pub label: Option<String>,
#[serde(default)]
pub width: Option<u32>,
#[serde(default)]
pub height: Option<u32>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EditorAgentToolCall {
pub tool_name: String,
#[serde(default)]
pub summary: String,
pub status: EditorAgentToolCallStatus,
pub args: serde_json::Value,
pub display_args: EditorAgentToolCallDisplayArgs,
#[serde(default)]
pub images: Vec<EditorAgentGeneratedImage>,
#[serde(default)]
@@ -195,46 +241,79 @@ pub struct EditorAgentMessageResponse {
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
use super::*;
use serde_json::json;
#[test]
fn tool_call_media_results_use_camel_case_and_default_for_existing_documents() {
let legacy: EditorAgentToolCall = serde_json::from_value(json!({
#[test]
fn tool_call_display_args_and_media_use_camel_case() {
let missing_display_args = serde_json::from_value::<EditorAgentToolCall>(json!({
"toolName": "generate-video",
"status": "completed",
"args": {},
"images": []
}))
.expect("existing message documents should remain readable");
assert!(legacy.videos.is_empty());
assert!(legacy.audios.is_empty());
}));
assert!(missing_display_args.is_err());
let tool_call = EditorAgentToolCall {
tool_name: "generate-video".to_string(),
summary: String::new(),
status: EditorAgentToolCallStatus::Completed,
args: json!({}),
images: Vec::new(),
videos: vec![EditorAgentGeneratedVideo {
resource_id: Some("resource-video-1".to_string()),
object_key: Some("generated/video.mp4".to_string()),
asset_object_id: Some("asset-video-1".to_string()),
video_src: "/generated-video.mp4".to_string(),
thumbnail_src: Some("/generated-video.png".to_string()),
width: Some(1280),
height: Some(720),
}],
audios: vec![EditorAgentGeneratedAudio {
resource_id: Some("resource-audio-1".to_string()),
object_key: Some("generated/audio.mp3".to_string()),
asset_object_id: Some("asset-audio-1".to_string()),
audio_src: "/generated-audio.mp3".to_string(),
}],
error: None,
};
let payload = serde_json::to_value(tool_call).expect("tool call should serialize");
assert_eq!(payload["videos"][0]["videoSrc"], "/generated-video.mp4");
assert_eq!(payload["audios"][0]["audioSrc"], "/generated-audio.mp3");
}
let tool_call = EditorAgentToolCall {
tool_name: "generate-video".to_string(),
status: EditorAgentToolCallStatus::Completed,
args: json!({}),
display_args: EditorAgentToolCallDisplayArgs {
string_args: vec![EditorAgentToolCallStringArg {
name: "prompt".to_string(),
label: "提示词".to_string(),
value: "生成像素风角色".to_string(),
}],
image_args: vec![EditorAgentToolCallImageArg {
name: "reference_image_ids".to_string(),
label: "参考图片".to_string(),
refs: vec![EditorAgentToolCallImageRef {
image_id: "sha256:reference-1".to_string(),
image_src: "/reference.png".to_string(),
object_key: Some("generated/reference.png".to_string()),
thumbnail_src: Some("/reference-thumbnail.png".to_string()),
label: Some("角色参考".to_string()),
width: Some(512),
height: Some(512),
}],
}],
extras: EditorAgentToolCallDisplayExtras {
price_mud_points: 5,
},
},
images: Vec::new(),
videos: vec![EditorAgentGeneratedVideo {
resource_id: Some("resource-video-1".to_string()),
object_key: Some("generated/video.mp4".to_string()),
asset_object_id: Some("asset-video-1".to_string()),
video_src: "/generated-video.mp4".to_string(),
thumbnail_src: Some("/generated-video.png".to_string()),
width: Some(1280),
height: Some(720),
}],
audios: vec![EditorAgentGeneratedAudio {
resource_id: Some("resource-audio-1".to_string()),
object_key: Some("generated/audio.mp3".to_string()),
asset_object_id: Some("asset-audio-1".to_string()),
audio_src: "/generated-audio.mp3".to_string(),
}],
error: None,
};
let payload = serde_json::to_value(tool_call).expect("tool call should serialize");
assert_eq!(payload["videos"][0]["videoSrc"], "/generated-video.mp4");
assert_eq!(payload["audios"][0]["audioSrc"], "/generated-audio.mp3");
assert_eq!(
payload["displayArgs"]["stringArgs"][0]["value"],
"生成像素风角色"
);
assert_eq!(
payload["displayArgs"]["imageArgs"][0]["refs"][0]["imageId"],
"sha256:reference-1"
);
assert_eq!(
payload["displayArgs"]["imageArgs"][0]["refs"][0]["objectKey"],
"generated/reference.png"
);
assert_eq!(payload["displayArgs"]["extras"]["priceMudPoints"], 5);
}
}
src/components/image-editor/EditorAgentConversation/EditorAgentConversationPanelView.test.tsx
+54 -9
View File
@@ -122,13 +122,53 @@ function createPendingToolCallMessage(): EditorAgentMessage {
attachments: [],
toolCall: {
toolName: 'edit-image',
summary: '',
status: 'pending_confirmation',
args: {
object_image_id: 'source-image-1',
reference_image_ids: ['reference-image-1', 'reference-image-2'],
prompt: '把角色换成像素风',
},
displayArgs: {
stringArgs: [
{
name: 'prompt',
label: '提示词',
value: '把角色换成像素风',
},
],
imageArgs: [
{
name: 'object_image_id',
label: '目标图片',
refs: [
{
imageId: 'source-image-1',
imageSrc: 'data:image/png;base64,c291cmNl',
label: '原角色',
width: 512,
height: 512,
},
],
},
{
name: 'reference_image_ids',
label: '参考图片',
refs: [
{
imageId: 'reference-image-1',
imageSrc: 'data:image/png;base64,cmVmZXJlbmNlLTE=',
label: '像素风参考',
},
{
imageId: 'reference-image-2',
imageSrc: 'data:image/png;base64,cmVmZXJlbmNlLTI=',
label: '配色参考',
},
],
},
],
extras: { priceMudPoints: 3 },
},
images: [],
error: null,
},
@@ -500,8 +540,14 @@ describe('EditorAgentConversationPanelView', () => {
);
expect(await screen.findByText('把角色换成像素风')).toBeTruthy();
expect(screen.getByText('source-image-1')).toBeTruthy();
expect(screen.getByText('提示词')).toBeTruthy();
expect(screen.getByText('2 张')).toBeTruthy();
expect(screen.getByAltText('目标图片：原角色')).toBeTruthy();
expect(screen.getByAltText('参考图片：像素风参考')).toBeTruthy();
expect(screen.getByAltText('参考图片：配色参考')).toBeTruthy();
expect(screen.getByText('预计消耗 3泥点')).toBeTruthy();
expect(screen.queryByText('source-image-1')).toBeNull();
expect(screen.queryByText('reference-image-1')).toBeNull();
expect(
screen.queryByText('internal system prompt that must stay hidden'),
).toBeNull();
@@ -511,13 +557,12 @@ describe('EditorAgentConversationPanelView', () => {
expect(client.confirmToolCall).toHaveBeenCalledWith('conversation-1', 2);
});
expect(
(screen.getByRole('button', { name: '执行中' }) as HTMLButtonElement)
.disabled,
).toBe(true);
expect(
(screen.getByRole('button', { name: '取消' }) as HTMLButtonElement)
.disabled,
).toBe(true);
within(screen.getByRole('article', { name: 'Agent操作' })).getByText(
'执行中',
),
).toBeTruthy();
expect(screen.queryByRole('button', { name: '确认' })).toBeNull();
expect(screen.queryByRole('button', { name: '取消' })).toBeNull();
await act(async () => {
resolveConfirmation({
src/components/image-editor/EditorAgentConversation/PendingToolCall.tsx
+61 -53
View File
@@ -1,7 +1,8 @@
import { Check, Loader2, Pencil, X } from 'lucide-react';
import { Check, Coins, Loader2, Pencil, X } from 'lucide-react';
import type { EditorAgentToolCall } from '@/packages/shared/src/contracts';
import { editorAgentToolLabel } from '@/src/components/image-editor/EditorAgentConversation/toolCallPresentation.ts';
import { ResolvedAssetImage } from '@/src/components/ResolvedAssetImage.tsx';
type PendingToolCallAction = 'confirm' | 'cancel' | null;
@@ -13,37 +14,10 @@ type PendingToolCallProps = {
onCancel: (messageId: number) => void;
};
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null;
}
function readString(value: unknown) {
return typeof value === 'string' && value.trim() ? value.trim() : null;
}
function readStringArray(value: unknown) {
return Array.isArray(value)
? value.map(readString).filter((item): item is string => item !== null)
: [];
}
function readPendingToolDetails(toolCall: EditorAgentToolCall) {
const args = isRecord(toolCall.args) ? toolCall.args : {};
const prompt = readString(args.prompt);
const targetImage =
readString(args.object_image_id) ?? readString(args.objectImageId);
const referenceImages = readStringArray(
args.reference_image_ids ?? args.referenceImageIds,
);
const summary = readString(toolCall.summary);
return {
prompt: prompt ?? (summary?.startsWith('{') ? null : summary),
targetImage,
referenceImageCount: referenceImages.length,
};
}
export function PendingToolCall({
messageId,
toolCall,
@@ -51,7 +25,7 @@ export function PendingToolCall({
onConfirm,
onCancel,
}: PendingToolCallProps) {
const details = readPendingToolDetails(toolCall);
const displayArgs = toolCall.displayArgs;
const label = editorAgentToolLabel(toolCall.toolName);
const isBusy = busyAction !== null;
@@ -66,32 +40,66 @@ export function PendingToolCall({
</span>
</div>
{details.prompt ? (
<p className="mt-2 whitespace-pre-wrap break-words text-sm leading-5">
{details.prompt}
</p>
) : null}
<div className="mt-3 space-y-3 empty:hidden">
{displayArgs.stringArgs.map((argument, index) => (
<div
key={`${argument.name}-${index}`}
className="rounded-lg bg-slate-50 px-2.5 py-2"
>
<div className="text-xs font-medium text-slate-500">
{argument.label}
</div>
<div className="mt-1 whitespace-pre-wrap break-words text-sm leading-5 text-slate-800">
{argument.value}
</div>
</div>
))}
{details.targetImage || details.referenceImageCount > 0 ? (
<dl className="mt-2 space-y-1 text-xs text-slate-500">
{details.targetImage ? (
<div className="flex min-w-0 gap-2">
<dt className="shrink-0">目标图片</dt>
<dd className="min-w-0 break-all text-slate-700">
{details.targetImage}
</dd>
{displayArgs.imageArgs.map((argument, argumentIndex) => (
<div key={`${argument.name}-${argumentIndex}`}>
<div className="flex items-center gap-2 text-xs font-medium text-slate-500">
<span>{argument.label}</span>
<span className="font-normal text-slate-400">
{argument.refs.length} 张
</span>
</div>
) : null}
{details.referenceImageCount > 0 ? (
<div className="flex gap-2">
<dt>参考图片</dt>
<dd className="text-slate-700">
{details.referenceImageCount} 张
</dd>
</div>
) : null}
</dl>
) : null}
{argument.refs.length ? (
<div className="mt-1.5 flex flex-wrap gap-2">
{argument.refs.map((image, imageIndex) => {
const imageLabel =
readString(image.label) ?? `图片 ${imageIndex + 1}`;
return (
<figure
key={`${image.imageId}-${imageIndex}`}
className="w-20 min-w-0"
>
<div className="overflow-hidden rounded-lg border border-slate-200 bg-slate-100">
<ResolvedAssetImage
src={image.thumbnailSrc ?? image.imageSrc}
objectKey={image.objectKey}
refreshKey={image.imageId}
alt={`${argument.label}：${imageLabel}`}
className="h-20 w-20 object-contain"
/>
</div>
{image.label ? (
<figcaption className="mt-1 truncate text-center text-[11px] text-slate-500">
{image.label}
</figcaption>
) : null}
</figure>
);
})}
</div>
) : null}
</div>
))}
</div>
<div className="mt-3 flex items-center gap-1.5 rounded-lg border border-amber-200 bg-amber-50 px-2.5 py-2 text-xs font-medium text-amber-800">
<Coins className="h-3.5 w-3.5 shrink-0" aria-hidden="true" />
<span>预计消耗 {displayArgs.extras.priceMudPoints}泥点</span>
</div>
<div className="mt-3 flex justify-end gap-2">
<button
src/components/image-editor/EditorAgentConversation/useEditorAgentConversation.test.tsx
+29 -5
View File
@@ -12,6 +12,26 @@ import {
useEditorAgentConversation,
} from './useEditorAgentConversation.ts';
function createEditImageDisplayArgs(prompt: string) {
return {
stringArgs: [{ name: 'prompt', label: '修改要求', value: prompt }],
imageArgs: [
{
name: 'object_image_id',
label: '目标图片',
refs: [
{
imageId: 'source-image',
imageSrc: '/source-image.png',
label: '源图片',
},
],
},
],
extras: { priceMudPoints: 3 },
};
}
function createClient(): EditorAgentConversationClient {
return {
listConversations: vi.fn().mockResolvedValue([
@@ -61,9 +81,13 @@ function createClient(): EditorAgentConversationClient {
attachments: [],
toolCall: {
toolName: 'generate_image',
summary: '生成像素风格角色',
status: 'completed',
args: {},
displayArgs: {
stringArgs: [],
imageArgs: [],
extras: { priceMudPoints: 0 },
},
images: [
{
resourceId: 'resource-result-1',
@@ -252,12 +276,12 @@ describe('useEditorAgentConversation', () => {
attachments: [],
toolCall: {
toolName: 'edit-image',
summary: '',
status: 'pending_confirmation',
args: {
object_image_id: 'source-image',
prompt: '换成像素风',
},
displayArgs: createEditImageDisplayArgs('换成像素风'),
images: [],
error: null,
},
@@ -273,12 +297,12 @@ describe('useEditorAgentConversation', () => {
attachments: [],
toolCall: {
toolName: 'edit-image',
summary: '',
status: 'completed',
args: {
object_image_id: 'source-image',
prompt: '换成像素风',
},
displayArgs: createEditImageDisplayArgs('换成像素风'),
images: [
{
resourceId: null,
@@ -339,9 +363,9 @@ describe('useEditorAgentConversation', () => {
attachments: [],
toolCall: {
toolName: 'edit-image',
summary: '',
status: 'pending_confirmation',
args: { object_image_id: 'source-image', prompt: '换成像素风' },
displayArgs: createEditImageDisplayArgs('换成像素风'),
images: [],
error: null,
},
@@ -407,9 +431,9 @@ describe('useEditorAgentConversation', () => {
attachments: [],
toolCall: {
toolName: 'edit-image',
summary: '',
status: 'pending_confirmation' as const,
args: { object_image_id: 'source-image', prompt: '换成像素风' },
displayArgs: createEditImageDisplayArgs('换成像素风'),
images: [],
error: null,
},
src/components/image-editor/ImageCanvasEditorView.test.tsx
+32 -5
View File
@@ -69,6 +69,7 @@ const getEditorAgentConversationMock = vi.hoisted(() =>
conversationId: 'editor-agent-conv-test',
projectId: 'editor-project-default',
title: '画布 Agent',
createdAt: '2026-07-03T00:00:00.000Z',
updatedAt: '2026-07-03T00:00:00.000Z',
messages: [],
})),
@@ -176,12 +177,32 @@ function createEditorAgentConversationSummary(
};
}
function createEditImageToolCallDisplayArgs(prompt: string) {
return {
stringArgs: [{ name: 'prompt', label: '修改要求', value: prompt }],
imageArgs: [
{
name: 'object_image_id',
label: '目标图片',
refs: [
{
imageId: 'source-image',
imageSrc: '/source-image.png',
label: '源图片',
},
],
},
],
extras: { priceMudPoints: 3 },
};
}
function createEditorAgentDetailWithGeneration(
resourceId: string,
): EditorAgentConversationDetail {
const summary = createEditorAgentConversationSummary();
const conversationSummary = createEditorAgentConversationSummary();
return {
...summary,
...conversationSummary,
title: '画布 Agent',
createdAt: '2026-07-03T00:00:00.000Z',
messages: [
@@ -192,9 +213,13 @@ function createEditorAgentDetailWithGeneration(
attachments: [],
toolCall: {
toolName: 'generate_image',
summary: '',
status: 'completed',
args: {},
displayArgs: {
stringArgs: [],
imageArgs: [],
extras: { priceMudPoints: 0 },
},
images: [
{
resourceId,
@@ -1855,12 +1880,13 @@ describe('ImageCanvasEditorView', () => {
attachments: [],
toolCall: {
toolName: 'edit-image',
summary: '',
status: 'pending_confirmation',
args: {
object_image_id: 'source-image',
prompt: '把图片换成像素风',
},
displayArgs:
createEditImageToolCallDisplayArgs('把图片换成像素风'),
images: [],
error: null,
},
@@ -1877,9 +1903,10 @@ describe('ImageCanvasEditorView', () => {
attachments: [],
toolCall: {
toolName: 'edit-image',
summary: '',
status: 'completed',
args: {},
displayArgs:
createEditImageToolCallDisplayArgs('把图片换成像素风'),
images: [
{
resourceId: null,
src/services/image-editor/editorAgentClient.test.ts
+5 -1
View File
@@ -154,9 +154,13 @@ describe('editorAgentClient', () => {
attachments: [],
toolCall: {
toolName: 'edit-image',
summary: '',
status: 'completed',
args: {},
displayArgs: {
stringArgs: [],
imageArgs: [],
extras: { priceMudPoints: 0 },
},
images: [],
error: null,
},