修复画布Agent聊天生成体验

让画布 Agent 消息统一走后端规划与 LLM 回复
修复生成图片在聊天、画布和素材库中的展示与恢复
补齐右键删除、画布焦点、聊天滚轮和面板避让交互
同步 Agent 事件契约、文档和定向回归测试
This commit is contained in:
2026-07-04 16:41:33 +08:00
parent 8dad52f040
commit 1671d35995
29 changed files with 909 additions and 173 deletions
@@ -1,6 +1,7 @@
/* @vitest-environment jsdom */
import {
act,
fireEvent,
render,
screen,
@@ -253,6 +254,135 @@ describe('EditorAgentConversationPanelView', () => {
});
});
it('keeps streamed reply and generating state when the panel is collapsed and reopened', async () => {
const client = createClient();
let finishStream = () => {};
vi.mocked(client.streamMessage).mockImplementation(
async (conversationId, _payload, options) => {
options.onEvent?.({
event: 'message_delta',
data: {
conversationId,
messageId: 'assistant-stream',
role: 'assistant',
kind: 'chat',
textDelta: '我来生成图片。',
},
});
options.onEvent?.({
event: 'stage',
data: { conversationId, stage: 'generating' },
});
options.onEvent?.({
event: 'tool_started',
data: {
conversationId,
messageId: 'assistant-stream',
toolCallId: 'tool-call-generating',
toolName: 'generate_image',
taskId: 'task-generating',
model: 'gpt-image-2',
},
});
await new Promise<void>((resolve) => {
finishStream = resolve;
});
},
);
const { rerender } = render(
<EditorAgentConversationPanelView
projectId="project-1"
open
onToggleOpen={vi.fn()}
client={client}
/>,
);
await waitFor(() => {
expect(screen.getByText('已经看到画布内容')).toBeTruthy();
});
fireEvent.change(screen.getByLabelText('发送给画布 Agent'), {
target: { value: '生成一张图片' },
});
fireEvent.click(screen.getByRole('button', { name: '发送' }));
await waitFor(() => {
expect(screen.getByText('我来生成图片。')).toBeTruthy();
});
expect(screen.getByText('生成中')).toBeTruthy();
expect(screen.getByRole('button', { name: '停止' })).toBeTruthy();
rerender(
<EditorAgentConversationPanelView
projectId="project-1"
open={false}
onToggleOpen={vi.fn()}
client={client}
/>,
);
expect(screen.getByRole('button', { name: '打开画布 Agent' })).toBeTruthy();
rerender(
<EditorAgentConversationPanelView
projectId="project-1"
open
onToggleOpen={vi.fn()}
client={client}
/>,
);
expect(screen.getByText('我来生成图片。')).toBeTruthy();
expect(screen.getByText('生成中')).toBeTruthy();
expect(screen.getByRole('button', { name: '停止' })).toBeTruthy();
await act(async () => {
finishStream();
});
});
it('keeps wheel scrolling inside the message history and input', async () => {
const client = createClient();
const parentWheel = vi.fn();
render(
<div onWheel={parentWheel}>
<EditorAgentConversationPanelView
projectId="project-1"
open
onToggleOpen={vi.fn()}
client={client}
/>
</div>,
);
await waitFor(() => {
expect(screen.getByText('已经看到画布内容')).toBeTruthy();
});
const messageWheel = new WheelEvent('wheel', {
bubbles: true,
cancelable: true,
deltaY: 120,
});
fireEvent(
screen.getByRole('log', { name: '画布 Agent 消息流' }),
messageWheel,
);
expect(parentWheel).not.toHaveBeenCalled();
expect(messageWheel.defaultPrevented).toBe(false);
const inputWheel = new WheelEvent('wheel', {
bubbles: true,
cancelable: true,
deltaY: 120,
});
fireEvent(screen.getByLabelText('发送给画布 Agent'), inputWheel);
expect(parentWheel).not.toHaveBeenCalled();
expect(inputWheel.defaultPrevented).toBe(false);
});
it('confirms conversation deletion from an independent dialog', async () => {
const client = createClient();
render(
@@ -10,11 +10,18 @@ import {
Trash2,
X,
} from 'lucide-react';
import { type FormEvent, useMemo, useState } from 'react';
import {
type FormEvent,
type WheelEvent as ReactWheelEvent,
useEffect,
useMemo,
useState,
} from 'react';
import {
EDITOR_AGENT_MAX_ATTACHMENTS,
type EditorAgentAttachmentRef,
type EditorAgentGenerationResultEvent,
type EditorAgentGenerationRecord,
type EditorAgentMessage,
type EditorAgentStage,
@@ -22,6 +29,7 @@ import {
import { PlatformActionButton } from '../common/PlatformActionButton';
import { PlatformDangerConfirmDialog } from '../common/PlatformDangerConfirmDialog';
import { UnifiedModal } from '../common/UnifiedModal';
import { ResolvedAssetImage } from '../ResolvedAssetImage';
import type { CanvasLayer, EditorAsset } from './ImageCanvasEditorTypes';
import {
type EditorAgentConversationClient,
@@ -42,10 +50,14 @@ type EditorAgentConversationPanelViewProps = {
onToggleOpen: () => void;
layers?: CanvasLayer[];
assets?: EditorAsset[];
onFocusResource?: (resourceId: string) => void;
onGenerationResult?: (event: EditorAgentGenerationResultEvent) => void;
client?: EditorAgentConversationClient;
};
function stopAgentPanelWheel(event: ReactWheelEvent<HTMLElement>) {
event.stopPropagation();
}
function attachmentKey(attachment: EditorAgentAttachmentRef) {
return `${attachment.source}:${attachment.referenceId}`;
}
@@ -169,8 +181,10 @@ function AttachmentChip({
) : null}
{attachment.thumbnailSrc || attachment.imageSrc ? (
<span className="pointer-events-none absolute bottom-[calc(100%+0.4rem)] left-0 hidden rounded-2xl border border-white bg-white p-1 shadow-xl group-hover:block">
<img
<ResolvedAssetImage
src={attachment.thumbnailSrc ?? attachment.imageSrc}
objectKey={attachment.objectKey}
refreshKey={attachment.referenceId}
alt=""
className="h-24 w-24 rounded-xl object-cover"
/>
@@ -182,10 +196,8 @@ function AttachmentChip({
function GenerationRecordsView({
generations,
onFocusResource,
}: {
generations: EditorAgentGenerationRecord[];
onFocusResource?: (resourceId: string) => void;
}) {
if (!generations.length) {
return null;
@@ -197,42 +209,42 @@ function GenerationRecordsView({
key={generation.toolCallId}
className="rounded-2xl border border-slate-200 bg-white/80 p-2 text-xs text-slate-600"
>
<div className="flex items-center gap-2">
{generation.status === 'generating' ? (
<Loader2
className="h-3.5 w-3.5 animate-spin"
aria-hidden="true"
/>
) : (
<ImageIcon className="h-3.5 w-3.5" aria-hidden="true" />
)}
<span>{toolLabel(generation.toolName)}</span>
{generation.model ? <span>{generation.model}</span> : null}
</div>
{generation.status === 'generating' || generation.error ? (
<div className="flex items-center gap-2">
{generation.status === 'generating' ? (
<Loader2
className="h-3.5 w-3.5 animate-spin"
aria-hidden="true"
/>
) : (
<ImageIcon className="h-3.5 w-3.5" aria-hidden="true" />
)}
<span>{toolLabel(generation.toolName)}</span>
{generation.model ? <span>{generation.model}</span> : null}
</div>
) : null}
{generation.error ? (
<div className="mt-1 text-red-600">{generation.error}</div>
) : null}
{generation.images.length ? (
<div className="mt-2 grid grid-cols-3 gap-2">
{generation.images.map((image, index) => (
<button
<div
key={`${generation.toolCallId}-${image.resourceId ?? index}`}
type="button"
className="overflow-hidden rounded-xl border border-slate-200 bg-slate-100"
title="生成结果"
disabled={!image.resourceId || !onFocusResource}
onClick={() => {
if (image.resourceId) {
onFocusResource?.(image.resourceId);
}
}}
>
<img
<ResolvedAssetImage
src={image.thumbnailSrc ?? image.imageSrc}
objectKey={image.objectKey}
refreshKey={
image.resourceId ??
generation.taskId ??
generation.toolCallId
}
alt=""
className="h-20 w-full object-cover"
/>
</button>
</div>
))}
</div>
) : null}
@@ -244,10 +256,8 @@ function GenerationRecordsView({
function MessageBubble({
message,
onFocusResource,
}: {
message: EditorAgentMessage;
onFocusResource?: (resourceId: string) => void;
}) {
const isUser = message.role === 'user';
return (
@@ -279,7 +289,6 @@ function MessageBubble({
) : null}
<GenerationRecordsView
generations={message.generations}
onFocusResource={onFocusResource}
/>
</div>
</article>
@@ -367,10 +376,12 @@ function AttachmentPickerModal({
key={option.key}
className="flex cursor-pointer flex-col gap-2 rounded-2xl border border-slate-200 bg-white p-2 text-sm text-slate-700 shadow-sm"
>
<img
<ResolvedAssetImage
src={
option.attachment.thumbnailSrc ?? option.attachment.imageSrc
}
objectKey={option.attachment.objectKey}
refreshKey={option.attachment.referenceId}
alt=""
className="aspect-square rounded-xl bg-slate-100 object-cover"
/>
@@ -398,10 +409,16 @@ export function EditorAgentConversationPanelView({
onToggleOpen,
layers = [],
assets = [],
onFocusResource,
onGenerationResult,
client,
}: EditorAgentConversationPanelViewProps) {
const effectiveProjectId = open ? projectId : null;
const [hasConversationMounted, setHasConversationMounted] = useState(open);
useEffect(() => {
if (open) {
setHasConversationMounted(true);
}
}, [open]);
const effectiveProjectId = hasConversationMounted ? projectId : null;
const {
conversations,
activeConversationId,
@@ -418,7 +435,11 @@ export function EditorAgentConversationPanelView({
sendMessage,
stopCurrentTurn,
deleteActiveConversation,
} = useEditorAgentConversation({ projectId: effectiveProjectId, client });
} = useEditorAgentConversation({
projectId: effectiveProjectId,
client,
onGenerationResult,
});
const [draftText, setDraftText] = useState('');
const [attachments, setAttachments] = useState<EditorAgentAttachmentRef[]>(
[],
@@ -517,7 +538,7 @@ export function EditorAgentConversationPanelView({
return (
<button
type="button"
className="absolute right-3 top-20 z-40 inline-flex items-center gap-2 rounded-full border border-white/70 bg-white/95 px-3 py-2 text-sm font-semibold text-slate-700 shadow-lg backdrop-blur hover:bg-white"
className="absolute bottom-24 right-4 z-40 inline-flex items-center gap-2 rounded-full border border-white/70 bg-white/95 px-3 py-2 text-sm font-semibold text-slate-700 shadow-lg backdrop-blur hover:bg-white"
aria-label="打开画布 Agent"
onClick={onToggleOpen}
>
@@ -533,6 +554,8 @@ export function EditorAgentConversationPanelView({
className="absolute inset-y-0 right-0 z-50 flex w-full flex-col border-l border-slate-200 bg-slate-50/95 shadow-2xl backdrop-blur sm:inset-y-3 sm:right-3 sm:w-[390px] sm:overflow-hidden sm:rounded-3xl sm:border"
aria-label="画布 Agent 对话"
onPointerDown={(event) => event.stopPropagation()}
onWheel={stopAgentPanelWheel}
onWheelCapture={stopAgentPanelWheel}
>
<header className="flex items-center gap-2 border-b border-slate-200 bg-white/90 px-3 py-3">
<Bot className="h-5 w-5 text-slate-700" aria-hidden="true" />
@@ -604,7 +627,7 @@ export function EditorAgentConversationPanelView({
</div>
) : null}
<div
className="min-h-0 flex-1 space-y-3 overflow-y-auto px-3 py-4"
className="min-h-0 flex-1 space-y-3 overflow-y-auto overscroll-contain px-3 py-4"
role="log"
aria-label="画布 Agent 消息流"
>
@@ -618,7 +641,6 @@ export function EditorAgentConversationPanelView({
<MessageBubble
key={message.id}
message={message}
onFocusResource={onFocusResource}
/>
))
) : (
@@ -657,7 +679,7 @@ export function EditorAgentConversationPanelView({
<Paperclip className="h-4 w-4" aria-hidden="true" />
</button>
<textarea
className="max-h-32 min-h-10 flex-1 resize-none rounded-3xl border border-slate-200 bg-slate-50 px-3 py-2 text-sm text-slate-800 outline-none focus:border-slate-400"
className="max-h-32 min-h-10 flex-1 resize-none overflow-y-auto overscroll-contain rounded-3xl border border-slate-200 bg-slate-50 px-3 py-2 text-sm text-slate-800 outline-none focus:border-slate-400"
aria-label="发送给画布 Agent"
value={draftText}
rows={1}
@@ -100,7 +100,11 @@ export function ImageCanvasContextMenusView({
? [
'context',
contextMenu.kind,
contextMenu.kind === 'layer' ? contextMenu.layerId : 'blank',
contextMenu.kind === 'layer'
? contextMenu.layerId
: contextMenu.kind === 'generation-dialog'
? contextMenu.dialogId
: 'blank',
contextMenu.x,
contextMenu.y,
isImageSequenceLayer ? 'sequence' : 'single',
@@ -259,7 +263,11 @@ export function ImageCanvasContextMenusView({
className={contextMenuClassName}
role="menu"
aria-label={
contextMenu.kind === 'blank' ? '画布右键菜单' : '图片功能面板'
contextMenu.kind === 'blank'
? '画布右键菜单'
: contextMenu.kind === 'generation-dialog'
? '生成器右键菜单'
: '图片功能面板'
}
style={{
left: currentMeasuredMenuLayout?.x ?? contextMenu.x,
@@ -308,6 +316,15 @@ export function ImageCanvasContextMenusView({
显示画布所有元素
</button>
</>
) : contextMenu.kind === 'generation-dialog' ? (
<button
type="button"
role="menuitem"
className="image-canvas-editor__context-menu-danger"
onClick={onDeleteContextLayers}
>
删除
</button>
) : (
<>
<button
@@ -48,6 +48,7 @@ export const CONTEXT_MENU_VIEWPORT_MARGIN = 8;
export const CONTEXT_MENU_SIZE = {
blank: { width: 188, height: 176 },
layer: { width: 188, height: 492 },
'generation-dialog': { width: 188, height: 64 },
} as const;
export const CANVAS_BACKGROUND_OPTIONS = [
{ label: '白色', value: '#ffffff' },
@@ -180,6 +180,7 @@ function createStageProps(): ImageCanvasStageViewProps {
onOpenLayerMetadata: vi.fn(),
onUpdateLayerAssetKind: vi.fn(),
onGenerationFramePointerDown: vi.fn(),
onGenerationFrameContextMenu: vi.fn(),
onActivateGenerationDialog: vi.fn(),
onFocusExternalTask: vi.fn(),
onToggleTaskSidebar: vi.fn(),
@@ -285,6 +285,13 @@ export type CanvasContextMenuState =
y: number;
layerId: string;
canvasPoint: { x: number; y: number };
}
| {
kind: 'generation-dialog';
x: number;
y: number;
dialogId: string;
canvasPoint: { x: number; y: number };
};
export type QuickEditPanelState = {
@@ -16,6 +16,7 @@ import {
ApiClientError,
AuthUiContext,
createAuthValue,
defaultEditorAssetLibraryAssets,
defaultEditorProjectLayers,
defaultEditorProjectResources,
dispatchPointerEvent,
@@ -937,6 +938,33 @@ describe('ImageCanvasEditorView', () => {
expect(screen.getByAltText('画布图片:大鱼素材')).toBeTruthy();
});
it('moves focus from the Agent input back to the canvas layer before handling shortcuts', async () => {
render(<ImageCanvasEditorView />);
fireEvent.click(screen.getByRole('button', { name: '画布 Agent' }));
const agentInput = await screen.findByLabelText('发送给画布 Agent');
agentInput.focus();
expect(document.activeElement).toBe(agentInput);
const layerButton = screen
.getByAltText('画布图片:拼图素材')
.closest('button')!;
fireEvent.pointerDown(layerButton, {
button: 0,
pointerId: 53,
clientX: 120,
clientY: 120,
});
expect(document.activeElement).toBe(layerButton);
await act(async () => {
fireEvent.keyDown(window, { key: 'Backspace', code: 'Backspace' });
});
expect(screen.queryByAltText('画布图片:拼图素材')).toBeNull();
expect(screen.getByAltText('画布图片:大鱼素材')).toBeTruthy();
});
it('opens the matching audio generation panel when remodeling an existing sound effect layer', async () => {
loadOrCreateRecentEditorProjectMock.mockResolvedValueOnce({
projectId: 'editor-project-audio-remodel',
@@ -1123,6 +1151,23 @@ describe('ImageCanvasEditorView', () => {
expect(screen.getByRole('menuitem', { name: '创建副本' })).toBeTruthy();
});
it('shows delete when right-clicking a generation placeholder', async () => {
render(<ImageCanvasEditorView />);
fireEvent.click(screen.getByRole('button', { name: '生成图片' }));
const frame = await screen.findByLabelText('图像生成占位图');
fireEvent.contextMenu(frame, {
clientX: 510,
clientY: 330,
});
const menu = screen.getByRole('menu', { name: '生成器右键菜单' });
fireEvent.click(within(menu).getByRole('menuitem', { name: '删除' }));
expect(screen.queryByLabelText('图像生成占位图')).toBeNull();
});
it('copies, cuts, and pastes layers from the context menus', () => {
render(<ImageCanvasEditorView />);
@@ -1511,7 +1556,7 @@ describe('ImageCanvasEditorView', () => {
expect(screen.queryByRole('button', { name: '画布小地图' })).toBeNull();
});
it('focuses an existing canvas layer from an Agent generation thumbnail', async () => {
it('renders Agent generation thumbnails as passive previews', async () => {
const detail = createEditorAgentDetailWithGeneration('resource-puzzle');
listEditorAgentConversationsMock.mockResolvedValueOnce([
createEditorAgentConversationSummary(),
@@ -1521,22 +1566,90 @@ describe('ImageCanvasEditorView', () => {
render(<ImageCanvasEditorView />);
fireEvent.click(screen.getByRole('button', { name: '画布 Agent' }));
fireEvent.click(await screen.findByRole('button', { name: '生成结果' }));
await waitFor(() => {
expect(
screen.getByAltText('画布图片:拼图素材').closest('button')?.className,
).toContain('image-canvas-editor__layer--selected');
const messageLog = await screen.findByRole('log', {
name: '画布 Agent 消息流',
});
expect(
within(messageLog).queryByRole('button', { name: '生成结果' }),
).toBeNull();
expect(within(messageLog).queryByText('gpt-image-2')).toBeNull();
expect(loadEditorProjectMock).not.toHaveBeenCalled();
});
it('refreshes the project and focuses a newly added Agent result layer', async () => {
const detail = createEditorAgentDetailWithGeneration('resource-agent-new');
it('refreshes canvas and asset library when Agent generation finishes', async () => {
let showGeneratedAsset = false;
loadEditorAssetLibraryMock.mockImplementation(async () => ({
folders: [
{
folderId: 'project',
label: '项目素材',
sortOrder: 0,
collapsed: false,
systemDefault: true,
},
],
assets: showGeneratedAsset
? [
...defaultEditorAssetLibraryAssets,
{
assetId: 'asset-agent-stream',
folderId: 'project',
label: 'Agent结果素材',
imageSrc: '/agent-stream.png',
width: 512,
height: 512,
sourceType: 'generated',
assetKind: 'editor_agent_generated_image',
},
]
: defaultEditorAssetLibraryAssets,
}));
listEditorAgentConversationsMock.mockResolvedValueOnce([
createEditorAgentConversationSummary(),
]);
getEditorAgentConversationMock.mockResolvedValueOnce(detail);
getEditorAgentConversationMock.mockResolvedValueOnce({
...createEditorAgentConversationSummary(),
messages: [],
});
streamEditorAgentMessageMock.mockImplementation(
async (conversationId, _payload, options) => {
options.onEvent?.({
event: 'message_delta',
data: {
conversationId,
messageId: 'assistant-agent-stream',
role: 'assistant',
kind: 'chat',
textDelta: '我来生成图片。',
},
});
showGeneratedAsset = true;
options.onEvent?.({
event: 'generation_result',
data: {
conversationId,
messageId: 'assistant-agent-stream',
toolCallId: 'tool-call-agent-stream',
toolName: 'generate_image',
model: 'gpt-image-2',
images: [
{
resourceId: 'resource-agent-stream',
imageSrc: '/agent-stream.png',
thumbnailSrc: null,
width: 512,
height: 512,
},
],
},
});
options.onEvent?.({
event: 'done',
data: { conversationId, title: null },
});
},
);
loadEditorProjectMock.mockResolvedValueOnce({
projectId: 'editor-project-default',
title: '默认项目',
@@ -1544,9 +1657,10 @@ describe('ImageCanvasEditorView', () => {
layers: [
...defaultEditorProjectLayers,
{
layerId: 'layer-agent-new',
resourceId: 'resource-agent-new',
layerId: 'layer-agent-stream',
resourceId: 'resource-agent-stream',
title: 'Agent结果图',
src: '/agent-stream.png',
x: 1500,
y: 300,
width: 512,
@@ -1560,38 +1674,44 @@ describe('ImageCanvasEditorView', () => {
resources: [
...defaultEditorProjectResources,
{
resourceId: 'resource-agent-new',
resourceId: 'resource-agent-stream',
projectId: 'editor-project-default',
imageSrc: '/agent-result.png',
imageSrc: '/agent-stream.png',
width: 512,
height: 512,
sourceType: 'generated',
prompt: 'Agent生成结果',
model: 'gpt-image-2',
provider: 'VectorEngine',
taskId: 'task-agent-1',
taskId: 'task-agent-stream',
assetKind: 'editor_agent_generated_image',
},
],
updatedAt: '2026-07-03T00:00:00.000Z',
updatedAt: '2026-07-03T00:02:00.000Z',
});
render(<ImageCanvasEditorView />);
fireEvent.click(screen.getByRole('button', { name: '画布 Agent' }));
fireEvent.click(await screen.findByRole('button', { name: '生成结果' }));
await waitFor(() => {
expect(getEditorAgentConversationMock).toHaveBeenCalledWith(
'editor-agent-conv-test',
);
});
fireEvent.change(await screen.findByLabelText('发送给画布 Agent'), {
target: { value: '生成一张图' },
});
fireEvent.click(screen.getByRole('button', { name: '发送' }));
await waitFor(() => {
expect(loadEditorProjectMock).toHaveBeenCalledWith(
'editor-project-default',
);
});
await waitFor(() => {
expect(
screen.getByAltText('画布图片:Agent结果图').closest('button')
?.className,
).toContain('image-canvas-editor__layer--selected');
});
expect(await screen.findByAltText('画布图片:Agent结果图')).toBeTruthy();
fireEvent.click(screen.getByRole('button', { name: '打开素材' }));
expect(await screen.findByText('Agent结果素材')).toBeTruthy();
});
it('resets the canvas view without forwarding the click event to fit layers', () => {
@@ -10,6 +10,7 @@ import {
} from 'react';
import type { ExternalGenerationTaskRecord } from '../../../packages/shared/src/contracts/externalGeneration';
import type { EditorAgentGenerationResultEvent } from '../../../packages/shared/src/contracts/editorAgent';
import {
createEditorAsset,
createEditorProjectResource,
@@ -1008,40 +1009,19 @@ export function ImageCanvasEditorView() {
},
[applyProjectSnapshot, refreshAssetLibrary],
);
const focusCanvasLayerByResourceId = useCallback(
(resourceId: string) => {
const targetLayer = layersRef.current.find(
(layer) => layer.resourceId === resourceId || layer.id === resourceId,
const handleEditorAgentGenerationResult = useCallback(
(event: EditorAgentGenerationResultEvent) => {
const hasGeneratedResource = event.images.some((image) =>
image.resourceId?.trim(),
);
if (targetLayer) {
focusCanvasLayerById(targetLayer.id);
return;
}
if (!projectId) {
if (!projectId || !hasGeneratedResource) {
return;
}
void loadEditorProject(projectId)
.then((project) => {
applyGeneratedProjectSnapshot(project);
const targetLayerItem = project.layers.find(
(layer) =>
layer.resourceId === resourceId || layer.layerId === resourceId,
);
const targetLayerId = targetLayerItem?.layerId;
if (!targetLayerId) {
return;
}
window.setTimeout(() => {
focusCanvasLayerById(targetLayerId);
}, 0);
})
.then(applyGeneratedProjectSnapshot)
.catch(() => undefined);
},
[
applyGeneratedProjectSnapshot,
focusCanvasLayerById,
projectId,
],
[applyGeneratedProjectSnapshot, projectId],
);
const persistUpdatedLayerResource = useCallback(
(layer: CanvasLayer) => {
@@ -1318,11 +1298,14 @@ export function ImageCanvasEditorView() {
clearCanvasFocus,
handleCanvasContextMenu,
handleLayerContextMenu,
handleGenerationFrameContextMenu,
} = useImageCanvasStageController({
layers,
canvasGenerationDialogs,
selectedLayerId,
selectedLayerIds,
setSelectedLayerId,
setSelectedLayerIds,
imageContextMenu,
setImageContextMenu,
contextMenu,
@@ -1979,9 +1962,10 @@ export function ImageCanvasEditorView() {
},
onUpdateLayerAssetKind: updateLayerAssetKind,
onGenerationFramePointerDown: handleGenerationFramePointerDown,
onGenerationFrameContextMenu: handleGenerationFrameContextMenu,
onActivateGenerationDialog: activateCanvasGenerationDialog,
onFocusExternalTask: focusExternalGenerationTask,
onFocusEditorAgentResource: focusCanvasLayerByResourceId,
onEditorAgentGenerationResult: handleEditorAgentGenerationResult,
onToggleTaskSidebar: toggleTaskSidebar,
onToggleAgentConversation: toggleAgentConversation,
onCropExpandHandlePointerDown: generationSurface.startCropExpandFrameResize,
@@ -3,6 +3,7 @@ import type {
CanvasContextMenuState,
CanvasLayer,
} from './ImageCanvasEditorTypes';
import { getCanvasGenerationSelectionId } from './ImageCanvasSelectionModel';
export type CanvasLayerMoveMode = 'up' | 'down' | 'top' | 'bottom';
@@ -31,12 +32,14 @@ export function resolveContextTargetLayerIds(
menu: CanvasContextMenuState | null,
selectedLayerIds: string[],
) {
if (menu?.kind !== 'layer') {
if (!menu || menu.kind === 'blank') {
return [];
}
return selectedLayerIds.includes(menu.layerId)
? [...selectedLayerIds]
: [menu.layerId];
const targetId =
menu.kind === 'generation-dialog'
? getCanvasGenerationSelectionId(menu.dialogId)
: menu.layerId;
return selectedLayerIds.includes(targetId) ? [...selectedLayerIds] : [targetId];
}
export function getCanvasLayersByIds(
@@ -57,6 +57,9 @@ describe('ImageCanvasPanelDockView', () => {
const toolbar = screen.getByRole('toolbar', { name: '画布面板入口' });
expect(toolbar.className).toContain(
'image-canvas-editor__panel-dock--sidebar-open',
);
expect(
within(toolbar)
.getByRole('button', { name: '打开素材' })
@@ -90,7 +90,14 @@ export function ImageCanvasPanelDockView({
/>
<div
className="image-canvas-editor__panel-dock"
className={[
'image-canvas-editor__panel-dock',
activeSidebarPanel
? 'image-canvas-editor__panel-dock--sidebar-open'
: '',
]
.filter(Boolean)
.join(' ')}
role="toolbar"
aria-label="画布面板入口"
onPointerDown={(event) => event.stopPropagation()}
@@ -135,3 +135,22 @@ export function createLayerCanvasContextMenus({
},
};
}
export function createGenerationDialogCanvasContextMenu({
clientX,
clientY,
dialogId,
canvasPoint,
}: {
clientX: number;
clientY: number;
dialogId: string;
canvasPoint: { x: number; y: number };
}): CanvasContextMenuState {
return {
kind: 'generation-dialog',
dialogId,
...resolveContextMenuPosition(clientX, clientY, 'generation-dialog'),
canvasPoint,
};
}
@@ -8,6 +8,7 @@ import type {
} from 'react';
import type { ExternalGenerationTaskRecord } from '../../../packages/shared/src/contracts/externalGeneration';
import type { EditorAgentGenerationResultEvent } from '../../../packages/shared/src/contracts/editorAgent';
import { EditorAgentConversationPanelView } from './EditorAgentConversationPanelView';
import { ImageCanvasBottomToolbarView } from './ImageCanvasBottomToolbarView';
import { ImageCanvasContextMenusView } from './ImageCanvasContextMenusView';
@@ -118,9 +119,15 @@ export type ImageCanvasStageViewProps = {
event: ReactPointerEvent<HTMLDivElement>,
dialog: CanvasGenerationDialogState,
) => void;
onGenerationFrameContextMenu: (
event: ReactMouseEvent<HTMLDivElement>,
dialog: CanvasGenerationDialogState,
) => void;
onActivateGenerationDialog: (dialog: CanvasGenerationDialogState) => void;
onFocusExternalTask: (task: ExternalGenerationTaskRecord) => void;
onFocusEditorAgentResource?: (resourceId: string) => void;
onEditorAgentGenerationResult?: (
event: EditorAgentGenerationResultEvent,
) => void;
onToggleTaskSidebar: () => void;
onToggleAgentConversation: () => void;
onCropExpandHandlePointerDown: (
@@ -247,9 +254,10 @@ export function ImageCanvasStageView({
onOpenLayerMetadata,
onUpdateLayerAssetKind,
onGenerationFramePointerDown,
onGenerationFrameContextMenu,
onActivateGenerationDialog,
onFocusExternalTask,
onFocusEditorAgentResource,
onEditorAgentGenerationResult,
onToggleTaskSidebar,
onToggleAgentConversation,
onCropExpandHandlePointerDown,
@@ -347,6 +355,7 @@ export function ImageCanvasStageView({
onOpenLayerMetadata={onOpenLayerMetadata}
onUpdateLayerAssetKind={onUpdateLayerAssetKind}
onGenerationFramePointerDown={onGenerationFramePointerDown}
onGenerationFrameContextMenu={onGenerationFrameContextMenu}
onActivateGenerationDialog={onActivateGenerationDialog}
onCropExpandHandlePointerDown={onCropExpandHandlePointerDown}
/>
@@ -464,7 +473,7 @@ export function ImageCanvasStageView({
onToggleOpen={onToggleAgentConversation}
layers={layers}
assets={editorAgentAssets}
onFocusResource={onFocusEditorAgentResource}
onGenerationResult={onEditorAgentGenerationResult}
/>
{isToolbarGuideVisible ? (
@@ -85,6 +85,7 @@ function renderWorldView(
onOpenLayerMetadata: vi.fn(),
onUpdateLayerAssetKind: vi.fn(),
onGenerationFramePointerDown: vi.fn(),
onGenerationFrameContextMenu: vi.fn(),
onActivateGenerationDialog: vi.fn(),
onCropExpandHandlePointerDown: vi.fn(),
...overrides,
@@ -274,6 +274,10 @@ export type ImageCanvasWorldViewProps = {
event: ReactPointerEvent<HTMLDivElement>,
dialog: CanvasGenerationDialogState,
) => void;
onGenerationFrameContextMenu: (
event: ReactMouseEvent<HTMLDivElement>,
dialog: CanvasGenerationDialogState,
) => void;
onActivateGenerationDialog: (dialog: CanvasGenerationDialogState) => void;
onCropExpandHandlePointerDown: (
event: ReactPointerEvent<HTMLButtonElement>,
@@ -921,6 +925,7 @@ export function ImageCanvasWorldView({
onOpenLayerMetadata,
onUpdateLayerAssetKind,
onGenerationFramePointerDown,
onGenerationFrameContextMenu,
onActivateGenerationDialog,
onCropExpandHandlePointerDown,
}: ImageCanvasWorldViewProps) {
@@ -1259,6 +1264,9 @@ export function ImageCanvasWorldView({
onPointerDown={(event) =>
onGenerationFramePointerDown(event, dialog)
}
onContextMenu={(event) =>
onGenerationFrameContextMenu(event, dialog)
}
onDoubleClick={() => onActivateGenerationDialog(dialog)}
>
{showFocusedChrome ? (
@@ -111,8 +111,13 @@ describe('useEditorAgentConversation', () => {
it('loads conversations and applies message stream events', async () => {
const client = createClient();
const onGenerationResult = vi.fn();
const { result } = renderHook(() =>
useEditorAgentConversation({ projectId: 'project-1', client }),
useEditorAgentConversation({
projectId: 'project-1',
client,
onGenerationResult,
}),
);
await waitFor(() => {
@@ -138,6 +143,16 @@ describe('useEditorAgentConversation', () => {
);
expect(result.current.stage).toBe('completed');
expect(result.current.isStreaming).toBe(false);
expect(onGenerationResult).toHaveBeenCalledWith(
expect.objectContaining({
toolCallId: 'tool-call-1',
images: [
expect.objectContaining({
resourceId: 'resource-result-1',
}),
],
}),
);
expect(result.current.messages.map((message) => message.text)).toEqual([
'把这个角色改成像素风',
'我来处理',
@@ -221,6 +236,65 @@ describe('useEditorAgentConversation', () => {
);
});
it('keeps LLM planning failures as one failed assistant message', async () => {
const client = createClient();
vi.mocked(client.streamMessage).mockImplementation(
async (_conversationId, _payload, options) => {
options.onEvent?.({
event: 'message_delta',
data: {
conversationId: 'conversation-1',
messageId: 'assistant-planning-error',
role: 'assistant',
kind: 'error',
textDelta: '画布 Agent 的 LLM 未配置,无法处理这句话。',
},
});
options.onEvent?.({
event: 'stage',
data: {
conversationId: 'conversation-1',
stage: 'failed',
},
});
options.onEvent?.({
event: 'done',
data: {
conversationId: 'conversation-1',
title: null,
},
});
},
);
const { result } = renderHook(() =>
useEditorAgentConversation({ projectId: 'project-1', client }),
);
await waitFor(() => {
expect(result.current.activeConversation?.conversationId).toBe(
'conversation-1',
);
});
await act(async () => {
await result.current.sendMessage('这是美术素材');
});
expect(result.current.stage).toBe('failed');
expect(result.current.errorMessage).toBe(
'画布 Agent 的 LLM 未配置,无法处理这句话。',
);
expect(
result.current.messages.filter((message) => message.kind === 'error'),
).toEqual([
expect.objectContaining({
id: 'assistant-planning-error',
text: '画布 Agent 的 LLM 未配置,无法处理这句话。',
status: 'failed',
}),
]);
});
it('shows a failed tool completion as a generation record', async () => {
const client = createClient();
vi.mocked(client.streamMessage).mockImplementation(
@@ -5,6 +5,7 @@ import type {
EditorAgentAttachmentRef,
EditorAgentConversationDetail,
EditorAgentConversationSummary,
EditorAgentGenerationResultEvent,
EditorAgentGenerationRecord,
EditorAgentMessage,
EditorAgentSseEvent,
@@ -44,6 +45,7 @@ export type EditorAgentConversationClient = {
type UseEditorAgentConversationOptions = {
projectId?: string | null;
client?: EditorAgentConversationClient;
onGenerationResult?: (event: EditorAgentGenerationResultEvent) => void;
};
const defaultEditorAgentConversationClient: EditorAgentConversationClient = {
@@ -208,6 +210,7 @@ function upsertConversationSummary(
export function useEditorAgentConversation({
projectId,
client = defaultEditorAgentConversationClient,
onGenerationResult,
}: UseEditorAgentConversationOptions) {
const normalizedProjectId = projectId?.trim() ?? '';
const [conversations, setConversations] = useState<
@@ -377,6 +380,12 @@ export function useEditorAgentConversation({
}
if (event.event === 'message_delta') {
if (event.data.kind === 'error') {
const nextErrorMessage = event.data.textDelta.trim();
if (nextErrorMessage) {
setErrorMessage(nextErrorMessage);
}
}
setMessages((currentMessages) =>
upsertMessage(
currentMessages,
@@ -437,6 +446,7 @@ export function useEditorAgentConversation({
}
if (event.event === 'generation_result') {
onGenerationResult?.(event.data);
const nextRecord: EditorAgentGenerationRecord = {
toolCallId: event.data.toolCallId,
toolName: event.data.toolName,
@@ -485,7 +495,9 @@ export function useEditorAgentConversation({
}
if (event.event === 'done') {
setStage('completed');
setStage((currentStage) =>
currentStage === 'failed' ? 'failed' : 'completed',
);
setMessages((currentMessages) =>
markStreamingMessages(currentMessages, 'completed'),
);
@@ -500,7 +512,7 @@ export function useEditorAgentConversation({
);
}
}
}, []);
}, [onGenerationResult]);
const ensureConversationForSend = useCallback(async () => {
if (activeConversationId) {
@@ -496,21 +496,41 @@ export function useImageCanvasLayerCommands({
const deleteContextLayers = useCallback(() => {
const targetIds = getContextTargetLayerIds();
if (!targetIds.length) {
const targetLayerIds = getSelectedLayerIds(targetIds);
const targetDialogIds = targetIds
.map(getCanvasGenerationDialogIdFromSelectionId)
.filter((dialogId): dialogId is string => Boolean(dialogId))
.filter((dialogId) =>
canvasGenerationDialogs.some((dialog) => dialog.id === dialogId),
);
if (!targetLayerIds.length && !targetDialogIds.length) {
return;
}
captureCanvasHistory();
setLayers((currentLayers) => removeCanvasLayers(currentLayers, targetIds));
if (targetLayerIds.length) {
setLayers((currentLayers) =>
removeCanvasLayers(currentLayers, targetLayerIds),
);
}
selectSingleLayer(null);
setHoveredLayerId(null);
setMetadataLayer((currentLayer) =>
currentLayer && targetIds.includes(currentLayer.id) ? null : currentLayer,
currentLayer && targetLayerIds.includes(currentLayer.id)
? null
: currentLayer,
);
targetLayerIds.forEach((targetId) => onDeleteLayerSideEffects(targetId));
targetDialogIds.forEach((targetId) =>
onDeleteGenerationDialogSideEffects?.(targetId),
);
closeContextMenus();
}, [
captureCanvasHistory,
canvasGenerationDialogs,
closeContextMenus,
getContextTargetLayerIds,
onDeleteGenerationDialogSideEffects,
onDeleteLayerSideEffects,
selectSingleLayer,
setHoveredLayerId,
setLayers,
@@ -53,6 +53,8 @@ function StageControllerHarness({
layers,
selectedLayerId,
selectedLayerIds,
setSelectedLayerId,
setSelectedLayerIds,
imageContextMenu,
setImageContextMenu,
contextMenu,
@@ -15,15 +15,19 @@ import type {
} from './ImageCanvasEditorTypes';
import {
createBlankCanvasContextMenu,
createGenerationDialogCanvasContextMenu,
createLayerCanvasContextMenus,
resolveImageCanvasStageControllerModel,
} from './ImageCanvasStageControllerModel';
import { getCanvasGenerationSelectionId } from './ImageCanvasSelectionModel';
type UseImageCanvasStageControllerOptions = {
layers: CanvasLayer[];
canvasGenerationDialogs?: CanvasGenerationDialogState[];
selectedLayerId: string | null;
selectedLayerIds: string[];
setSelectedLayerId: Dispatch<SetStateAction<string | null>>;
setSelectedLayerIds: Dispatch<SetStateAction<string[]>>;
imageContextMenu: ImageContextMenuState | null;
setImageContextMenu: Dispatch<SetStateAction<ImageContextMenuState | null>>;
contextMenu: CanvasContextMenuState | null;
@@ -44,6 +48,8 @@ export function useImageCanvasStageController({
canvasGenerationDialogs,
selectedLayerId,
selectedLayerIds,
setSelectedLayerId,
setSelectedLayerIds,
imageContextMenu,
setImageContextMenu,
contextMenu,
@@ -134,10 +140,42 @@ export function useImageCanvasStageController({
],
);
const handleGenerationFrameContextMenu = useCallback(
(
event: ReactMouseEvent<HTMLElement>,
dialog: CanvasGenerationDialogState,
) => {
event.preventDefault();
event.stopPropagation();
const selectionId = getCanvasGenerationSelectionId(dialog.id);
if (!selectedLayerIds.includes(selectionId)) {
setSelectedLayerId(null);
setSelectedLayerIds([selectionId]);
}
const nextMenu = createGenerationDialogCanvasContextMenu({
clientX: event.clientX,
clientY: event.clientY,
dialogId: dialog.id,
canvasPoint: getCanvasPointFromClient(event.clientX, event.clientY),
});
setContextMenu(nextMenu);
setImageContextMenu(null);
},
[
getCanvasPointFromClient,
selectedLayerIds,
setContextMenu,
setImageContextMenu,
setSelectedLayerId,
setSelectedLayerIds,
],
);
return {
...model,
clearCanvasFocus,
handleCanvasContextMenu,
handleLayerContextMenu,
handleGenerationFrameContextMenu,
};
}
@@ -93,6 +93,10 @@ type UseImageCanvasStageInteractionsOptions = {
onCloseImageContextMenu: () => void;
};
function focusCanvasInteractionTarget(target: HTMLElement) {
target.focus({ preventScroll: true });
}
export function useImageCanvasStageInteractions({
canvasViewportRef,
activeTool,
@@ -248,6 +252,7 @@ export function useImageCanvasStageInteractions({
generateDialog?.mode === 'video' ||
generateDialog?.mode === 'spec')
) {
focusCanvasInteractionTarget(event.currentTarget);
event.preventDefault();
event.stopPropagation();
suppressNextLayerClickRef.current = true;
@@ -255,6 +260,7 @@ export function useImageCanvasStageInteractions({
return;
}
if (isPickingQuickEditReferenceFromCanvas) {
focusCanvasInteractionTarget(event.currentTarget);
event.preventDefault();
event.stopPropagation();
suppressNextLayerClickRef.current = true;
@@ -265,6 +271,7 @@ export function useImageCanvasStageInteractions({
isPickingCharacterSpecFromCanvas &&
generateDialog?.mode === 'character'
) {
focusCanvasInteractionTarget(event.currentTarget);
event.preventDefault();
event.stopPropagation();
suppressNextLayerClickRef.current = true;
@@ -275,6 +282,7 @@ export function useImageCanvasStageInteractions({
isPickingCharacterReferenceFromCanvas &&
generateDialog?.mode === 'character'
) {
focusCanvasInteractionTarget(event.currentTarget);
event.preventDefault();
event.stopPropagation();
suppressNextLayerClickRef.current = true;
@@ -282,6 +290,7 @@ export function useImageCanvasStageInteractions({
return;
}
if (isPickingIconSpecFromCanvas && generateDialog?.mode === 'icon') {
focusCanvasInteractionTarget(event.currentTarget);
event.preventDefault();
event.stopPropagation();
suppressNextLayerClickRef.current = true;
@@ -292,6 +301,7 @@ export function useImageCanvasStageInteractions({
isPickingUiDesignSpecFromCanvas &&
generateDialog?.mode === 'ui-design'
) {
focusCanvasInteractionTarget(event.currentTarget);
event.preventDefault();
event.stopPropagation();
suppressNextLayerClickRef.current = true;
@@ -302,6 +312,7 @@ export function useImageCanvasStageInteractions({
isPickingPublicationReferenceFromCanvas &&
generateDialog?.mode === 'publication'
) {
focusCanvasInteractionTarget(event.currentTarget);
event.preventDefault();
event.stopPropagation();
suppressNextLayerClickRef.current = true;
@@ -309,6 +320,7 @@ export function useImageCanvasStageInteractions({
return;
}
focusCanvasInteractionTarget(event.currentTarget);
event.preventDefault();
event.stopPropagation();
const pointer = getPointerClient(event);
@@ -431,6 +443,7 @@ export function useImageCanvasStageInteractions({
return;
}
focusCanvasInteractionTarget(event.currentTarget);
event.preventDefault();
event.stopPropagation();
const pointer = getPointerClient(event);
+18
View File
@@ -4321,6 +4321,20 @@ html[data-mobile-keyboard-open='true'] .platform-mobile-bottom-dock {
user-select: none;
}
.image-canvas-editor [aria-label='画布 Agent 对话'],
.image-canvas-editor [aria-label='画布 Agent 对话'] * {
-webkit-touch-callout: default;
-webkit-user-select: text;
user-select: text;
}
.image-canvas-editor [aria-label='画布 Agent 对话'] button,
.image-canvas-editor [aria-label='画布 Agent 对话'] select,
.image-canvas-editor [aria-label='画布 Agent 对话'] textarea {
-webkit-user-select: auto;
user-select: auto;
}
.image-editor-creation-entry-stack {
display: grid;
min-width: 0;
@@ -6556,6 +6570,10 @@ html[data-mobile-keyboard-open='true'] .platform-mobile-bottom-dock {
box-shadow: 0 16px 34px rgba(15, 23, 42, 0.14);
}
.image-canvas-editor__panel-dock--sidebar-open {
bottom: 4.2rem;
}
.image-canvas-editor__panel-dock button {
display: inline-flex;
width: 2.25rem;