From 0c14e514191c76dcfc11aedbcf79213d803592f6 Mon Sep 17 00:00:00 2001 From: kvtodev Date: Tue, 7 Jul 2026 14:56:28 +0800 Subject: [PATCH] Editor agent paste image (#72) ![shotmd-1783397773.webp](/attachments/659f35ce-87c1-449d-8342-8afc09934da8) --------- Co-authored-by: kdletters Reviewed-on: http://genarrative-station/git/GenarrativeAI/Genarrative/pulls/72 Reviewed-by: kdletters Co-authored-by: kvtodev Co-committed-by: kvtodev --- package.json | 3 +- .../EditorAgentConversationPanelView.test.tsx | 133 ++++++++++++++++-- .../EditorAgentConversationPanelView.tsx | 127 ++++++++++++++++- .../ImageCanvasEditorShellView.test.tsx | 2 - .../image-editor/ImageCanvasEditorView.tsx | 9 +- .../image-editor/ImageCanvasStageView.tsx | 4 - .../ImageCanvasTaskSidebarView.test.tsx | 8 +- .../ImageCanvasTaskSidebarView.tsx | 4 +- .../ImageCanvasTopbarView.test.tsx | 5 +- .../image-editor/ImageCanvasTopbarView.tsx | 4 +- .../useImageCanvasContextStore.ts | 14 ++ 11 files changed, 281 insertions(+), 32 deletions(-) create mode 100644 src/components/image-editor/useImageCanvasContextStore.ts diff --git a/package.json b/package.json index 87c2d3280..0136a88a0 100644 --- a/package.json +++ b/package.json @@ -150,7 +150,8 @@ "react-native-safe-area-context": "^5.8.0", "react-native-webview": "^13.16.1", "three": "^0.184.0", - "vite": "^6.2.0" + "vite": "^6.2.0", + "zustand": "^5.0.14" }, "devDependencies": { "@colbymchenry/codegraph": "^0.8.0", diff --git a/src/components/image-editor/EditorAgentConversationPanelView.test.tsx b/src/components/image-editor/EditorAgentConversationPanelView.test.tsx index c6b7dc8b1..30bf11244 100644 --- a/src/components/image-editor/EditorAgentConversationPanelView.test.tsx +++ b/src/components/image-editor/EditorAgentConversationPanelView.test.tsx @@ -8,11 +8,40 @@ import { waitFor, within, } from '@testing-library/react'; -import { describe, expect, it, vi } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { EditorAgentConversationPanelView } from './EditorAgentConversationPanelView'; +import { useImageCanvasContextStore } from './useImageCanvasContextStore.ts'; import type { EditorAgentConversationClient } from './useEditorAgentConversation'; +const createEditorProjectResourceMock = vi.hoisted(() => vi.fn()); +const uploadEditorMediaAssetFileMock = vi.hoisted(() => vi.fn()); +const probeImageFileDimensionsMock = vi.hoisted(() => vi.fn()); + +vi.mock('../../services/image-editor/editorProjectClient', async () => { + const actual = await vi.importActual< + typeof import('../../services/image-editor/editorProjectClient') + >('../../services/image-editor/editorProjectClient'); + return { + ...actual, + createEditorProjectResource: createEditorProjectResourceMock, + }; +}); + +vi.mock('../../services/image-editor/editorMediaAssetUploadClient', () => ({ + uploadEditorMediaAssetFile: uploadEditorMediaAssetFileMock, +})); + +vi.mock('./ImageCanvasFileModel', async () => { + const actual = await vi.importActual( + './ImageCanvasFileModel', + ); + return { + ...actual, + probeImageFileDimensions: probeImageFileDimensionsMock, + }; +}); + function createClient(): EditorAgentConversationClient { return { listConversations: vi.fn().mockResolvedValue([ @@ -85,12 +114,33 @@ function createClient(): EditorAgentConversationClient { } describe('EditorAgentConversationPanelView', () => { + beforeEach(() => { + useImageCanvasContextStore.getState().setProjectId('project-1'); + uploadEditorMediaAssetFileMock.mockReset(); + uploadEditorMediaAssetFileMock.mockResolvedValue({ + src: '/generated/pasted.png', + objectKey: 'generated-character-drafts/editor/agent-paste/image/pasted.png', + assetObjectId: 'asset-object-pasted', + legacyPublicPath: '/generated/pasted.png', + }); + probeImageFileDimensionsMock.mockReset(); + probeImageFileDimensionsMock.mockResolvedValue({ width: 320, height: 240 }); + createEditorProjectResourceMock.mockReset(); + createEditorProjectResourceMock.mockResolvedValue({ + resourceId: 'resource-pasted', + imageSrc: '/generated/pasted.png', + objectKey: 'generated-character-drafts/editor/agent-paste/image/pasted.png', + label: '粘贴图片', + width: 320, + height: 240, + }); + }); + it('manages conversations, attachments and message sending inside the panel', async () => { const client = createClient(); render( { ); }); + it('uploads pasted images as canvas attachments before sending', async () => { + const client = createClient(); + + render( + , + ); + + await waitFor(() => { + expect(screen.getByText('已经看到画布内容')).toBeTruthy(); + }); + + const pastedImage = new File(['pasted-image'], 'pasted.png', { + type: 'image/png', + }); + const input = screen.getByLabelText('发送给画布 Agent'); + fireEvent.paste(input, { + clipboardData: { + files: [pastedImage], + }, + }); + + await waitFor(() => { + expect(uploadEditorMediaAssetFileMock).toHaveBeenCalledWith( + pastedImage, + 'image', + expect.objectContaining({ + entityId: 'project-1', + metadata: { source: 'agent-input-paste' }, + }), + ); + }); + expect(probeImageFileDimensionsMock).toHaveBeenCalledWith(pastedImage); + expect(createEditorProjectResourceMock).toHaveBeenCalledWith( + 'project-1', + expect.objectContaining({ + imageSrc: '/generated/pasted.png', + objectKey: 'generated-character-drafts/editor/agent-paste/image/pasted.png', + assetObjectId: 'asset-object-pasted', + width: 320, + height: 240, + sourceType: 'uploaded', + }), + ); + await waitFor(() => { + expect(screen.getByText('粘贴图片')).toBeTruthy(); + }); + + fireEvent.click(screen.getByRole('button', { name: '发送' })); + + await waitFor(() => { + expect(client.streamMessage).toHaveBeenCalledWith( + 'conversation-1', + expect.objectContaining({ + text: '', + attachments: [ + expect.objectContaining({ + source: 'canvas_resource', + referenceId: 'resource-pasted', + imageSrc: '/generated/pasted.png', + width: 320, + height: 240, + }), + ], + }), + expect.any(Object), + ); + }); + }); + it('sends selected attachments even when the text input is empty', async () => { const client = createClient(); render( { const { rerender } = render( { rerender( { rerender( { render(
{ const client = createClient(); render( void; layers?: CanvasLayer[]; @@ -404,7 +408,6 @@ function AttachmentPickerModal({ } export function EditorAgentConversationPanelView({ - projectId, open, onToggleOpen, layers = [], @@ -418,6 +421,9 @@ export function EditorAgentConversationPanelView({ setHasConversationMounted(true); } }, [open]); + const projectId = useImageCanvasContextStore( + (state) => state.projectId, + ); const effectiveProjectId = hasConversationMounted ? projectId : null; const { conversations, @@ -448,6 +454,7 @@ export function EditorAgentConversationPanelView({ const [attachmentPickerTab, setAttachmentPickerTab] = useState('canvas'); const [attachmentError, setAttachmentError] = useState(null); + const [isPastingAttachment, setIsPastingAttachment] = useState(false); const [draftAttachmentKeys, setDraftAttachmentKeys] = useState>( () => new Set(), ); @@ -510,6 +517,9 @@ export function EditorAgentConversationPanelView({ stopCurrentTurn(); return; } + if (isPastingAttachment) { + return; + } const text = draftText.trim(); if (!text && !attachments.length) { return; @@ -524,6 +534,113 @@ export function EditorAgentConversationPanelView({ ); }); }; + const appendAttachments = (nextAttachments: EditorAgentAttachmentRef[]) => { + function mergeAttachments( + currentAttachments: EditorAgentAttachmentRef[], + nextAttachments: EditorAgentAttachmentRef[], + ) { + const merged = [...currentAttachments]; + const existingKeys = new Set(currentAttachments.map(attachmentKey)); + nextAttachments.forEach((attachment) => { + const key = attachmentKey(attachment); + if (!existingKeys.has(key)) { + existingKeys.add(key); + merged.push(attachment); + } + }); + return merged; + } + const mergedAttachments = mergeAttachments(attachments, nextAttachments); + if (mergedAttachments.length > EDITOR_AGENT_MAX_ATTACHMENTS) { + setAttachmentError(`最多 ${EDITOR_AGENT_MAX_ATTACHMENTS} 张`); + return false; + } + setAttachments(mergedAttachments); + setAttachmentError(null); + return true; + }; + + const createPastedAgentImageAttachment = async ( + file: File, + ): Promise => { + if (!projectId?.trim()) { + throw new Error('缺少画布项目'); + } + const [upload, dimensions] = await Promise.all([ + uploadEditorMediaAssetFile(file, 'image', { + pathSegments: ['editor', 'agent-paste', 'image', `${Date.now()}`], + entityId: projectId, + metadata: { + source: 'agent-input-paste', + }, + }), + probeImageFileDimensions(file), + ]); + const width = dimensions?.width ?? 1; + const height = dimensions?.height ?? 1; + const resource = await createEditorProjectResource(projectId, { + imageSrc: upload.src, + objectKey: upload.objectKey, + assetObjectId: upload.assetObjectId, + width, + height, + sourceType: 'uploaded', + }); + return { + source: 'canvas_resource', + referenceId: resource.resourceId, + objectKey: resource.objectKey ?? upload.objectKey, + imageSrc: resource.imageSrc, + thumbnailSrc: null, + label: resource.label ?? '粘贴图片', + width: resource.width, + height: resource.height, + }; + }; + function extractClipboardImageFiles( + clipboardData: DataTransfer | null, + ): File[] { + if (!clipboardData) { + return []; + } + const fileItems = Array.from(clipboardData.files ?? []).filter((file) => + file.type.startsWith('image/'), + ); + return [...fileItems]; + } + + const handleInputPaste = (event: ReactClipboardEvent) => { + const imageFiles = extractClipboardImageFiles(event.clipboardData); + if (!imageFiles.length) { + return; + } + + if (isPastingAttachment) { + return; + } + + event.preventDefault(); + if (attachments.length >= EDITOR_AGENT_MAX_ATTACHMENTS) { + setAttachmentError(`最多 ${EDITOR_AGENT_MAX_ATTACHMENTS} 张`); + return; + } + + // TODO: deduplicate those existing assets + setIsPastingAttachment(true); + setAttachmentError('图片上传中'); + void Promise.all( + imageFiles.map((file) => createPastedAgentImageAttachment(file)), + ) + .then((pastedAttachments) => { + appendAttachments(pastedAttachments); + }) + .catch(() => { + setAttachmentError('图片粘贴失败，请重试'); + }) + .finally(() => { + setIsPastingAttachment(false); + }); + }; const removeAttachment = (targetAttachment: EditorAgentAttachmentRef) => { const key = attachmentKey(targetAttachment); @@ -654,6 +771,11 @@ export function EditorAgentConversationPanelView({ {errorMessage}
) : null} + {attachmentError ? ( +
+ {attachmentError} +
+ ) : null}
setDraftText(event.currentTarget.value)} + onPaste={handleInputPaste} onKeyDown={(event) => { if (event.key === 'Enter' && !event.shiftKey) { event.preventDefault(); diff --git a/src/components/image-editor/ImageCanvasEditorShellView.test.tsx b/src/components/image-editor/ImageCanvasEditorShellView.test.tsx index dc5fa045a..bbfb494b6 100644 --- a/src/components/image-editor/ImageCanvasEditorShellView.test.tsx +++ b/src/components/image-editor/ImageCanvasEditorShellView.test.tsx @@ -93,7 +93,6 @@ function createSidebarProps(): ImageCanvasSidebarViewProps { function createTopbarProps(): ImageCanvasTopbarViewProps { return { - projectId: 'project-1', projectTitle: '默认项目', projectRenameValue: '默认项目', isRenamingProject: false, @@ -119,7 +118,6 @@ function createTopbarProps(): ImageCanvasTopbarViewProps { function createStageProps(): ImageCanvasStageViewProps { return { - projectId: 'project-1', canvasViewportRef: createRef(), specToolWrapRef: createRef(), musicToolWrapRef: createRef(), diff --git a/src/components/image-editor/ImageCanvasEditorView.tsx b/src/components/image-editor/ImageCanvasEditorView.tsx index 8f96c6ef3..10ae75459 100644 --- a/src/components/image-editor/ImageCanvasEditorView.tsx +++ b/src/components/image-editor/ImageCanvasEditorView.tsx @@ -82,6 +82,7 @@ import { useImageCanvasEditorChrome } from './useImageCanvasEditorChrome'; import { useImageCanvasGenerationSurface } from './useImageCanvasGenerationSurface'; import { useImageCanvasKeyboardShortcuts } from './useImageCanvasKeyboardShortcuts'; import { useImageCanvasLayerCommands } from './useImageCanvasLayerCommands'; +import { useImageCanvasContextStore } from './useImageCanvasContextStore.ts'; import { useImageCanvasProjectPersistence } from './useImageCanvasProjectPersistence'; import { useImageCanvasStageController } from './useImageCanvasStageController'; import { useImageCanvasStageInteractions } from './useImageCanvasStageInteractions'; @@ -1035,6 +1036,12 @@ export function ImageCanvasEditorView({ openEditorLoginModal, onProjectAccessLost, }); + const setEditorProjectContextId = useImageCanvasContextStore( + (state) => state.setProjectId, + ); + useEffect(() => { + setEditorProjectContextId(projectId); + }, [projectId, setEditorProjectContextId]); const applyGeneratedProjectSnapshot = useCallback( (project: EditorProjectSnapshot) => { applyProjectSnapshot(project); @@ -1906,7 +1913,6 @@ export function ImageCanvasEditorView({ getCanvasPointFromClient, }; const topbarProps = { - projectId, projectTitle, projectRenameValue, isRenamingProject, @@ -1935,7 +1941,6 @@ export function ImageCanvasEditorView({ }, }; const stageProps = { - projectId, canvasViewportRef, specToolWrapRef, musicToolWrapRef, diff --git a/src/components/image-editor/ImageCanvasStageView.tsx b/src/components/image-editor/ImageCanvasStageView.tsx index 68ba896b0..3867fb717 100644 --- a/src/components/image-editor/ImageCanvasStageView.tsx +++ b/src/components/image-editor/ImageCanvasStageView.tsx @@ -41,7 +41,6 @@ import { ImageCanvasUiAssetExtractionOverlayView } from './ImageCanvasUiAssetExt import { ImageCanvasWorldView } from './ImageCanvasWorldView'; export type ImageCanvasStageViewProps = { - projectId?: string | null; canvasViewportRef: RefObject; specToolWrapRef: RefObject; musicToolWrapRef: RefObject; @@ -192,7 +191,6 @@ export type ImageCanvasStageViewProps = { }; export function ImageCanvasStageView({ - projectId, canvasViewportRef, specToolWrapRef, musicToolWrapRef, @@ -463,7 +461,6 @@ export function ImageCanvasStageView({ /> ({ listExternalGenerationTasks: vi.fn().mockResolvedValue({ @@ -22,6 +23,7 @@ vi.mock('../../services/external-generation', () => ({ const listExternalGenerationTasksMock = vi.mocked(listExternalGenerationTasks); beforeEach(() => { + useImageCanvasContextStore.getState().setProjectId('project-1'); listExternalGenerationTasksMock.mockClear(); listExternalGenerationTasksMock.mockResolvedValue({ overview: { @@ -115,7 +117,6 @@ describe('ImageCanvasTaskSidebarView', () => { render(
{ render( { render( { const { rerender } = render( { ]; rerender( { render( void; @@ -252,12 +252,12 @@ function trimStoredExternalTasks( } export function ImageCanvasTaskSidebarView({ - projectId, refreshKey = 0, open, onToggleOpen, onFocusExternalTask, }: ImageCanvasTaskSidebarViewProps) { + const projectId = useImageCanvasContextStore((state) => state.projectId); const normalizedProjectId = projectId?.trim() ?? ''; const [activeTab, setActiveTab] = useState('active'); const [now, setNow] = useState(() => Date.now()); diff --git a/src/components/image-editor/ImageCanvasTopbarView.test.tsx b/src/components/image-editor/ImageCanvasTopbarView.test.tsx index b9f2c584b..ba0c69689 100644 --- a/src/components/image-editor/ImageCanvasTopbarView.test.tsx +++ b/src/components/image-editor/ImageCanvasTopbarView.test.tsx @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from 'vitest'; import type { CanvasLayer } from './ImageCanvasEditorTypes'; import { ImageCanvasTopbarView } from './ImageCanvasTopbarView'; +import { useImageCanvasContextStore } from './useImageCanvasContextStore.ts'; function createLayer(overrides: Partial = {}): CanvasLayer { const id = overrides.id ?? 'layer-a'; @@ -30,8 +31,8 @@ function renderTopbar( Parameters[0] > = {}, ) { + useImageCanvasContextStore.getState().setProjectId('project-a'); const props: Parameters[0] = { - projectId: 'project-a', projectTitle: '默认项目', projectRenameValue: '默认项目', isRenamingProject: false, @@ -144,7 +145,6 @@ describe('ImageCanvasTopbarView', () => { const exportCanvasAssets = vi.fn(); const { rerender } = render( { rerender( state.projectId); const hasExportableLayer = layers.some( (layer) => layer.src.trim().length > 0, ); diff --git a/src/components/image-editor/useImageCanvasContextStore.ts b/src/components/image-editor/useImageCanvasContextStore.ts new file mode 100644 index 000000000..0869656f5 --- /dev/null +++ b/src/components/image-editor/useImageCanvasContextStore.ts @@ -0,0 +1,14 @@ +import { create } from 'zustand'; + +type ImageCanvasContextState = { + projectId: string | null; + setProjectId: (projectId?: string | null) => void; +}; + +export const useImageCanvasContextStore = + create((set) => ({ + projectId: null, + setProjectId: (projectId) => { + set({ projectId: projectId?.trim() || null }); + }, + }));