Editor agent paste image (#72)

![shotmd-1783397773.webp](/attachments/659f35ce-87c1-449d-8342-8afc09934da8)

---------

Co-authored-by: kdletters <kdletters@qq.com>
Reviewed-on: https://git.genarrative.world/git/GenarrativeAI/Genarrative/pulls/72
Reviewed-by: kdletters <kdletters@qq.com>
Co-authored-by: kvtodev <kvtodev@outlook.com>
Co-committed-by: kvtodev <kvtodev@outlook.com>
This commit was merged in pull request #72.
This commit is contained in:
2026-07-07 14:56:28 +08:00
committed by kdletters
parent 5477abd7ba
commit 0c14e51419
11 changed files with 281 additions and 32 deletions
+2 -1
View File
@@ -150,7 +150,8 @@
"react-native-safe-area-context": "^5.8.0",
"react-native-webview": "^13.16.1",
"three": "^0.184.0",
"vite": "^6.2.0"
"vite": "^6.2.0",
"zustand": "^5.0.14"
},
"devDependencies": {
"@colbymchenry/codegraph": "^0.8.0",
@@ -8,11 +8,40 @@ import {
waitFor,
within,
} from '@testing-library/react';
import { describe, expect, it, vi } from 'vitest';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { EditorAgentConversationPanelView } from './EditorAgentConversationPanelView';
import { useImageCanvasContextStore } from './useImageCanvasContextStore.ts';
import type { EditorAgentConversationClient } from './useEditorAgentConversation';
const createEditorProjectResourceMock = vi.hoisted(() => vi.fn());
const uploadEditorMediaAssetFileMock = vi.hoisted(() => vi.fn());
const probeImageFileDimensionsMock = vi.hoisted(() => vi.fn());
vi.mock('../../services/image-editor/editorProjectClient', async () => {
const actual = await vi.importActual<
typeof import('../../services/image-editor/editorProjectClient')
>('../../services/image-editor/editorProjectClient');
return {
...actual,
createEditorProjectResource: createEditorProjectResourceMock,
};
});
vi.mock('../../services/image-editor/editorMediaAssetUploadClient', () => ({
uploadEditorMediaAssetFile: uploadEditorMediaAssetFileMock,
}));
vi.mock('./ImageCanvasFileModel', async () => {
const actual = await vi.importActual<typeof import('./ImageCanvasFileModel')>(
'./ImageCanvasFileModel',
);
return {
...actual,
probeImageFileDimensions: probeImageFileDimensionsMock,
};
});
function createClient(): EditorAgentConversationClient {
return {
listConversations: vi.fn().mockResolvedValue([
@@ -85,12 +114,33 @@ function createClient(): EditorAgentConversationClient {
}
describe('EditorAgentConversationPanelView', () => {
beforeEach(() => {
useImageCanvasContextStore.getState().setProjectId('project-1');
uploadEditorMediaAssetFileMock.mockReset();
uploadEditorMediaAssetFileMock.mockResolvedValue({
src: '/generated/pasted.png',
objectKey: 'generated-character-drafts/editor/agent-paste/image/pasted.png',
assetObjectId: 'asset-object-pasted',
legacyPublicPath: '/generated/pasted.png',
});
probeImageFileDimensionsMock.mockReset();
probeImageFileDimensionsMock.mockResolvedValue({ width: 320, height: 240 });
createEditorProjectResourceMock.mockReset();
createEditorProjectResourceMock.mockResolvedValue({
resourceId: 'resource-pasted',
imageSrc: '/generated/pasted.png',
objectKey: 'generated-character-drafts/editor/agent-paste/image/pasted.png',
label: '粘贴图片',
width: 320,
height: 240,
});
});
it('manages conversations, attachments and message sending inside the panel', async () => {
const client = createClient();
render(
<EditorAgentConversationPanelView
projectId="project-1"
open
onToggleOpen={vi.fn()}
client={client}
@@ -191,12 +241,84 @@ describe('EditorAgentConversationPanelView', () => {
);
});
it('uploads pasted images as canvas attachments before sending', async () => {
const client = createClient();
render(
<EditorAgentConversationPanelView
open
onToggleOpen={vi.fn()}
client={client}
/>,
);
await waitFor(() => {
expect(screen.getByText('已经看到画布内容')).toBeTruthy();
});
const pastedImage = new File(['pasted-image'], 'pasted.png', {
type: 'image/png',
});
const input = screen.getByLabelText('发送给画布 Agent');
fireEvent.paste(input, {
clipboardData: {
files: [pastedImage],
},
});
await waitFor(() => {
expect(uploadEditorMediaAssetFileMock).toHaveBeenCalledWith(
pastedImage,
'image',
expect.objectContaining({
entityId: 'project-1',
metadata: { source: 'agent-input-paste' },
}),
);
});
expect(probeImageFileDimensionsMock).toHaveBeenCalledWith(pastedImage);
expect(createEditorProjectResourceMock).toHaveBeenCalledWith(
'project-1',
expect.objectContaining({
imageSrc: '/generated/pasted.png',
objectKey: 'generated-character-drafts/editor/agent-paste/image/pasted.png',
assetObjectId: 'asset-object-pasted',
width: 320,
height: 240,
sourceType: 'uploaded',
}),
);
await waitFor(() => {
expect(screen.getByText('粘贴图片')).toBeTruthy();
});
fireEvent.click(screen.getByRole('button', { name: '发送' }));
await waitFor(() => {
expect(client.streamMessage).toHaveBeenCalledWith(
'conversation-1',
expect.objectContaining({
text: '',
attachments: [
expect.objectContaining({
source: 'canvas_resource',
referenceId: 'resource-pasted',
imageSrc: '/generated/pasted.png',
width: 320,
height: 240,
}),
],
}),
expect.any(Object),
);
});
});
it('sends selected attachments even when the text input is empty', async () => {
const client = createClient();
render(
<EditorAgentConversationPanelView
projectId="project-1"
open
onToggleOpen={vi.fn()}
client={client}
@@ -292,7 +414,6 @@ describe('EditorAgentConversationPanelView', () => {
const { rerender } = render(
<EditorAgentConversationPanelView
projectId="project-1"
open
onToggleOpen={vi.fn()}
client={client}
@@ -316,7 +437,6 @@ describe('EditorAgentConversationPanelView', () => {
rerender(
<EditorAgentConversationPanelView
projectId="project-1"
open={false}
onToggleOpen={vi.fn()}
client={client}
@@ -326,7 +446,6 @@ describe('EditorAgentConversationPanelView', () => {
rerender(
<EditorAgentConversationPanelView
projectId="project-1"
open
onToggleOpen={vi.fn()}
client={client}
@@ -349,7 +468,6 @@ describe('EditorAgentConversationPanelView', () => {
render(
<div onWheel={parentWheel}>
<EditorAgentConversationPanelView
projectId="project-1"
open
onToggleOpen={vi.fn()}
client={client}
@@ -387,7 +505,6 @@ describe('EditorAgentConversationPanelView', () => {
const client = createClient();
render(
<EditorAgentConversationPanelView
projectId="project-1"
open
onToggleOpen={vi.fn()}
client={client}
@@ -11,6 +11,7 @@ import {
X,
} from 'lucide-react';
import {
type ClipboardEvent as ReactClipboardEvent,
type FormEvent,
type WheelEvent as ReactWheelEvent,
useEffect,
@@ -30,7 +31,11 @@ import { PlatformActionButton } from '../common/PlatformActionButton';
import { PlatformDangerConfirmDialog } from '../common/PlatformDangerConfirmDialog';
import { UnifiedModal } from '../common/UnifiedModal';
import { ResolvedAssetImage } from '../ResolvedAssetImage';
import { uploadEditorMediaAssetFile } from '../../services/image-editor/editorMediaAssetUploadClient';
import { createEditorProjectResource } from '../../services/image-editor/editorProjectClient';
import type { CanvasLayer, EditorAsset } from './ImageCanvasEditorTypes';
import { probeImageFileDimensions } from './ImageCanvasFileModel';
import { useImageCanvasContextStore } from './useImageCanvasContextStore.ts';
import {
type EditorAgentConversationClient,
useEditorAgentConversation,
@@ -45,7 +50,6 @@ type EditorAgentAttachmentOption = {
};
type EditorAgentConversationPanelViewProps = {
projectId?: string | null;
open: boolean;
onToggleOpen: () => void;
layers?: CanvasLayer[];
@@ -404,7 +408,6 @@ function AttachmentPickerModal({
}
export function EditorAgentConversationPanelView({
projectId,
open,
onToggleOpen,
layers = [],
@@ -418,6 +421,9 @@ export function EditorAgentConversationPanelView({
setHasConversationMounted(true);
}
}, [open]);
const projectId = useImageCanvasContextStore(
(state) => state.projectId,
);
const effectiveProjectId = hasConversationMounted ? projectId : null;
const {
conversations,
@@ -448,6 +454,7 @@ export function EditorAgentConversationPanelView({
const [attachmentPickerTab, setAttachmentPickerTab] =
useState<AttachmentPickerTab>('canvas');
const [attachmentError, setAttachmentError] = useState<string | null>(null);
const [isPastingAttachment, setIsPastingAttachment] = useState(false);
const [draftAttachmentKeys, setDraftAttachmentKeys] = useState<Set<string>>(
() => new Set(),
);
@@ -510,6 +517,9 @@ export function EditorAgentConversationPanelView({
stopCurrentTurn();
return;
}
if (isPastingAttachment) {
return;
}
const text = draftText.trim();
if (!text && !attachments.length) {
return;
@@ -524,6 +534,113 @@ export function EditorAgentConversationPanelView({
);
});
};
const appendAttachments = (nextAttachments: EditorAgentAttachmentRef[]) => {
function mergeAttachments(
currentAttachments: EditorAgentAttachmentRef[],
nextAttachments: EditorAgentAttachmentRef[],
) {
const merged = [...currentAttachments];
const existingKeys = new Set(currentAttachments.map(attachmentKey));
nextAttachments.forEach((attachment) => {
const key = attachmentKey(attachment);
if (!existingKeys.has(key)) {
existingKeys.add(key);
merged.push(attachment);
}
});
return merged;
}
const mergedAttachments = mergeAttachments(attachments, nextAttachments);
if (mergedAttachments.length > EDITOR_AGENT_MAX_ATTACHMENTS) {
setAttachmentError(`最多 ${EDITOR_AGENT_MAX_ATTACHMENTS}`);
return false;
}
setAttachments(mergedAttachments);
setAttachmentError(null);
return true;
};
const createPastedAgentImageAttachment = async (
file: File,
): Promise<EditorAgentAttachmentRef> => {
if (!projectId?.trim()) {
throw new Error('缺少画布项目');
}
const [upload, dimensions] = await Promise.all([
uploadEditorMediaAssetFile(file, 'image', {
pathSegments: ['editor', 'agent-paste', 'image', `${Date.now()}`],
entityId: projectId,
metadata: {
source: 'agent-input-paste',
},
}),
probeImageFileDimensions(file),
]);
const width = dimensions?.width ?? 1;
const height = dimensions?.height ?? 1;
const resource = await createEditorProjectResource(projectId, {
imageSrc: upload.src,
objectKey: upload.objectKey,
assetObjectId: upload.assetObjectId,
width,
height,
sourceType: 'uploaded',
});
return {
source: 'canvas_resource',
referenceId: resource.resourceId,
objectKey: resource.objectKey ?? upload.objectKey,
imageSrc: resource.imageSrc,
thumbnailSrc: null,
label: resource.label ?? '粘贴图片',
width: resource.width,
height: resource.height,
};
};
function extractClipboardImageFiles(
clipboardData: DataTransfer | null,
): File[] {
if (!clipboardData) {
return [];
}
const fileItems = Array.from(clipboardData.files ?? []).filter((file) =>
file.type.startsWith('image/'),
);
return [...fileItems];
}
const handleInputPaste = (event: ReactClipboardEvent<HTMLTextAreaElement>) => {
const imageFiles = extractClipboardImageFiles(event.clipboardData);
if (!imageFiles.length) {
return;
}
if (isPastingAttachment) {
return;
}
event.preventDefault();
if (attachments.length >= EDITOR_AGENT_MAX_ATTACHMENTS) {
setAttachmentError(`最多 ${EDITOR_AGENT_MAX_ATTACHMENTS}`);
return;
}
// TODO: deduplicate those existing assets
setIsPastingAttachment(true);
setAttachmentError('图片上传中');
void Promise.all(
imageFiles.map((file) => createPastedAgentImageAttachment(file)),
)
.then((pastedAttachments) => {
appendAttachments(pastedAttachments);
})
.catch(() => {
setAttachmentError('图片粘贴失败,请重试');
})
.finally(() => {
setIsPastingAttachment(false);
});
};
const removeAttachment = (targetAttachment: EditorAgentAttachmentRef) => {
const key = attachmentKey(targetAttachment);
@@ -654,6 +771,11 @@ export function EditorAgentConversationPanelView({
{errorMessage}
</div>
) : null}
{attachmentError ? (
<div className="mx-3 mb-2 rounded-2xl bg-amber-50 px-3 py-2 text-sm text-amber-700">
{attachmentError}
</div>
) : null}
<form
className="border-t border-slate-200 bg-white/95 p-3"
onSubmit={submitMessage}
@@ -684,6 +806,7 @@ export function EditorAgentConversationPanelView({
value={draftText}
rows={1}
onChange={(event) => setDraftText(event.currentTarget.value)}
onPaste={handleInputPaste}
onKeyDown={(event) => {
if (event.key === 'Enter' && !event.shiftKey) {
event.preventDefault();
@@ -93,7 +93,6 @@ function createSidebarProps(): ImageCanvasSidebarViewProps {
function createTopbarProps(): ImageCanvasTopbarViewProps {
return {
projectId: 'project-1',
projectTitle: '默认项目',
projectRenameValue: '默认项目',
isRenamingProject: false,
@@ -119,7 +118,6 @@ function createTopbarProps(): ImageCanvasTopbarViewProps {
function createStageProps(): ImageCanvasStageViewProps {
return {
projectId: 'project-1',
canvasViewportRef: createRef<HTMLDivElement>(),
specToolWrapRef: createRef<HTMLSpanElement>(),
musicToolWrapRef: createRef<HTMLSpanElement>(),
@@ -82,6 +82,7 @@ import { useImageCanvasEditorChrome } from './useImageCanvasEditorChrome';
import { useImageCanvasGenerationSurface } from './useImageCanvasGenerationSurface';
import { useImageCanvasKeyboardShortcuts } from './useImageCanvasKeyboardShortcuts';
import { useImageCanvasLayerCommands } from './useImageCanvasLayerCommands';
import { useImageCanvasContextStore } from './useImageCanvasContextStore.ts';
import { useImageCanvasProjectPersistence } from './useImageCanvasProjectPersistence';
import { useImageCanvasStageController } from './useImageCanvasStageController';
import { useImageCanvasStageInteractions } from './useImageCanvasStageInteractions';
@@ -1035,6 +1036,12 @@ export function ImageCanvasEditorView({
openEditorLoginModal,
onProjectAccessLost,
});
const setEditorProjectContextId = useImageCanvasContextStore(
(state) => state.setProjectId,
);
useEffect(() => {
setEditorProjectContextId(projectId);
}, [projectId, setEditorProjectContextId]);
const applyGeneratedProjectSnapshot = useCallback(
(project: EditorProjectSnapshot) => {
applyProjectSnapshot(project);
@@ -1906,7 +1913,6 @@ export function ImageCanvasEditorView({
getCanvasPointFromClient,
};
const topbarProps = {
projectId,
projectTitle,
projectRenameValue,
isRenamingProject,
@@ -1935,7 +1941,6 @@ export function ImageCanvasEditorView({
},
};
const stageProps = {
projectId,
canvasViewportRef,
specToolWrapRef,
musicToolWrapRef,
@@ -41,7 +41,6 @@ import { ImageCanvasUiAssetExtractionOverlayView } from './ImageCanvasUiAssetExt
import { ImageCanvasWorldView } from './ImageCanvasWorldView';
export type ImageCanvasStageViewProps = {
projectId?: string | null;
canvasViewportRef: RefObject<HTMLDivElement | null>;
specToolWrapRef: RefObject<HTMLSpanElement | null>;
musicToolWrapRef: RefObject<HTMLSpanElement | null>;
@@ -192,7 +191,6 @@ export type ImageCanvasStageViewProps = {
};
export function ImageCanvasStageView({
projectId,
canvasViewportRef,
specToolWrapRef,
musicToolWrapRef,
@@ -463,7 +461,6 @@ export function ImageCanvasStageView({
/>
<ImageCanvasTaskSidebarView
projectId={projectId}
refreshKey={taskListRefreshKey}
open={isTaskSidebarOpen}
onToggleOpen={onToggleTaskSidebar}
@@ -472,7 +469,6 @@ export function ImageCanvasStageView({
{isAgentConversationEnabled ? (
<EditorAgentConversationPanelView
projectId={projectId}
open={isAgentConversationOpen}
onToggleOpen={onToggleAgentConversation}
layers={layers}
@@ -6,6 +6,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
import type { ExternalGenerationTaskRecord } from '../../../packages/shared/src/contracts/externalGeneration';
import { listExternalGenerationTasks } from '../../services/external-generation';
import { ImageCanvasTaskSidebarView } from './ImageCanvasTaskSidebarView';
import { useImageCanvasContextStore } from './useImageCanvasContextStore.ts';
vi.mock('../../services/external-generation', () => ({
listExternalGenerationTasks: vi.fn().mockResolvedValue({
@@ -22,6 +23,7 @@ vi.mock('../../services/external-generation', () => ({
const listExternalGenerationTasksMock = vi.mocked(listExternalGenerationTasks);
beforeEach(() => {
useImageCanvasContextStore.getState().setProjectId('project-1');
listExternalGenerationTasksMock.mockClear();
listExternalGenerationTasksMock.mockResolvedValue({
overview: {
@@ -115,7 +117,6 @@ describe('ImageCanvasTaskSidebarView', () => {
render(
<div onWheel={parentWheel}>
<ImageCanvasTaskSidebarView
projectId="project-1"
open
onToggleOpen={vi.fn()}
onFocusExternalTask={focusExternalTask}
@@ -230,7 +231,6 @@ describe('ImageCanvasTaskSidebarView', () => {
render(
<ImageCanvasTaskSidebarView
projectId="project-1"
open
onToggleOpen={vi.fn()}
onFocusExternalTask={vi.fn()}
@@ -287,7 +287,6 @@ describe('ImageCanvasTaskSidebarView', () => {
render(
<ImageCanvasTaskSidebarView
projectId="project-1"
open
onToggleOpen={vi.fn()}
onFocusExternalTask={vi.fn()}
@@ -325,7 +324,6 @@ describe('ImageCanvasTaskSidebarView', () => {
const { rerender } = render(
<ImageCanvasTaskSidebarView
projectId="project-1"
refreshKey={0}
open
onToggleOpen={vi.fn()}
@@ -347,7 +345,6 @@ describe('ImageCanvasTaskSidebarView', () => {
];
rerender(
<ImageCanvasTaskSidebarView
projectId="project-1"
refreshKey={1}
open
onToggleOpen={vi.fn()}
@@ -388,7 +385,6 @@ describe('ImageCanvasTaskSidebarView', () => {
render(
<ImageCanvasTaskSidebarView
projectId="project-1"
open
onToggleOpen={vi.fn()}
onFocusExternalTask={vi.fn()}
@@ -19,6 +19,7 @@ import type { ExternalGenerationTaskRecord } from '../../../packages/shared/src/
import { listExternalGenerationTasks } from '../../services/external-generation';
import { EditorIconButton } from './ImageCanvasEditorPrimitives';
import type { CanvasTaskStatus } from './ImageCanvasEditorTypes';
import { useImageCanvasContextStore } from './useImageCanvasContextStore.ts';
const COMPLETED_TASK_LIST_LIMIT = 20;
const COMPLETED_TASK_LIST_MAX_LIMIT = 100;
@@ -42,7 +43,6 @@ type TaskSidebarItem = {
};
type ImageCanvasTaskSidebarViewProps = {
projectId?: string | null;
refreshKey?: number;
open: boolean;
onToggleOpen: () => void;
@@ -252,12 +252,12 @@ function trimStoredExternalTasks(
}
export function ImageCanvasTaskSidebarView({
projectId,
refreshKey = 0,
open,
onToggleOpen,
onFocusExternalTask,
}: ImageCanvasTaskSidebarViewProps) {
const projectId = useImageCanvasContextStore((state) => state.projectId);
const normalizedProjectId = projectId?.trim() ?? '';
const [activeTab, setActiveTab] = useState<TaskSidebarTab>('active');
const [now, setNow] = useState(() => Date.now());
@@ -5,6 +5,7 @@ import { describe, expect, it, vi } from 'vitest';
import type { CanvasLayer } from './ImageCanvasEditorTypes';
import { ImageCanvasTopbarView } from './ImageCanvasTopbarView';
import { useImageCanvasContextStore } from './useImageCanvasContextStore.ts';
function createLayer(overrides: Partial<CanvasLayer> = {}): CanvasLayer {
const id = overrides.id ?? 'layer-a';
@@ -30,8 +31,8 @@ function renderTopbar(
Parameters<typeof ImageCanvasTopbarView>[0]
> = {},
) {
useImageCanvasContextStore.getState().setProjectId('project-a');
const props: Parameters<typeof ImageCanvasTopbarView>[0] = {
projectId: 'project-a',
projectTitle: '默认项目',
projectRenameValue: '默认项目',
isRenamingProject: false,
@@ -144,7 +145,6 @@ describe('ImageCanvasTopbarView', () => {
const exportCanvasAssets = vi.fn();
const { rerender } = render(
<ImageCanvasTopbarView
projectId="project-a"
projectTitle="默认项目"
projectRenameValue="默认项目"
isRenamingProject={false}
@@ -175,7 +175,6 @@ describe('ImageCanvasTopbarView', () => {
rerender(
<ImageCanvasTopbarView
projectId="project-a"
projectTitle="默认项目"
projectRenameValue="默认项目"
isRenamingProject={false}
@@ -12,9 +12,9 @@ import { PlatformTextField } from '../common/PlatformTextField';
import { EditorIconButton } from './ImageCanvasEditorPrimitives';
import type { CanvasLayer } from './ImageCanvasEditorTypes';
import type { AssetExportStatus } from './useImageCanvasAssetExportWorkflow';
import { useImageCanvasContextStore } from './useImageCanvasContextStore.ts';
export type ImageCanvasTopbarViewProps = {
projectId: string | null;
projectTitle: string;
projectRenameValue: string;
isRenamingProject: boolean;
@@ -57,7 +57,6 @@ function buildCanvasUserCode(
}
export function ImageCanvasTopbarView({
projectId,
projectTitle,
projectRenameValue,
isRenamingProject,
@@ -79,6 +78,7 @@ export function ImageCanvasTopbarView({
onOpenWallet,
onOpenAccount,
}: ImageCanvasTopbarViewProps) {
const projectId = useImageCanvasContextStore((state) => state.projectId);
const hasExportableLayer = layers.some(
(layer) => layer.src.trim().length > 0,
);
@@ -0,0 +1,14 @@
import { create } from 'zustand';
type ImageCanvasContextState = {
projectId: string | null;
setProjectId: (projectId?: string | null) => void;
};
export const useImageCanvasContextStore =
create<ImageCanvasContextState>((set) => ({
projectId: null,
setProjectId: (projectId) => {
set({ projectId: projectId?.trim() || null });
},
}));