GenarrativeAI/Genarrative
11
0
Fork 0
Code Issues 20 Pull Requests 5 Actions 2 Packages Projects 3 Releases Wiki Activity

editor agent: add host bridge support for exporting images and audio from right-click menu

Browse Source
This commit is contained in:
王德宇
2026-07-21 13:41:28 +08:00
parent 3bea67d01a
commit 4ff482fcc1
2 changed files with 320 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/image-editor/EditorAgentConversation/MessageBubble.test.tsx
+182 -10
View File
@@ -1,11 +1,16 @@
/* @vitest-environment jsdom */
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import type { EditorAgentMessage } from '@/packages/shared/src/contracts';
import { readAssetBytes } from '@/src/services/assetReadUrlService.ts';
import { copyTextToClipboard } from '@/src/services/clipboard.ts';
import {
resetHostRuntimeCacheForTest,
setHostRuntimeCacheForTest,
} from '@/src/services/host-bridge/hostBridge.ts';
import { resetNativeAppHostBridgeForTest } from '@/src/services/host-bridge/nativeAppHostBridge.ts';
import { MessageBubble } from './MessageBubble.tsx';
@@ -41,6 +46,12 @@ describe('MessageBubble', () => {
vi.mocked(copyTextToClipboard).mockResolvedValue(true);
});
afterEach(() => {
delete window.__TAURI__;
resetNativeAppHostBridgeForTest();
resetHostRuntimeCacheForTest();
});
it('shows prefixed system errors as red Agent errors without the wire prefix', () => {
renderMessage({
id: 2,
@@ -96,9 +107,7 @@ describe('MessageBubble', () => {
expect(copyTextToClipboard).toHaveBeenCalledWith('第一行\n第二行'),
);
await waitFor(() =>
expect(
screen.queryByRole('menu', { name: '消息右键菜单' }),
).toBeNull(),
expect(screen.queryByRole('menu', { name: '消息右键菜单' })).toBeNull(),
);
});
@@ -332,9 +341,9 @@ describe('MessageBubble', () => {
const onReferenceImage = vi.fn(() => true);
const { container } = renderMessage(message, onReferenceImage);
await waitFor(() =>
expect(container.querySelectorAll('.grid.grid-cols-3 > div')).toHaveLength(
2,
),
expect(
container.querySelectorAll('.grid.grid-cols-3 > div'),
).toHaveLength(2),
);
const imageCards = container.querySelectorAll('.grid.grid-cols-3 > div');
@@ -354,6 +363,171 @@ describe('MessageBubble', () => {
expect(screen.getByRole('menuitem', { name: '下载图片' })).toBeTruthy();
});
it('exports right-click images through the native HostBridge', async () => {
const invoke = vi.fn(
async (_command: string, args?: Record<string, unknown>) => {
const request = (args as { request: { id: string } }).request;
return {
bridge: 'GenarrativeHostBridge',
version: 1,
id: request.id,
ok: true,
result: {
action: 'saved',
fileName: '参考图.png',
bytes: 5,
},
};
},
);
setHostRuntimeCacheForTest({
shell: 'tauri_desktop',
platform: 'linux',
hostVersion: '0.1.0',
bridgeVersion: 1,
capabilities: ['file.exportImage'],
});
window.__TAURI__ = {
core: {
invoke: async <Result,>(
command: string,
args?: Record<string, unknown>,
) => (await invoke(command, args)) as Result,
},
};
const anchorClick = vi.spyOn(HTMLAnchorElement.prototype, 'click');
vi.mocked(readAssetBytes).mockResolvedValue({
blob: vi
.fn()
.mockResolvedValue(new Blob(['image'], { type: 'image/png' })),
} as unknown as Response);
const { container } = renderMessage({
id: 9,
role: 'user',
text: '保存参考图',
attachments: [
{
source: 'canvas_resource',
referenceId: 'resource-download-image',
objectKey: 'editor/reference.png',
imageSrc: '/generated-editor-images/reference.png',
label: '参考图.webp',
},
],
toolCall: null,
createdAt: '2026-07-20T00:00:00Z',
});
fireEvent.contextMenu(container.querySelector('.group')!, {
clientX: 30,
clientY: 40,
});
fireEvent.click(screen.getByRole('menuitem', { name: '下载图片' }));
await waitFor(() =>
expect(invoke).toHaveBeenCalledWith('host_bridge_request', {
request: expect.objectContaining({
method: 'file.exportImage',
payload: {
fileName: '参考图.png',
base64Data: 'aW1hZ2U=',
mimeType: 'image/png',
},
}),
}),
);
expect(anchorClick).not.toHaveBeenCalled();
expect(screen.queryByRole('menu', { name: '消息素材右键菜单' })).toBeNull();
});
it('exports right-click audio through the native HostBridge', async () => {
const invoke = vi.fn(
async (_command: string, args?: Record<string, unknown>) => {
const request = (args as { request: { id: string } }).request;
return {
bridge: 'GenarrativeHostBridge',
version: 1,
id: request.id,
ok: true,
result: {
action: 'saved',
fileName: 'Agent生成音频-1.wav',
bytes: 5,
},
};
},
);
setHostRuntimeCacheForTest({
shell: 'tauri_desktop',
platform: 'linux',
hostVersion: '0.1.0',
bridgeVersion: 1,
capabilities: ['file.exportAudio'],
});
window.__TAURI__ = {
core: {
invoke: async <Result,>(
command: string,
args?: Record<string, unknown>,
) => (await invoke(command, args)) as Result,
},
};
const anchorClick = vi.spyOn(HTMLAnchorElement.prototype, 'click');
vi.mocked(readAssetBytes).mockResolvedValue({
blob: vi
.fn()
.mockResolvedValue(new Blob(['audio'], { type: 'audio/wav' })),
} as unknown as Response);
const { container } = renderMessage({
id: 10,
role: 'system',
text: 'internal tool result',
attachments: [],
toolCall: {
toolName: 'generate_sound_effect',
status: 'completed',
args: {},
displayArgs: {
stringArgs: [],
imageArgs: [],
extras: { priceMudPoints: 1 },
},
images: [],
videos: [],
audios: [
{
resourceId: 'audio-1',
objectKey: 'editor/audio.wav',
audioSrc: '/generated-editor-audios/audio.wav',
},
],
},
createdAt: '2026-07-20T00:00:00Z',
});
await waitFor(() => expect(container.querySelector('audio')).toBeTruthy());
fireEvent.contextMenu(container.querySelector('audio')!.parentElement!, {
clientX: 30,
clientY: 40,
});
fireEvent.click(screen.getByRole('menuitem', { name: '下载音频' }));
await waitFor(() =>
expect(invoke).toHaveBeenCalledWith('host_bridge_request', {
request: expect.objectContaining({
method: 'file.exportAudio',
payload: {
fileName: 'Agent生成音频-1.wav',
base64Data: 'YXVkaW8=',
mimeType: 'audio/wav',
},
}),
}),
);
expect(anchorClick).not.toHaveBeenCalled();
expect(screen.queryByRole('menu', { name: '消息素材右键菜单' })).toBeNull();
});
it('downloads generated video with the inferred extension', async () => {
const createObjectURL = vi.fn(() => 'blob:agent-video');
const revokeObjectURL = vi.fn();
@@ -410,9 +584,7 @@ describe('MessageBubble', () => {
fireEvent.click(screen.getByRole('menuitem', { name: '下载视频' }));
await waitFor(() => expect(anchorClick).toHaveBeenCalledTimes(1));
expect(
screen.queryByRole('menu', { name: '消息素材右键菜单' }),
).toBeNull();
expect(screen.queryByRole('menu', { name: '消息素材右键菜单' })).toBeNull();
expect(createObjectURL).toHaveBeenCalledWith(expect.any(Blob));
expect(revokeObjectURL).toHaveBeenCalledWith('blob:agent-video');
});
src/components/image-editor/EditorAgentConversation/useRightClickMenu.ts
+138 -40
View File
@@ -6,6 +6,16 @@ import {
import { readAssetBytes } from '@/src/services/assetReadUrlService.ts';
import { copyTextToClipboard } from '@/src/services/clipboard.ts';
import {
type HostFileExportAudioRequest,
type HostFileExportImageRequest,
canUseNativeHostCapability,
exportHostAudioFile,
exportHostImageFile,
isNativeAppRuntime,
} from '@/src/services/host-bridge/hostBridge.ts';
import { getLayerAssetExtensionFromTypeOrSrc } from '../ImageCanvasExportModel.ts';
import {
contextAssetMediaSrc,
@@ -23,34 +33,97 @@ type RightClickMenuState = {
result: 'success' | 'error' | null;
};
const MIME_FILE_EXTENSIONS: Record<string, string> = {
'image/png': 'png',
'image/jpeg': 'jpg',
'image/webp': 'webp',
'image/gif': 'gif',
'video/mp4': 'mp4',
'video/webm': 'webm',
'audio/mpeg': 'mp3',
'audio/mp4': 'm4a',
'audio/ogg': 'ogg',
'audio/wav': 'wav',
};
function sanitizeDownloadName(value: string, mimeType: string) {
function sanitizeDownloadName(value: string, extension: string) {
const normalized = value
.trim()
.replace(/[<>:"/\\|?*]/gu, '-')
.replace(/\p{Cc}/gu, '-')
.replace(/[. ]+$/u, '')
.slice(0, 100);
const baseName = normalized || 'Agent素材';
if (/\.[a-z0-9]{2,5}$/iu.test(baseName)) {
return baseName;
}
const extension = MIME_FILE_EXTENSIONS[mimeType.toLowerCase()] ?? '';
const baseName = (normalized || 'Agent素材').replace(
/\.[a-z0-9]{1,8}$/iu,
'',
);
return extension ? `${baseName}.${extension}` : baseName;
}
function blobToBase64Data(blob: Blob) {
return new Promise<string>((resolve, reject) => {
if (typeof FileReader === 'undefined') {
reject(new Error('当前环境不支持文件编码'));
return;
}
const reader = new FileReader();
reader.onerror = () => reject(new Error('文件编码失败'));
reader.onload = () => {
const result = typeof reader.result === 'string' ? reader.result : '';
const base64Data = result.split(',')[1] ?? '';
if (base64Data) {
resolve(base64Data);
} else {
reject(new Error('文件编码失败'));
}
};
reader.readAsDataURL(blob);
});
}
function hostImageMimeTypeFromExtension(
extension: string,
): HostFileExportImageRequest['mimeType'] | null {
if (extension === 'png') {
return 'image/png';
}
if (extension === 'jpg' || extension === 'jpeg') {
return 'image/jpeg';
}
if (extension === 'webp') {
return 'image/webp';
}
return null;
}
function hostAudioMimeTypeFromExtension(
extension: string,
): HostFileExportAudioRequest['mimeType'] | null {
if (extension === 'mp3') {
return 'audio/mpeg';
}
if (extension === 'm4a' || extension === 'mp4') {
return 'audio/mp4';
}
if (extension === 'wav') {
return 'audio/wav';
}
if (extension === 'ogg') {
return 'audio/ogg';
}
if (extension === 'webm') {
return 'audio/webm';
}
return null;
}
function triggerBrowserDownload(blob: Blob, fileName: string) {
if (
typeof document === 'undefined' ||
typeof URL.createObjectURL !== 'function' ||
typeof URL.revokeObjectURL !== 'function'
) {
return false;
}
const downloadUrl = URL.createObjectURL(blob);
const link = document.createElement('a');
link.href = downloadUrl;
link.download = fileName;
link.style.display = 'none';
document.body.appendChild(link);
link.click();
link.remove();
window.setTimeout(() => URL.revokeObjectURL(downloadUrl), 0);
return true;
}
async function convertImageBlobToPng(blob: Blob) {
if (blob.type.toLowerCase() === 'image/png') {
return blob;
@@ -109,27 +182,52 @@ async function copyAssetImage(asset: EditorAgentContextAsset) {
}
async function downloadAsset(asset: EditorAgentContextAsset) {
if (
typeof document === 'undefined' ||
typeof URL.createObjectURL !== 'function'
) {
return false;
}
try {
const response = await readAssetBytes(contextAssetMediaSrc(asset), {
const source = contextAssetMediaSrc(asset);
const response = await readAssetBytes(source, {
objectKey: asset.objectKey,
});
const blob = await response.blob();
const downloadUrl = URL.createObjectURL(blob);
const link = document.createElement('a');
link.href = downloadUrl;
link.download = sanitizeDownloadName(asset.suggestedFileName, blob.type);
link.style.display = 'none';
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
URL.revokeObjectURL(downloadUrl);
return true;
const extension = getLayerAssetExtensionFromTypeOrSrc(
asset.mediaType,
// application/octet-stream is impossible by contract
blob.type,
asset.objectKey ?? source,
);
const fileName = sanitizeDownloadName(asset.suggestedFileName, extension);
if (!isNativeAppRuntime()) {
return triggerBrowserDownload(blob, fileName);
}
if (
asset.mediaType === 'image' &&
canUseNativeHostCapability('file.exportImage')
) {
const mimeType = hostImageMimeTypeFromExtension(extension);
if (!mimeType) {
return false;
}
const base64Data = await blobToBase64Data(blob);
return Boolean(
await exportHostImageFile({ fileName, base64Data, mimeType }),
);
}
if (
asset.mediaType === 'audio' &&
canUseNativeHostCapability('file.exportAudio')
) {
const mimeType = hostAudioMimeTypeFromExtension(extension);
if (!mimeType) {
return false;
}
const base64Data = await blobToBase64Data(blob);
return Boolean(
await exportHostAudioFile({ fileName, base64Data, mimeType }),
);
}
// video is not yet supported to export on native shell
return false;
} catch {
return false;
}
@@ -192,10 +290,10 @@ export function useRightClickMenu({
target.asset.mediaType === 'image' &&
contextAssetMediaSrc(target.asset).trim()
? (onReferenceImage?.(target.asset) ?? false)
: action === EditorAgentRightClickAction.DownloadAsset &&
target.kind === 'asset'
? await downloadAsset(target.asset)
: false;
: action === EditorAgentRightClickAction.DownloadAsset &&
target.kind === 'asset'
? await downloadAsset(target.asset)
: false;
setRightClickMenu((current) =>
current?.target === target && current.pendingAction === action
? succeeded