From 4ff482fcc1c80acb47f67eb850556c2b4febe184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Tue, 21 Jul 2026 13:41:28 +0800 Subject: [PATCH] editor agent: add host bridge support for exporting images and audio from right-click menu --- .../MessageBubble.test.tsx | 192 +++++++++++++++++- .../useRightClickMenu.ts | 178 ++++++++++++---- 2 files changed, 320 insertions(+), 50 deletions(-) diff --git a/src/components/image-editor/EditorAgentConversation/MessageBubble.test.tsx b/src/components/image-editor/EditorAgentConversation/MessageBubble.test.tsx index 2d9866437..328738e8b 100644 --- a/src/components/image-editor/EditorAgentConversation/MessageBubble.test.tsx +++ b/src/components/image-editor/EditorAgentConversation/MessageBubble.test.tsx @@ -1,11 +1,16 @@ /* @vitest-environment jsdom */ import { fireEvent, render, screen, waitFor } from '@testing-library/react'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { EditorAgentMessage } from '@/packages/shared/src/contracts'; import { readAssetBytes } from '@/src/services/assetReadUrlService.ts'; import { copyTextToClipboard } from '@/src/services/clipboard.ts'; +import { + resetHostRuntimeCacheForTest, + setHostRuntimeCacheForTest, +} from '@/src/services/host-bridge/hostBridge.ts'; +import { resetNativeAppHostBridgeForTest } from '@/src/services/host-bridge/nativeAppHostBridge.ts'; import { MessageBubble } from './MessageBubble.tsx'; @@ -41,6 +46,12 @@ describe('MessageBubble', () => { vi.mocked(copyTextToClipboard).mockResolvedValue(true); }); + afterEach(() => { + delete window.__TAURI__; + resetNativeAppHostBridgeForTest(); + resetHostRuntimeCacheForTest(); + }); + it('shows prefixed system errors as red Agent errors without the wire prefix', () => { renderMessage({ id: 2, @@ -96,9 +107,7 @@ describe('MessageBubble', () => { expect(copyTextToClipboard).toHaveBeenCalledWith('第一行\n第二行'), ); await waitFor(() => - expect( - screen.queryByRole('menu', { name: '消息右键菜单' }), - ).toBeNull(), + expect(screen.queryByRole('menu', { name: '消息右键菜单' })).toBeNull(), ); }); @@ -332,9 +341,9 @@ describe('MessageBubble', () => { const onReferenceImage = vi.fn(() => true); const { container } = renderMessage(message, onReferenceImage); await waitFor(() => - expect(container.querySelectorAll('.grid.grid-cols-3 > div')).toHaveLength( - 2, - ), + expect( + container.querySelectorAll('.grid.grid-cols-3 > div'), + ).toHaveLength(2), ); const imageCards = container.querySelectorAll('.grid.grid-cols-3 > div'); @@ -354,6 +363,171 @@ describe('MessageBubble', () => { expect(screen.getByRole('menuitem', { name: '下载图片' })).toBeTruthy(); }); + it('exports right-click images through the native HostBridge', async () => { + const invoke = vi.fn( + async (_command: string, args?: Record) => { + const request = (args as { request: { id: string } }).request; + return { + bridge: 'GenarrativeHostBridge', + version: 1, + id: request.id, + ok: true, + result: { + action: 'saved', + fileName: '参考图.png', + bytes: 5, + }, + }; + }, + ); + setHostRuntimeCacheForTest({ + shell: 'tauri_desktop', + platform: 'linux', + hostVersion: '0.1.0', + bridgeVersion: 1, + capabilities: ['file.exportImage'], + }); + window.__TAURI__ = { + core: { + invoke: async ( + command: string, + args?: Record, + ) => (await invoke(command, args)) as Result, + }, + }; + const anchorClick = vi.spyOn(HTMLAnchorElement.prototype, 'click'); + vi.mocked(readAssetBytes).mockResolvedValue({ + blob: vi + .fn() + .mockResolvedValue(new Blob(['image'], { type: 'image/png' })), + } as unknown as Response); + const { container } = renderMessage({ + id: 9, + role: 'user', + text: '保存参考图', + attachments: [ + { + source: 'canvas_resource', + referenceId: 'resource-download-image', + objectKey: 'editor/reference.png', + imageSrc: '/generated-editor-images/reference.png', + label: '参考图.webp', + }, + ], + toolCall: null, + createdAt: '2026-07-20T00:00:00Z', + }); + + fireEvent.contextMenu(container.querySelector('.group')!, { + clientX: 30, + clientY: 40, + }); + fireEvent.click(screen.getByRole('menuitem', { name: '下载图片' })); + + await waitFor(() => + expect(invoke).toHaveBeenCalledWith('host_bridge_request', { + request: expect.objectContaining({ + method: 'file.exportImage', + payload: { + fileName: '参考图.png', + base64Data: 'aW1hZ2U=', + mimeType: 'image/png', + }, + }), + }), + ); + expect(anchorClick).not.toHaveBeenCalled(); + expect(screen.queryByRole('menu', { name: '消息素材右键菜单' })).toBeNull(); + }); + + it('exports right-click audio through the native HostBridge', async () => { + const invoke = vi.fn( + async (_command: string, args?: Record) => { + const request = (args as { request: { id: string } }).request; + return { + bridge: 'GenarrativeHostBridge', + version: 1, + id: request.id, + ok: true, + result: { + action: 'saved', + fileName: 'Agent生成音频-1.wav', + bytes: 5, + }, + }; + }, + ); + setHostRuntimeCacheForTest({ + shell: 'tauri_desktop', + platform: 'linux', + hostVersion: '0.1.0', + bridgeVersion: 1, + capabilities: ['file.exportAudio'], + }); + window.__TAURI__ = { + core: { + invoke: async ( + command: string, + args?: Record, + ) => (await invoke(command, args)) as Result, + }, + }; + const anchorClick = vi.spyOn(HTMLAnchorElement.prototype, 'click'); + vi.mocked(readAssetBytes).mockResolvedValue({ + blob: vi + .fn() + .mockResolvedValue(new Blob(['audio'], { type: 'audio/wav' })), + } as unknown as Response); + const { container } = renderMessage({ + id: 10, + role: 'system', + text: 'internal tool result', + attachments: [], + toolCall: { + toolName: 'generate_sound_effect', + status: 'completed', + args: {}, + displayArgs: { + stringArgs: [], + imageArgs: [], + extras: { priceMudPoints: 1 }, + }, + images: [], + videos: [], + audios: [ + { + resourceId: 'audio-1', + objectKey: 'editor/audio.wav', + audioSrc: '/generated-editor-audios/audio.wav', + }, + ], + }, + createdAt: '2026-07-20T00:00:00Z', + }); + await waitFor(() => expect(container.querySelector('audio')).toBeTruthy()); + + fireEvent.contextMenu(container.querySelector('audio')!.parentElement!, { + clientX: 30, + clientY: 40, + }); + fireEvent.click(screen.getByRole('menuitem', { name: '下载音频' })); + + await waitFor(() => + expect(invoke).toHaveBeenCalledWith('host_bridge_request', { + request: expect.objectContaining({ + method: 'file.exportAudio', + payload: { + fileName: 'Agent生成音频-1.wav', + base64Data: 'YXVkaW8=', + mimeType: 'audio/wav', + }, + }), + }), + ); + expect(anchorClick).not.toHaveBeenCalled(); + expect(screen.queryByRole('menu', { name: '消息素材右键菜单' })).toBeNull(); + }); + it('downloads generated video with the inferred extension', async () => { const createObjectURL = vi.fn(() => 'blob:agent-video'); const revokeObjectURL = vi.fn(); @@ -410,9 +584,7 @@ describe('MessageBubble', () => { fireEvent.click(screen.getByRole('menuitem', { name: '下载视频' })); await waitFor(() => expect(anchorClick).toHaveBeenCalledTimes(1)); - expect( - screen.queryByRole('menu', { name: '消息素材右键菜单' }), - ).toBeNull(); + expect(screen.queryByRole('menu', { name: '消息素材右键菜单' })).toBeNull(); expect(createObjectURL).toHaveBeenCalledWith(expect.any(Blob)); expect(revokeObjectURL).toHaveBeenCalledWith('blob:agent-video'); }); diff --git a/src/components/image-editor/EditorAgentConversation/useRightClickMenu.ts b/src/components/image-editor/EditorAgentConversation/useRightClickMenu.ts index 5d29b1230..fddbb6564 100644 --- a/src/components/image-editor/EditorAgentConversation/useRightClickMenu.ts +++ b/src/components/image-editor/EditorAgentConversation/useRightClickMenu.ts @@ -6,6 +6,16 @@ import { import { readAssetBytes } from '@/src/services/assetReadUrlService.ts'; import { copyTextToClipboard } from '@/src/services/clipboard.ts'; +import { + type HostFileExportAudioRequest, + type HostFileExportImageRequest, + canUseNativeHostCapability, + exportHostAudioFile, + exportHostImageFile, + isNativeAppRuntime, +} from '@/src/services/host-bridge/hostBridge.ts'; + +import { getLayerAssetExtensionFromTypeOrSrc } from '../ImageCanvasExportModel.ts'; import { contextAssetMediaSrc, @@ -23,34 +33,97 @@ type RightClickMenuState = { result: 'success' | 'error' | null; }; -const MIME_FILE_EXTENSIONS: Record = { - 'image/png': 'png', - 'image/jpeg': 'jpg', - 'image/webp': 'webp', - 'image/gif': 'gif', - 'video/mp4': 'mp4', - 'video/webm': 'webm', - 'audio/mpeg': 'mp3', - 'audio/mp4': 'm4a', - 'audio/ogg': 'ogg', - 'audio/wav': 'wav', -}; - -function sanitizeDownloadName(value: string, mimeType: string) { +function sanitizeDownloadName(value: string, extension: string) { const normalized = value .trim() .replace(/[<>:"/\\|?*]/gu, '-') .replace(/\p{Cc}/gu, '-') .replace(/[. ]+$/u, '') .slice(0, 100); - const baseName = normalized || 'Agent素材'; - if (/\.[a-z0-9]{2,5}$/iu.test(baseName)) { - return baseName; - } - const extension = MIME_FILE_EXTENSIONS[mimeType.toLowerCase()] ?? ''; + const baseName = (normalized || 'Agent素材').replace( + /\.[a-z0-9]{1,8}$/iu, + '', + ); return extension ? `${baseName}.${extension}` : baseName; } +function blobToBase64Data(blob: Blob) { + return new Promise((resolve, reject) => { + if (typeof FileReader === 'undefined') { + reject(new Error('当前环境不支持文件编码')); + return; + } + const reader = new FileReader(); + reader.onerror = () => reject(new Error('文件编码失败')); + reader.onload = () => { + const result = typeof reader.result === 'string' ? reader.result : ''; + const base64Data = result.split(',')[1] ?? ''; + if (base64Data) { + resolve(base64Data); + } else { + reject(new Error('文件编码失败')); + } + }; + reader.readAsDataURL(blob); + }); +} + +function hostImageMimeTypeFromExtension( + extension: string, +): HostFileExportImageRequest['mimeType'] | null { + if (extension === 'png') { + return 'image/png'; + } + if (extension === 'jpg' || extension === 'jpeg') { + return 'image/jpeg'; + } + if (extension === 'webp') { + return 'image/webp'; + } + return null; +} + +function hostAudioMimeTypeFromExtension( + extension: string, +): HostFileExportAudioRequest['mimeType'] | null { + if (extension === 'mp3') { + return 'audio/mpeg'; + } + if (extension === 'm4a' || extension === 'mp4') { + return 'audio/mp4'; + } + if (extension === 'wav') { + return 'audio/wav'; + } + if (extension === 'ogg') { + return 'audio/ogg'; + } + if (extension === 'webm') { + return 'audio/webm'; + } + return null; +} + +function triggerBrowserDownload(blob: Blob, fileName: string) { + if ( + typeof document === 'undefined' || + typeof URL.createObjectURL !== 'function' || + typeof URL.revokeObjectURL !== 'function' + ) { + return false; + } + const downloadUrl = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = downloadUrl; + link.download = fileName; + link.style.display = 'none'; + document.body.appendChild(link); + link.click(); + link.remove(); + window.setTimeout(() => URL.revokeObjectURL(downloadUrl), 0); + return true; +} + async function convertImageBlobToPng(blob: Blob) { if (blob.type.toLowerCase() === 'image/png') { return blob; @@ -109,27 +182,52 @@ async function copyAssetImage(asset: EditorAgentContextAsset) { } async function downloadAsset(asset: EditorAgentContextAsset) { - if ( - typeof document === 'undefined' || - typeof URL.createObjectURL !== 'function' - ) { - return false; - } try { - const response = await readAssetBytes(contextAssetMediaSrc(asset), { + const source = contextAssetMediaSrc(asset); + const response = await readAssetBytes(source, { objectKey: asset.objectKey, }); const blob = await response.blob(); - const downloadUrl = URL.createObjectURL(blob); - const link = document.createElement('a'); - link.href = downloadUrl; - link.download = sanitizeDownloadName(asset.suggestedFileName, blob.type); - link.style.display = 'none'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - URL.revokeObjectURL(downloadUrl); - return true; + const extension = getLayerAssetExtensionFromTypeOrSrc( + asset.mediaType, + // application/octet-stream is impossible by contract + blob.type, + asset.objectKey ?? source, + ); + const fileName = sanitizeDownloadName(asset.suggestedFileName, extension); + + if (!isNativeAppRuntime()) { + return triggerBrowserDownload(blob, fileName); + } + + if ( + asset.mediaType === 'image' && + canUseNativeHostCapability('file.exportImage') + ) { + const mimeType = hostImageMimeTypeFromExtension(extension); + if (!mimeType) { + return false; + } + const base64Data = await blobToBase64Data(blob); + return Boolean( + await exportHostImageFile({ fileName, base64Data, mimeType }), + ); + } + if ( + asset.mediaType === 'audio' && + canUseNativeHostCapability('file.exportAudio') + ) { + const mimeType = hostAudioMimeTypeFromExtension(extension); + if (!mimeType) { + return false; + } + const base64Data = await blobToBase64Data(blob); + return Boolean( + await exportHostAudioFile({ fileName, base64Data, mimeType }), + ); + } + // video is not yet supported to export on native shell + return false; } catch { return false; } @@ -192,10 +290,10 @@ export function useRightClickMenu({ target.asset.mediaType === 'image' && contextAssetMediaSrc(target.asset).trim() ? (onReferenceImage?.(target.asset) ?? false) - : action === EditorAgentRightClickAction.DownloadAsset && - target.kind === 'asset' - ? await downloadAsset(target.asset) - : false; + : action === EditorAgentRightClickAction.DownloadAsset && + target.kind === 'asset' + ? await downloadAsset(target.asset) + : false; setRightClickMenu((current) => current?.target === target && current.pendingAction === action ? succeeded