GenarrativeAI/Genarrative
12
0
Fork 0
Code Issues 27 Pull Requests 6 Actions Packages Projects 1 Releases Wiki Activity

接入原生壳音频文件导入能力

Browse Source 
新增 file.importAudio HostBridge 契约和 H5 facade

移动端通过 Expo DocumentPicker 读取受控音频文件

桌面端通过 Tauri 文件选择框读取受控音频文件

通用音频输入面板在原生壳复用宿主导入和现有音频处理管线

更新壳能力检查、测试、方案文档和共享决策记录
This commit is contained in:
段舒康
2026-06-18 05:25:39 +08:00
parent 51dcff6d16
commit 3be997e286
15 changed files with 627 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/mobile-shell/src/mobileHostBridge.test.ts
+93
View File
@@ -50,6 +50,7 @@ vi.mock('expo-clipboard', () => ({
}));
const fileTexts = vi.hoisted(() => new Map<string, string>());
const fileBase64Data = vi.hoisted(() => new Map<string, string>());
const writtenFiles = vi.hoisted(
() =>
@@ -83,6 +84,10 @@ vi.mock('expo-file-system', () => ({
text() {
return Promise.resolve(fileTexts.get(this.uri) ?? '');
}
base64() {
return Promise.resolve(fileBase64Data.get(this.uri) ?? '');
}
},
}));
@@ -260,6 +265,7 @@ afterEach(() => {
vi.mocked(Sharing.shareAsync).mockReset();
vi.mocked(Share.share).mockReset();
fileTexts.clear();
fileBase64Data.clear();
writtenFiles.length = 0;
resetMobileHostBridgeForTest();
});
@@ -286,6 +292,9 @@ describe('handleMobileHostBridgeMessage', () => {
expect(
(okResponse.result as { capabilities: string[] }).capabilities,
).toContain('file.importImage');
expect(
(okResponse.result as { capabilities: string[] }).capabilities,
).toContain('file.importAudio');
expect(
(okResponse.result as { capabilities: string[] }).capabilities,
).toEqual(
@@ -954,4 +963,88 @@ describe('handleMobileHostBridgeMessage', () => {
expect(expectFailed(oversized).error.code).toBe('invalid_request');
});
test('file.importAudio 调起系统文档选择器并返回受控音频数据', async () => {
fileBase64Data.set('file:///private/mobile/hit.webm', 'YXVkaW8=');
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: false,
assets: [
{
uri: 'file:///private/mobile/hit.webm',
name: ' ../敲击:音效?.webm ',
mimeType: 'audio/webm',
size: 5,
lastModified: 1,
},
],
});
const response = await send(request('file.importAudio'));
expect(expectOk(response).result).toEqual({
action: 'selected',
fileName: '敲击-音效-.webm',
base64Data: 'YXVkaW8=',
mimeType: 'audio/webm',
bytes: 5,
});
expect(DocumentPicker.getDocumentAsync).toHaveBeenCalledWith({
copyToCacheDirectory: true,
multiple: false,
type: [
'audio/mpeg',
'audio/mp4',
'audio/wav',
'audio/ogg',
'audio/webm',
],
});
});
test('file.importAudio 取消选择并拒绝非法 MIME 与超限音频', async () => {
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: true,
assets: null,
});
const cancelled = await send(request('file.importAudio'));
expect(expectFailed(cancelled).error.code).toBe('cancelled');
fileBase64Data.set('file:///private/mobile/hit.txt', 'YXVkaW8=');
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: false,
assets: [
{
uri: 'file:///private/mobile/hit.txt',
name: 'hit.txt',
mimeType: 'text/plain',
size: 5,
lastModified: 1,
},
],
});
const unsupportedMime = await send(request('file.importAudio'));
expect(expectFailed(unsupportedMime).error.code).toBe('invalid_request');
fileBase64Data.set('file:///private/mobile/hit.webm', 'YXVkaW8=');
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: false,
assets: [
{
uri: 'file:///private/mobile/hit.webm',
name: 'hit.webm',
mimeType: 'audio/webm',
size: 20 * 1024 * 1024 + 1,
lastModified: 1,
},
],
});
const oversized = await send(request('file.importAudio'));
expect(expectFailed(oversized).error.code).toBe('invalid_request');
});
});
apps/mobile-shell/src/mobileHostBridge.ts
+98
View File
@@ -20,11 +20,13 @@ import {
type FileExportImageResult,
type FileExportTextPayload,
type FileExportTextResult,
type FileImportAudioResult,
type FileImportImageResult,
type FileImportTextResult,
type HapticsImpactPayload,
HOST_BRIDGE_PROTOCOL,
HOST_BRIDGE_VERSION,
type HostBridgeAudioMimeType,
type HostBridgeCapability,
type HostBridgeError,
type HostBridgeImageMimeType,
@@ -51,6 +53,7 @@ const EXPORT_TEXT_MAX_BYTES = 5 * 1024 * 1024;
const EXPORT_IMAGE_MAX_BYTES = 5 * 1024 * 1024;
const IMPORT_TEXT_MAX_BYTES = 5 * 1024 * 1024;
const IMPORT_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
const IMPORT_AUDIO_MAX_BYTES = 20 * 1024 * 1024;
const LOCAL_NOTIFICATION_CHANNEL_ID = 'genarrative-local';
const HOST_BRIDGE_TEXT_MIME_TYPES = new Set<HostBridgeTextMimeType>([
'text/plain',
@@ -63,6 +66,13 @@ const HOST_BRIDGE_IMAGE_MIME_TYPES = new Set<HostBridgeImageMimeType>([
'image/jpeg',
'image/webp',
]);
const HOST_BRIDGE_AUDIO_MIME_TYPES = new Set<HostBridgeAudioMimeType>([
'audio/mpeg',
'audio/mp4',
'audio/wav',
'audio/ogg',
'audio/webm',
]);
Notifications.setNotificationHandler({
handleNotification: async () => ({
@@ -92,6 +102,7 @@ export const MOBILE_HOST_CAPABILITIES: HostBridgeCapability[] = [
'file.importText',
'file.exportImage',
'file.importImage',
'file.importAudio',
'haptics.impact',
'notification.showLocal',
];
@@ -438,6 +449,37 @@ function fallbackImportedImageFileName(mimeType: HostBridgeImageMimeType) {
return 'genarrative-import.png';
}
function normalizeImportedAudioMimeType(
value: unknown,
fileName: string,
): HostBridgeAudioMimeType | null {
if (typeof value === 'string') {
const mimeType = value.toLowerCase();
if (HOST_BRIDGE_AUDIO_MIME_TYPES.has(mimeType as HostBridgeAudioMimeType)) {
return mimeType as HostBridgeAudioMimeType;
}
}
const normalizedName = fileName.toLowerCase();
if (normalizedName.endsWith('.mp3')) {
return 'audio/mpeg';
}
if (normalizedName.endsWith('.m4a') || normalizedName.endsWith('.mp4')) {
return 'audio/mp4';
}
if (normalizedName.endsWith('.wav')) {
return 'audio/wav';
}
if (normalizedName.endsWith('.ogg')) {
return 'audio/ogg';
}
if (normalizedName.endsWith('.webm')) {
return 'audio/webm';
}
return null;
}
async function importImageFile(): Promise<FileImportImageResult> {
const permission = await ImagePicker.requestMediaLibraryPermissionsAsync();
if (permission.status !== ImagePicker.PermissionStatus.GRANTED) {
@@ -499,6 +541,60 @@ async function importImageFile(): Promise<FileImportImageResult> {
};
}
async function importAudioFile(): Promise<FileImportAudioResult> {
const result = await DocumentPicker.getDocumentAsync({
copyToCacheDirectory: true,
multiple: false,
type: [
'audio/mpeg',
'audio/mp4',
'audio/wav',
'audio/ogg',
'audio/webm',
],
});
if (result.canceled) {
throw {
code: 'cancelled',
message: 'file import cancelled',
} satisfies HostBridgeError;
}
const asset = result.assets[0];
if (!asset?.uri) {
throw invalidRequest('audio file is required');
}
const fileName = normalizeHostBridgeExportFileName(
asset.name || 'genarrative-import-audio.webm',
);
const mimeType = normalizeImportedAudioMimeType(asset.mimeType, fileName);
if (!mimeType) {
throw invalidRequest('mimeType must be an allowed audio type');
}
if (
typeof asset.size === 'number' &&
(asset.size <= 0 || asset.size > IMPORT_AUDIO_MAX_BYTES)
) {
throw invalidRequest('audio exceeds file import size limit');
}
const file = new File(asset.uri);
const base64Data = await file.base64();
const bytes = base64DecodedByteLength(base64Data);
if (bytes <= 0 || bytes > IMPORT_AUDIO_MAX_BYTES) {
throw invalidRequest('audio exceeds file import size limit');
}
return {
action: 'selected',
fileName,
base64Data,
mimeType,
bytes,
};
}
async function runHaptics(payload: unknown) {
const style = (payload as HapticsImpactPayload | undefined)?.style;
const impactStyle =
@@ -733,6 +829,8 @@ async function handleRequest(request: HostBridgeRequest) {
return ok(request, await exportImageFile(request.payload));
case 'file.importImage':
return ok(request, await importImageFile());
case 'file.importAudio':
return ok(request, await importAudioFile());
case 'haptics.impact':
return ok(request, await runHaptics(request.payload));
case 'notification.showLocal':