GenarrativeAI/Genarrative
11
0
Fork 0
Code Issues 35 Pull Requests 12 Actions Packages Projects 1 Releases Wiki Activity

接入原生壳音频文件导入能力

Browse Source 
新增 file.importAudio HostBridge 契约和 H5 facade

移动端通过 Expo DocumentPicker 读取受控音频文件

桌面端通过 Tauri 文件选择框读取受控音频文件

通用音频输入面板在原生壳复用宿主导入和现有音频处理管线

更新壳能力检查、测试、方案文档和共享决策记录
This commit is contained in:
段舒康
2026-06-18 05:25:39 +08:00
parent 51dcff6d16
commit 3be997e286
15 changed files with 627 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/desktop-shell/scripts/check-config.mjs
+2
View File
@@ -149,6 +149,7 @@ const requiredMainSnippets = [
'"file.importText"',
'"file.exportImage"',
'"file.importImage"',
'"file.importAudio"',
'"file.imageDropped"',
'"notification.showLocal"',
'tauri_plugin_dialog::init()',
@@ -161,6 +162,7 @@ const requiredMainSnippets = [
'blocking_pick_file',
'import_text_file_payload',
'import_image_file_payload',
'import_audio_file_payload',
'set_title',
'set_badge_count',
'window.reload()',
apps/desktop-shell/src-tauri/src/main.rs
+131
View File
@@ -25,6 +25,7 @@ const EXPORT_TEXT_MAX_BYTES: usize = 5 * 1024 * 1024;
const EXPORT_IMAGE_MAX_BYTES: usize = 5 * 1024 * 1024;
const IMPORT_TEXT_MAX_BYTES: u64 = 5 * 1024 * 1024;
const IMPORT_IMAGE_MAX_BYTES: u64 = 10 * 1024 * 1024;
const IMPORT_AUDIO_MAX_BYTES: u64 = 20 * 1024 * 1024;
const EXPORT_FILE_NAME_FALLBACK: &str = "genarrative-export.txt";
const EXPORT_FILE_NAME_MAX_LENGTH: usize = 120;
const BADGE_COUNT_MAX: i64 = 99999;
@@ -109,6 +110,7 @@ fn capabilities() -> Vec<&'static str> {
"file.importText",
"file.exportImage",
"file.importImage",
"file.importAudio",
"file.imageDropped",
"notification.showLocal",
]
@@ -457,6 +459,22 @@ fn import_image_mime_type(path: &Path) -> Option<&'static str> {
}
}
fn import_audio_mime_type(path: &Path) -> Option<&'static str> {
match path
.extension()
.and_then(|extension| extension.to_str())
.map(|extension| extension.to_ascii_lowercase())
.as_deref()
{
Some("mp3") => Some("audio/mpeg"),
Some("m4a") | Some("mp4") => Some("audio/mp4"),
Some("wav") => Some("audio/wav"),
Some("ogg") => Some("audio/ogg"),
Some("webm") => Some("audio/webm"),
_ => None,
}
}
fn normalize_export_image_file_name(raw_file_name: &str, mime_type: &str) -> String {
let mut file_name = normalize_export_file_name(raw_file_name);
let extension = export_image_extension(mime_type).unwrap_or("png");
@@ -585,6 +603,39 @@ fn import_image_file_payload(
Ok(payload)
}
fn import_audio_file_payload(path: PathBuf) -> Result<Value, String> {
if !path.is_file() {
return Err("audio file is required".to_string());
}
let mime_type =
import_audio_mime_type(&path).ok_or_else(|| "audio MIME must be allowed".to_string())?;
let metadata = fs::metadata(&path).map_err(|error| error.to_string())?;
let byte_count = metadata.len();
if byte_count == 0 || byte_count > IMPORT_AUDIO_MAX_BYTES {
return Err("audio exceeds import size limit".to_string());
}
let bytes = fs::read(&path).map_err(|error| error.to_string())?;
let byte_count = bytes.len() as u64;
if byte_count == 0 || byte_count > IMPORT_AUDIO_MAX_BYTES {
return Err("audio exceeds import size limit".to_string());
}
let file_name = path
.file_name()
.and_then(|name| name.to_str())
.map(normalize_export_file_name)
.unwrap_or_else(|| "genarrative-import-audio.webm".to_string());
Ok(json!({
"action": "selected",
"fileName": file_name,
"base64Data": BASE64_STANDARD.encode(bytes),
"mimeType": mime_type,
"bytes": byte_count,
}))
}
fn host_bridge_event_script(event: &str, payload: Value) -> Result<String, serde_json::Error> {
let message = json!({
"bridge": HOST_BRIDGE_PROTOCOL,
@@ -1047,6 +1098,28 @@ async fn host_bridge_request(
Err(error) => failed(request.id, "host_error", error.to_string()),
}
}
"file.importAudio" => {
let file_path = app
.dialog()
.file()
.add_filter("Audio", &["mp3", "m4a", "mp4", "wav", "ogg", "webm"])
.blocking_pick_file();
let Some(file_path) = file_path else {
return failed(request.id, "cancelled", "file import cancelled");
};
let path = match file_path.into_path() {
Ok(path) => path,
Err(error) => return failed(request.id, "host_error", error.to_string()),
};
let import_result =
tauri::async_runtime::spawn_blocking(move || import_audio_file_payload(path))
.await;
match import_result {
Ok(Ok(payload)) => ok(request.id, payload),
Ok(Err(error)) => failed(request.id, "invalid_request", error),
Err(error) => failed(request.id, "host_error", error.to_string()),
}
}
"app.setTitle" => {
let title = match required_string_payload(&request, "title")
.ok()
@@ -1248,6 +1321,10 @@ mod tests {
.as_array()
.unwrap()
.contains(&json!("file.importImage")));
assert!(result["capabilities"]
.as_array()
.unwrap()
.contains(&json!("file.importAudio")));
assert!(result["capabilities"]
.as_array()
.unwrap()
@@ -1666,6 +1743,60 @@ mod tests {
fs::remove_file(large_path).expect("remove large image");
}
#[test]
fn import_audio_file_payload_reads_allowed_audio_without_exposing_path() {
let path = std::env::temp_dir().join(format!(
"genarrative-host-bridge-import-audio-{}.webm",
std::process::id()
));
fs::write(&path, b"audio").expect("write import audio");
let payload = import_audio_file_payload(path.clone()).expect("payload");
assert_eq!(payload["action"], "selected");
assert_eq!(
payload["fileName"],
path.file_name().unwrap().to_str().unwrap()
);
assert_eq!(payload["base64Data"], "YXVkaW8=");
assert_eq!(payload["mimeType"], "audio/webm");
assert_eq!(payload["bytes"], 5);
assert!(!payload
.to_string()
.contains(path.to_string_lossy().as_ref()));
fs::remove_file(path).expect("remove import audio");
}
#[test]
fn import_audio_file_payload_rejects_invalid_or_oversized_audio() {
let text_path = std::env::temp_dir().join(format!(
"genarrative-host-bridge-import-audio-{}.txt",
std::process::id()
));
fs::write(&text_path, b"audio").expect("write text file");
assert_eq!(
import_audio_file_payload(text_path.clone()).unwrap_err(),
"audio MIME must be allowed"
);
fs::remove_file(text_path).expect("remove text file");
let large_path = std::env::temp_dir().join(format!(
"genarrative-host-bridge-import-audio-large-{}.mp3",
std::process::id()
));
fs::write(
&large_path,
vec![1u8; (IMPORT_AUDIO_MAX_BYTES + 1) as usize],
)
.expect("write large audio");
assert_eq!(
import_audio_file_payload(large_path.clone()).unwrap_err(),
"audio exceeds import size limit"
);
fs::remove_file(large_path).expect("remove large audio");
}
#[test]
fn export_image_payload_rejects_invalid_mime_and_base64() {
let mut invalid_mime = request("file.exportImage");
apps/desktop-shell/src-tauri/tauri.conf.json
+2 -2
View File
@@ -6,7 +6,7 @@
"build": {
"beforeDevCommand": "npm --prefix ../.. run dev:web",
"beforeBuildCommand": "npm --prefix ../.. run build:raw && npm run typecheck",
"devUrl": "http://127.0.0.1:3000/?clientRuntime=native_app&clientType=native_app&hostShell=tauri_desktop&hostPlatform=unknown&hostVersion=0.1.0&bridgeVersion=1&hostCapabilities=host.getRuntime,appearance.getColorScheme,app.lifecycle,share.open,share.setTarget,navigation.openNativePage,app.reloadWebView,app.openExternalUrl,app.setTitle,app.setBadgeCount,network.status,network.statusChanged,clipboard.writeText,clipboard.readText,file.exportText,file.importText,file.exportImage,file.importImage,file.imageDropped,notification.showLocal",
"devUrl": "http://127.0.0.1:3000/?clientRuntime=native_app&clientType=native_app&hostShell=tauri_desktop&hostPlatform=unknown&hostVersion=0.1.0&bridgeVersion=1&hostCapabilities=host.getRuntime,appearance.getColorScheme,app.lifecycle,share.open,share.setTarget,navigation.openNativePage,app.reloadWebView,app.openExternalUrl,app.setTitle,app.setBadgeCount,network.status,network.statusChanged,clipboard.writeText,clipboard.readText,file.exportText,file.importText,file.exportImage,file.importImage,file.importAudio,file.imageDropped,notification.showLocal",
"frontendDist": "../../../dist"
},
"app": {
@@ -14,7 +14,7 @@
{
"create": false,
"label": "main",
"url": "index.html?clientRuntime=native_app&clientType=native_app&hostShell=tauri_desktop&hostPlatform=unknown&hostVersion=0.1.0&bridgeVersion=1&hostCapabilities=host.getRuntime,appearance.getColorScheme,app.lifecycle,share.open,share.setTarget,navigation.openNativePage,app.reloadWebView,app.openExternalUrl,app.setTitle,app.setBadgeCount,network.status,network.statusChanged,clipboard.writeText,clipboard.readText,file.exportText,file.importText,file.exportImage,file.importImage,file.imageDropped,notification.showLocal",
"url": "index.html?clientRuntime=native_app&clientType=native_app&hostShell=tauri_desktop&hostPlatform=unknown&hostVersion=0.1.0&bridgeVersion=1&hostCapabilities=host.getRuntime,appearance.getColorScheme,app.lifecycle,share.open,share.setTarget,navigation.openNativePage,app.reloadWebView,app.openExternalUrl,app.setTitle,app.setBadgeCount,network.status,network.statusChanged,clipboard.writeText,clipboard.readText,file.exportText,file.importText,file.exportImage,file.importImage,file.importAudio,file.imageDropped,notification.showLocal",
"title": "Genarrative",
"width": 1280,
"height": 820,
apps/mobile-shell/scripts/check-config.mjs
+4
View File
@@ -182,6 +182,7 @@ for (const snippet of [
'file.importText',
'file.exportImage',
'file.importImage',
'file.importAudio',
'clipboard.readText',
'notification.showLocal',
'network.status',
@@ -196,6 +197,8 @@ for (const snippet of [
'Clipboard.getStringAsync',
'ImagePicker.launchImageLibraryAsync',
'ImagePicker.requestMediaLibraryPermissionsAsync',
'File(asset.uri)',
'file.base64()',
'normalizeHostBridgeExportFileName',
'normalizeHostBridgeClipboardText',
'base64Data',
@@ -232,6 +235,7 @@ for (const capability of [
'file.importText',
'file.exportImage',
'file.importImage',
'file.importAudio',
'haptics.impact',
'notification.showLocal',
]) {
apps/mobile-shell/src/mobileHostBridge.test.ts
+93
View File
@@ -50,6 +50,7 @@ vi.mock('expo-clipboard', () => ({
}));
const fileTexts = vi.hoisted(() => new Map<string, string>());
const fileBase64Data = vi.hoisted(() => new Map<string, string>());
const writtenFiles = vi.hoisted(
() =>
@@ -83,6 +84,10 @@ vi.mock('expo-file-system', () => ({
text() {
return Promise.resolve(fileTexts.get(this.uri) ?? '');
}
base64() {
return Promise.resolve(fileBase64Data.get(this.uri) ?? '');
}
},
}));
@@ -260,6 +265,7 @@ afterEach(() => {
vi.mocked(Sharing.shareAsync).mockReset();
vi.mocked(Share.share).mockReset();
fileTexts.clear();
fileBase64Data.clear();
writtenFiles.length = 0;
resetMobileHostBridgeForTest();
});
@@ -286,6 +292,9 @@ describe('handleMobileHostBridgeMessage', () => {
expect(
(okResponse.result as { capabilities: string[] }).capabilities,
).toContain('file.importImage');
expect(
(okResponse.result as { capabilities: string[] }).capabilities,
).toContain('file.importAudio');
expect(
(okResponse.result as { capabilities: string[] }).capabilities,
).toEqual(
@@ -954,4 +963,88 @@ describe('handleMobileHostBridgeMessage', () => {
expect(expectFailed(oversized).error.code).toBe('invalid_request');
});
test('file.importAudio 调起系统文档选择器并返回受控音频数据', async () => {
fileBase64Data.set('file:///private/mobile/hit.webm', 'YXVkaW8=');
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: false,
assets: [
{
uri: 'file:///private/mobile/hit.webm',
name: ' ../敲击:音效?.webm ',
mimeType: 'audio/webm',
size: 5,
lastModified: 1,
},
],
});
const response = await send(request('file.importAudio'));
expect(expectOk(response).result).toEqual({
action: 'selected',
fileName: '敲击-音效-.webm',
base64Data: 'YXVkaW8=',
mimeType: 'audio/webm',
bytes: 5,
});
expect(DocumentPicker.getDocumentAsync).toHaveBeenCalledWith({
copyToCacheDirectory: true,
multiple: false,
type: [
'audio/mpeg',
'audio/mp4',
'audio/wav',
'audio/ogg',
'audio/webm',
],
});
});
test('file.importAudio 取消选择并拒绝非法 MIME 与超限音频', async () => {
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: true,
assets: null,
});
const cancelled = await send(request('file.importAudio'));
expect(expectFailed(cancelled).error.code).toBe('cancelled');
fileBase64Data.set('file:///private/mobile/hit.txt', 'YXVkaW8=');
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: false,
assets: [
{
uri: 'file:///private/mobile/hit.txt',
name: 'hit.txt',
mimeType: 'text/plain',
size: 5,
lastModified: 1,
},
],
});
const unsupportedMime = await send(request('file.importAudio'));
expect(expectFailed(unsupportedMime).error.code).toBe('invalid_request');
fileBase64Data.set('file:///private/mobile/hit.webm', 'YXVkaW8=');
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: false,
assets: [
{
uri: 'file:///private/mobile/hit.webm',
name: 'hit.webm',
mimeType: 'audio/webm',
size: 20 * 1024 * 1024 + 1,
lastModified: 1,
},
],
});
const oversized = await send(request('file.importAudio'));
expect(expectFailed(oversized).error.code).toBe('invalid_request');
});
});
apps/mobile-shell/src/mobileHostBridge.ts
+98
View File
@@ -20,11 +20,13 @@ import {
type FileExportImageResult,
type FileExportTextPayload,
type FileExportTextResult,
type FileImportAudioResult,
type FileImportImageResult,
type FileImportTextResult,
type HapticsImpactPayload,
HOST_BRIDGE_PROTOCOL,
HOST_BRIDGE_VERSION,
type HostBridgeAudioMimeType,
type HostBridgeCapability,
type HostBridgeError,
type HostBridgeImageMimeType,
@@ -51,6 +53,7 @@ const EXPORT_TEXT_MAX_BYTES = 5 * 1024 * 1024;
const EXPORT_IMAGE_MAX_BYTES = 5 * 1024 * 1024;
const IMPORT_TEXT_MAX_BYTES = 5 * 1024 * 1024;
const IMPORT_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
const IMPORT_AUDIO_MAX_BYTES = 20 * 1024 * 1024;
const LOCAL_NOTIFICATION_CHANNEL_ID = 'genarrative-local';
const HOST_BRIDGE_TEXT_MIME_TYPES = new Set<HostBridgeTextMimeType>([
'text/plain',
@@ -63,6 +66,13 @@ const HOST_BRIDGE_IMAGE_MIME_TYPES = new Set<HostBridgeImageMimeType>([
'image/jpeg',
'image/webp',
]);
const HOST_BRIDGE_AUDIO_MIME_TYPES = new Set<HostBridgeAudioMimeType>([
'audio/mpeg',
'audio/mp4',
'audio/wav',
'audio/ogg',
'audio/webm',
]);
Notifications.setNotificationHandler({
handleNotification: async () => ({
@@ -92,6 +102,7 @@ export const MOBILE_HOST_CAPABILITIES: HostBridgeCapability[] = [
'file.importText',
'file.exportImage',
'file.importImage',
'file.importAudio',
'haptics.impact',
'notification.showLocal',
];
@@ -438,6 +449,37 @@ function fallbackImportedImageFileName(mimeType: HostBridgeImageMimeType) {
return 'genarrative-import.png';
}
function normalizeImportedAudioMimeType(
value: unknown,
fileName: string,
): HostBridgeAudioMimeType | null {
if (typeof value === 'string') {
const mimeType = value.toLowerCase();
if (HOST_BRIDGE_AUDIO_MIME_TYPES.has(mimeType as HostBridgeAudioMimeType)) {
return mimeType as HostBridgeAudioMimeType;
}
}
const normalizedName = fileName.toLowerCase();
if (normalizedName.endsWith('.mp3')) {
return 'audio/mpeg';
}
if (normalizedName.endsWith('.m4a') || normalizedName.endsWith('.mp4')) {
return 'audio/mp4';
}
if (normalizedName.endsWith('.wav')) {
return 'audio/wav';
}
if (normalizedName.endsWith('.ogg')) {
return 'audio/ogg';
}
if (normalizedName.endsWith('.webm')) {
return 'audio/webm';
}
return null;
}
async function importImageFile(): Promise<FileImportImageResult> {
const permission = await ImagePicker.requestMediaLibraryPermissionsAsync();
if (permission.status !== ImagePicker.PermissionStatus.GRANTED) {
@@ -499,6 +541,60 @@ async function importImageFile(): Promise<FileImportImageResult> {
};
}
async function importAudioFile(): Promise<FileImportAudioResult> {
const result = await DocumentPicker.getDocumentAsync({
copyToCacheDirectory: true,
multiple: false,
type: [
'audio/mpeg',
'audio/mp4',
'audio/wav',
'audio/ogg',
'audio/webm',
],
});
if (result.canceled) {
throw {
code: 'cancelled',
message: 'file import cancelled',
} satisfies HostBridgeError;
}
const asset = result.assets[0];
if (!asset?.uri) {
throw invalidRequest('audio file is required');
}
const fileName = normalizeHostBridgeExportFileName(
asset.name || 'genarrative-import-audio.webm',
);
const mimeType = normalizeImportedAudioMimeType(asset.mimeType, fileName);
if (!mimeType) {
throw invalidRequest('mimeType must be an allowed audio type');
}
if (
typeof asset.size === 'number' &&
(asset.size <= 0 || asset.size > IMPORT_AUDIO_MAX_BYTES)
) {
throw invalidRequest('audio exceeds file import size limit');
}
const file = new File(asset.uri);
const base64Data = await file.base64();
const bytes = base64DecodedByteLength(base64Data);
if (bytes <= 0 || bytes > IMPORT_AUDIO_MAX_BYTES) {
throw invalidRequest('audio exceeds file import size limit');
}
return {
action: 'selected',
fileName,
base64Data,
mimeType,
bytes,
};
}
async function runHaptics(payload: unknown) {
const style = (payload as HapticsImpactPayload | undefined)?.style;
const impactStyle =
@@ -733,6 +829,8 @@ async function handleRequest(request: HostBridgeRequest) {
return ok(request, await exportImageFile(request.payload));
case 'file.importImage':
return ok(request, await importImageFile());
case 'file.importAudio':
return ok(request, await importAudioFile());
case 'haptics.impact':
return ok(request, await runHaptics(request.payload));
case 'notification.showLocal':