GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

接入原生壳音频文件导入能力

Browse Source 
新增 file.importAudio HostBridge 契约和 H5 facade

移动端通过 Expo DocumentPicker 读取受控音频文件

桌面端通过 Tauri 文件选择框读取受控音频文件

通用音频输入面板在原生壳复用宿主导入和现有音频处理管线

更新壳能力检查、测试、方案文档和共享决策记录
This commit is contained in:
kdletters
2026-06-18 05:25:39 +08:00
parent 51dcff6d16
commit 3be997e286
15 changed files with 627 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/desktop-shell/scripts/check-config.mjs
View File

@@ -149,6 +149,7 @@ const requiredMainSnippets = [
'"file.importText"',
'"file.exportImage"',
'"file.importImage"',
'"file.importAudio"',
'"file.imageDropped"',
'"notification.showLocal"',
'tauri_plugin_dialog::init()',
@@ -161,6 +162,7 @@ const requiredMainSnippets = [
'blocking_pick_file',
'import_text_file_payload',
'import_image_file_payload',
'import_audio_file_payload',
'set_title',
'set_badge_count',
'window.reload()',

131
apps/desktop-shell/src-tauri/src/main.rs
View File

@@ -25,6 +25,7 @@ const EXPORT_TEXT_MAX_BYTES: usize = 5 * 1024 * 1024;
const EXPORT_IMAGE_MAX_BYTES: usize = 5 * 1024 * 1024;
const IMPORT_TEXT_MAX_BYTES: u64 = 5 * 1024 * 1024;
const IMPORT_IMAGE_MAX_BYTES: u64 = 10 * 1024 * 1024;
const IMPORT_AUDIO_MAX_BYTES: u64 = 20 * 1024 * 1024;
const EXPORT_FILE_NAME_FALLBACK: &str = "genarrative-export.txt";
const EXPORT_FILE_NAME_MAX_LENGTH: usize = 120;
const BADGE_COUNT_MAX: i64 = 99999;
@@ -109,6 +110,7 @@ fn capabilities() -> Vec<&'static str> {
"file.importText",
"file.exportImage",
"file.importImage",
"file.importAudio",
"file.imageDropped",
"notification.showLocal",
]
@@ -457,6 +459,22 @@ fn import_image_mime_type(path: &Path) -> Option<&'static str> {
}
}
fn import_audio_mime_type(path: &Path) -> Option<&'static str> {
match path
.extension()
.and_then(|extension| extension.to_str())
.map(|extension| extension.to_ascii_lowercase())
.as_deref()
{
Some("mp3") => Some("audio/mpeg"),
Some("m4a") | Some("mp4") => Some("audio/mp4"),
Some("wav") => Some("audio/wav"),
Some("ogg") => Some("audio/ogg"),
Some("webm") => Some("audio/webm"),
_ => None,
}
}
fn normalize_export_image_file_name(raw_file_name: &str, mime_type: &str) -> String {
let mut file_name = normalize_export_file_name(raw_file_name);
let extension = export_image_extension(mime_type).unwrap_or("png");
@@ -585,6 +603,39 @@ fn import_image_file_payload(
Ok(payload)
}
fn import_audio_file_payload(path: PathBuf) -> Result<Value, String> {
if !path.is_file() {
return Err("audio file is required".to_string());
}
let mime_type =
import_audio_mime_type(&path).ok_or_else(|| "audio MIME must be allowed".to_string())?;
let metadata = fs::metadata(&path).map_err(|error| error.to_string())?;
let byte_count = metadata.len();
if byte_count == 0 || byte_count > IMPORT_AUDIO_MAX_BYTES {
return Err("audio exceeds import size limit".to_string());
}
let bytes = fs::read(&path).map_err(|error| error.to_string())?;
let byte_count = bytes.len() as u64;
if byte_count == 0 || byte_count > IMPORT_AUDIO_MAX_BYTES {
return Err("audio exceeds import size limit".to_string());
}
let file_name = path
.file_name()
.and_then(|name| name.to_str())
.map(normalize_export_file_name)
.unwrap_or_else(|| "genarrative-import-audio.webm".to_string());
Ok(json!({
"action": "selected",
"fileName": file_name,
"base64Data": BASE64_STANDARD.encode(bytes),
"mimeType": mime_type,
"bytes": byte_count,
}))
}
fn host_bridge_event_script(event: &str, payload: Value) -> Result<String, serde_json::Error> {
let message = json!({
"bridge": HOST_BRIDGE_PROTOCOL,
@@ -1047,6 +1098,28 @@ async fn host_bridge_request(
Err(error) => failed(request.id, "host_error", error.to_string()),
}
}
"file.importAudio" => {
let file_path = app
.dialog()
.file()
.add_filter("Audio", &["mp3", "m4a", "mp4", "wav", "ogg", "webm"])
.blocking_pick_file();
let Some(file_path) = file_path else {
return failed(request.id, "cancelled", "file import cancelled");
};
let path = match file_path.into_path() {
Ok(path) => path,
Err(error) => return failed(request.id, "host_error", error.to_string()),
};
let import_result =
tauri::async_runtime::spawn_blocking(move || import_audio_file_payload(path))
.await;
match import_result {
Ok(Ok(payload)) => ok(request.id, payload),
Ok(Err(error)) => failed(request.id, "invalid_request", error),
Err(error) => failed(request.id, "host_error", error.to_string()),
}
}
"app.setTitle" => {
let title = match required_string_payload(&request, "title")
.ok()
@@ -1248,6 +1321,10 @@ mod tests {
.as_array()
.unwrap()
.contains(&json!("file.importImage")));
assert!(result["capabilities"]
.as_array()
.unwrap()
.contains(&json!("file.importAudio")));
assert!(result["capabilities"]
.as_array()
.unwrap()
@@ -1666,6 +1743,60 @@ mod tests {
fs::remove_file(large_path).expect("remove large image");
}
#[test]
fn import_audio_file_payload_reads_allowed_audio_without_exposing_path() {
let path = std::env::temp_dir().join(format!(
"genarrative-host-bridge-import-audio-{}.webm",
std::process::id()
));
fs::write(&path, b"audio").expect("write import audio");
let payload = import_audio_file_payload(path.clone()).expect("payload");
assert_eq!(payload["action"], "selected");
assert_eq!(
payload["fileName"],
path.file_name().unwrap().to_str().unwrap()
);
assert_eq!(payload["base64Data"], "YXVkaW8=");
assert_eq!(payload["mimeType"], "audio/webm");
assert_eq!(payload["bytes"], 5);
assert!(!payload
.to_string()
.contains(path.to_string_lossy().as_ref()));
fs::remove_file(path).expect("remove import audio");
}
#[test]
fn import_audio_file_payload_rejects_invalid_or_oversized_audio() {
let text_path = std::env::temp_dir().join(format!(
"genarrative-host-bridge-import-audio-{}.txt",
std::process::id()
));
fs::write(&text_path, b"audio").expect("write text file");
assert_eq!(
import_audio_file_payload(text_path.clone()).unwrap_err(),
"audio MIME must be allowed"
);
fs::remove_file(text_path).expect("remove text file");
let large_path = std::env::temp_dir().join(format!(
"genarrative-host-bridge-import-audio-large-{}.mp3",
std::process::id()
));
fs::write(
&large_path,
vec![1u8; (IMPORT_AUDIO_MAX_BYTES + 1) as usize],
)
.expect("write large audio");
assert_eq!(
import_audio_file_payload(large_path.clone()).unwrap_err(),
"audio exceeds import size limit"
);
fs::remove_file(large_path).expect("remove large audio");
}
#[test]
fn export_image_payload_rejects_invalid_mime_and_base64() {
let mut invalid_mime = request("file.exportImage");

4
apps/desktop-shell/src-tauri/tauri.conf.json
View File

@@ -6,7 +6,7 @@
"build": {
"beforeDevCommand": "npm --prefix ../.. run dev:web",
"beforeBuildCommand": "npm --prefix ../.. run build:raw && npm run typecheck",
"devUrl": "http://127.0.0.1:3000/?clientRuntime=native_app&clientType=native_app&hostShell=tauri_desktop&hostPlatform=unknown&hostVersion=0.1.0&bridgeVersion=1&hostCapabilities=host.getRuntime,appearance.getColorScheme,app.lifecycle,share.open,share.setTarget,navigation.openNativePage,app.reloadWebView,app.openExternalUrl,app.setTitle,app.setBadgeCount,network.status,network.statusChanged,clipboard.writeText,clipboard.readText,file.exportText,file.importText,file.exportImage,file.importImage,file.imageDropped,notification.showLocal",
"devUrl": "http://127.0.0.1:3000/?clientRuntime=native_app&clientType=native_app&hostShell=tauri_desktop&hostPlatform=unknown&hostVersion=0.1.0&bridgeVersion=1&hostCapabilities=host.getRuntime,appearance.getColorScheme,app.lifecycle,share.open,share.setTarget,navigation.openNativePage,app.reloadWebView,app.openExternalUrl,app.setTitle,app.setBadgeCount,network.status,network.statusChanged,clipboard.writeText,clipboard.readText,file.exportText,file.importText,file.exportImage,file.importImage,file.importAudio,file.imageDropped,notification.showLocal",
"frontendDist": "../../../dist"
},
"app": {
@@ -14,7 +14,7 @@
{
"create": false,
"label": "main",
"url": "index.html?clientRuntime=native_app&clientType=native_app&hostShell=tauri_desktop&hostPlatform=unknown&hostVersion=0.1.0&bridgeVersion=1&hostCapabilities=host.getRuntime,appearance.getColorScheme,app.lifecycle,share.open,share.setTarget,navigation.openNativePage,app.reloadWebView,app.openExternalUrl,app.setTitle,app.setBadgeCount,network.status,network.statusChanged,clipboard.writeText,clipboard.readText,file.exportText,file.importText,file.exportImage,file.importImage,file.imageDropped,notification.showLocal",
"url": "index.html?clientRuntime=native_app&clientType=native_app&hostShell=tauri_desktop&hostPlatform=unknown&hostVersion=0.1.0&bridgeVersion=1&hostCapabilities=host.getRuntime,appearance.getColorScheme,app.lifecycle,share.open,share.setTarget,navigation.openNativePage,app.reloadWebView,app.openExternalUrl,app.setTitle,app.setBadgeCount,network.status,network.statusChanged,clipboard.writeText,clipboard.readText,file.exportText,file.importText,file.exportImage,file.importImage,file.importAudio,file.imageDropped,notification.showLocal",
"title": "Genarrative",
"width": 1280,
"height": 820,

4
apps/mobile-shell/scripts/check-config.mjs
View File

@@ -182,6 +182,7 @@ for (const snippet of [
'file.importText',
'file.exportImage',
'file.importImage',
'file.importAudio',
'clipboard.readText',
'notification.showLocal',
'network.status',
@@ -196,6 +197,8 @@ for (const snippet of [
'Clipboard.getStringAsync',
'ImagePicker.launchImageLibraryAsync',
'ImagePicker.requestMediaLibraryPermissionsAsync',
'File(asset.uri)',
'file.base64()',
'normalizeHostBridgeExportFileName',
'normalizeHostBridgeClipboardText',
'base64Data',
@@ -232,6 +235,7 @@ for (const capability of [
'file.importText',
'file.exportImage',
'file.importImage',
'file.importAudio',
'haptics.impact',
'notification.showLocal',
]) {

93
apps/mobile-shell/src/mobileHostBridge.test.ts
View File

@@ -50,6 +50,7 @@ vi.mock('expo-clipboard', () => ({
}));
const fileTexts = vi.hoisted(() => new Map<string, string>());
const fileBase64Data = vi.hoisted(() => new Map<string, string>());
const writtenFiles = vi.hoisted(
() =>
@@ -83,6 +84,10 @@ vi.mock('expo-file-system', () => ({
text() {
return Promise.resolve(fileTexts.get(this.uri) ?? '');
}
base64() {
return Promise.resolve(fileBase64Data.get(this.uri) ?? '');
}
},
}));
@@ -260,6 +265,7 @@ afterEach(() => {
vi.mocked(Sharing.shareAsync).mockReset();
vi.mocked(Share.share).mockReset();
fileTexts.clear();
fileBase64Data.clear();
writtenFiles.length = 0;
resetMobileHostBridgeForTest();
});
@@ -286,6 +292,9 @@ describe('handleMobileHostBridgeMessage', () => {
expect(
(okResponse.result as { capabilities: string[] }).capabilities,
).toContain('file.importImage');
expect(
(okResponse.result as { capabilities: string[] }).capabilities,
).toContain('file.importAudio');
expect(
(okResponse.result as { capabilities: string[] }).capabilities,
).toEqual(
@@ -954,4 +963,88 @@ describe('handleMobileHostBridgeMessage', () => {
expect(expectFailed(oversized).error.code).toBe('invalid_request');
});
test('file.importAudio 调起系统文档选择器并返回受控音频数据', async () => {
fileBase64Data.set('file:///private/mobile/hit.webm', 'YXVkaW8=');
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: false,
assets: [
{
uri: 'file:///private/mobile/hit.webm',
name: ' ../敲击:音效?.webm ',
mimeType: 'audio/webm',
size: 5,
lastModified: 1,
},
],
});
const response = await send(request('file.importAudio'));
expect(expectOk(response).result).toEqual({
action: 'selected',
fileName: '敲击-音效-.webm',
base64Data: 'YXVkaW8=',
mimeType: 'audio/webm',
bytes: 5,
});
expect(DocumentPicker.getDocumentAsync).toHaveBeenCalledWith({
copyToCacheDirectory: true,
multiple: false,
type: [
'audio/mpeg',
'audio/mp4',
'audio/wav',
'audio/ogg',
'audio/webm',
],
});
});
test('file.importAudio 取消选择并拒绝非法 MIME 与超限音频', async () => {
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: true,
assets: null,
});
const cancelled = await send(request('file.importAudio'));
expect(expectFailed(cancelled).error.code).toBe('cancelled');
fileBase64Data.set('file:///private/mobile/hit.txt', 'YXVkaW8=');
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: false,
assets: [
{
uri: 'file:///private/mobile/hit.txt',
name: 'hit.txt',
mimeType: 'text/plain',
size: 5,
lastModified: 1,
},
],
});
const unsupportedMime = await send(request('file.importAudio'));
expect(expectFailed(unsupportedMime).error.code).toBe('invalid_request');
fileBase64Data.set('file:///private/mobile/hit.webm', 'YXVkaW8=');
vi.mocked(DocumentPicker.getDocumentAsync).mockResolvedValue({
canceled: false,
assets: [
{
uri: 'file:///private/mobile/hit.webm',
name: 'hit.webm',
mimeType: 'audio/webm',
size: 20 * 1024 * 1024 + 1,
lastModified: 1,
},
],
});
const oversized = await send(request('file.importAudio'));
expect(expectFailed(oversized).error.code).toBe('invalid_request');
});
});

98
apps/mobile-shell/src/mobileHostBridge.ts
View File

@@ -20,11 +20,13 @@ import {
type FileExportImageResult,
type FileExportTextPayload,
type FileExportTextResult,
type FileImportAudioResult,
type FileImportImageResult,
type FileImportTextResult,
type HapticsImpactPayload,
HOST_BRIDGE_PROTOCOL,
HOST_BRIDGE_VERSION,
type HostBridgeAudioMimeType,
type HostBridgeCapability,
type HostBridgeError,
type HostBridgeImageMimeType,
@@ -51,6 +53,7 @@ const EXPORT_TEXT_MAX_BYTES = 5 * 1024 * 1024;
const EXPORT_IMAGE_MAX_BYTES = 5 * 1024 * 1024;
const IMPORT_TEXT_MAX_BYTES = 5 * 1024 * 1024;
const IMPORT_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
const IMPORT_AUDIO_MAX_BYTES = 20 * 1024 * 1024;
const LOCAL_NOTIFICATION_CHANNEL_ID = 'genarrative-local';
const HOST_BRIDGE_TEXT_MIME_TYPES = new Set<HostBridgeTextMimeType>([
'text/plain',
@@ -63,6 +66,13 @@ const HOST_BRIDGE_IMAGE_MIME_TYPES = new Set<HostBridgeImageMimeType>([
'image/jpeg',
'image/webp',
]);
const HOST_BRIDGE_AUDIO_MIME_TYPES = new Set<HostBridgeAudioMimeType>([
'audio/mpeg',
'audio/mp4',
'audio/wav',
'audio/ogg',
'audio/webm',
]);
Notifications.setNotificationHandler({
handleNotification: async () => ({
@@ -92,6 +102,7 @@ export const MOBILE_HOST_CAPABILITIES: HostBridgeCapability[] = [
'file.importText',
'file.exportImage',
'file.importImage',
'file.importAudio',
'haptics.impact',
'notification.showLocal',
];
@@ -438,6 +449,37 @@ function fallbackImportedImageFileName(mimeType: HostBridgeImageMimeType) {
return 'genarrative-import.png';
}
function normalizeImportedAudioMimeType(
value: unknown,
fileName: string,
): HostBridgeAudioMimeType | null {
if (typeof value === 'string') {
const mimeType = value.toLowerCase();
if (HOST_BRIDGE_AUDIO_MIME_TYPES.has(mimeType as HostBridgeAudioMimeType)) {
return mimeType as HostBridgeAudioMimeType;
}
}
const normalizedName = fileName.toLowerCase();
if (normalizedName.endsWith('.mp3')) {
return 'audio/mpeg';
}
if (normalizedName.endsWith('.m4a') || normalizedName.endsWith('.mp4')) {
return 'audio/mp4';
}
if (normalizedName.endsWith('.wav')) {
return 'audio/wav';
}
if (normalizedName.endsWith('.ogg')) {
return 'audio/ogg';
}
if (normalizedName.endsWith('.webm')) {
return 'audio/webm';
}
return null;
}
async function importImageFile(): Promise<FileImportImageResult> {
const permission = await ImagePicker.requestMediaLibraryPermissionsAsync();
if (permission.status !== ImagePicker.PermissionStatus.GRANTED) {
@@ -499,6 +541,60 @@ async function importImageFile(): Promise<FileImportImageResult> {
};
}
async function importAudioFile(): Promise<FileImportAudioResult> {
const result = await DocumentPicker.getDocumentAsync({
copyToCacheDirectory: true,
multiple: false,
type: [
'audio/mpeg',
'audio/mp4',
'audio/wav',
'audio/ogg',
'audio/webm',
],
});
if (result.canceled) {
throw {
code: 'cancelled',
message: 'file import cancelled',
} satisfies HostBridgeError;
}
const asset = result.assets[0];
if (!asset?.uri) {
throw invalidRequest('audio file is required');
}
const fileName = normalizeHostBridgeExportFileName(
asset.name || 'genarrative-import-audio.webm',
);
const mimeType = normalizeImportedAudioMimeType(asset.mimeType, fileName);
if (!mimeType) {
throw invalidRequest('mimeType must be an allowed audio type');
}
if (
typeof asset.size === 'number' &&
(asset.size <= 0 || asset.size > IMPORT_AUDIO_MAX_BYTES)
) {
throw invalidRequest('audio exceeds file import size limit');
}
const file = new File(asset.uri);
const base64Data = await file.base64();
const bytes = base64DecodedByteLength(base64Data);
if (bytes <= 0 || bytes > IMPORT_AUDIO_MAX_BYTES) {
throw invalidRequest('audio exceeds file import size limit');
}
return {
action: 'selected',
fileName,
base64Data,
mimeType,
bytes,
};
}
async function runHaptics(payload: unknown) {
const style = (payload as HapticsImpactPayload | undefined)?.style;
const impactStyle =
@@ -733,6 +829,8 @@ async function handleRequest(request: HostBridgeRequest) {
return ok(request, await exportImageFile(request.payload));
case 'file.importImage':
return ok(request, await importImageFile());
case 'file.importAudio':
return ok(request, await importAudioFile());
case 'haptics.impact':
return ok(request, await runHaptics(request.payload));
case 'notification.showLocal':

7
docs/project-memory/shared-memory/decision-log.md
View File

@@ -2290,3 +2290,10 @@
- 决策：新增 HostBridge method `app.reloadWebView` 和 H5 facade `reloadHostWebView()`。移动端只调用当前 `react-native-webview` 的 `reload()`，桌面端只调用 Tauri 主 `WebviewWindow.reload()`；该 method 不接受 payload，成功只表示宿主已发起刷新，刷新后当前 H5 上下文会卸载。继续把同源跳转留给 `navigation.openNativePage`，外链离开容器留给 `app.openExternalUrl`。
- 影响范围：`packages/shared/src/contracts/hostBridge.ts`、`src/services/host-bridge/hostBridge.ts`、`apps/mobile-shell/`、`apps/desktop-shell/`、原生壳能力检查脚本和 HostBridge 架构文档。
- 验证方式：`npm run check:native-shells`、`npm run typecheck`、`npm run check:encoding`、`git diff --check`。
## 2026-06-18 原生壳音频文件导入只返回受控内容副本
- 背景：木鱼等固定玩法的音频上传面板需要在 Expo 移动壳和 Tauri 桌面壳内走真实系统选择器；如果直接暴露设备 URI、本机路径或通用文件系统能力，会把一次用户选择扩大成长期本地文件权限。
- 决策：新增 HostBridge method `file.importAudio`、H5 facade `importHostAudioFile()` 和通用音频输入面板接入。移动端通过 Expo DocumentPicker，桌面端通过 Tauri 系统文件选择框；两端只接受 `audio/mpeg`、`audio/mp4`、`audio/wav`、`audio/ogg`、`audio/webm` 或对应扩展名，单次不超过 20 MiB。宿主成功时只返回清洗后的文件名、MIME、base64 内容和字节数，不返回设备 URI 或本机绝对路径，也不开放通用文件系统。H5 将宿主结果转换成现有浏览器 `File`，继续复用 `readFileAsAsset(file, 'uploaded')` 音频处理链路。
- 影响范围：`packages/shared/src/contracts/hostBridge.ts`、`src/services/host-bridge/hostBridge.ts`、`src/components/common/CreativeAudioInputPanel.tsx`、`apps/mobile-shell/`、`apps/desktop-shell/`、原生壳能力检查脚本和 HostBridge 架构文档。
- 验证方式：`npm run check:native-shells`、`npm run test -- src/components/common/CreativeAudioInputPanel.test.tsx`、针对变更文件执行 ESLint、`npm run typecheck`、`npm run check:encoding`、`git diff --check`。

5
docs/【前端架构】ExpoReactNative与Tauri宿主壳方案-2026-06-17.md
View File

File diff suppressed because one or more lines are too long

1
docs/【前端架构】宿主壳能力统一协议-2026-06-17.md
View File

@@ -61,6 +61,7 @@ AI H5 sandbox
- `importHostTextFile()`：原生 App 宿主的受控文本导入入口。Expo 移动壳通过 Expo DocumentPicker 打开系统文档选择器，Tauri 桌面壳通过系统文件选择框读取用户选择的文本文件；两端都只接受 `text/plain`、`text/markdown`、`text/csv`、`application/json` 或对应扩展名，单次不超过 5 MiB，成功只返回清洗后的文件名、MIME、UTF-8 文本内容和字节数，不暴露设备本地 URI 或本机绝对路径，也不开放通用文件系统能力；用户取消时由 H5 facade 归为 `false`。
- `exportHostImageFile()`：原生 App 宿主的受控图片导出入口。H5 只传自己生成的图片 `base64Data`、清洗后的文件名和允许的 `image/png` / `image/jpeg` / `image/webp` MIME；Expo 移动壳写入缓存图片后交给系统分享 / 保存面板，Tauri 桌面壳打开系统保存对话框并写入图片字节。单次图片不超过 5 MiB，成功只返回文件名和字节数，不回传本机绝对路径。当前分享卡下载在 native app 中优先走 `file.exportImage`，宿主未声明时保留浏览器下载路径。
- `importHostImageFile()` / `subscribeHostImageDrop()`：原生 App 宿主的受控图片导入入口。Expo 移动壳通过 Expo ImagePicker 请求相册权限并打开系统相册选择器，Tauri 壳通过系统文件选择框或主窗口拖拽事件读取用户选择 / 拖入的图片；两端都只接受 `image/png`、`image/jpeg`、`image/webp`，单次不超过 10 MiB，成功只返回文件名、MIME、base64 内容、字节数和可选拖入坐标，不暴露设备本地 URI 或本机绝对路径，也不开放通用文件系统能力。H5 的通用图片输入面板 `CreativeImageInputPanel` 在 `native_app` 且声明 `file.importImage` 时优先调用宿主导入，并把结果转换成现有 `File` 回调；在桌面壳同时声明 `file.imageDropped` 时，只有拖入坐标命中当前主图卡片且未被上层元素遮挡的面板会消费该事件。普通浏览器、小程序和未声明能力的裁剪壳继续使用浏览器文件输入。
- `importHostAudioFile()`：原生 App 宿主的受控音频导入入口。Expo 移动壳通过 Expo DocumentPicker 打开系统音频选择器，Tauri 壳通过系统文件选择框读取用户选择的音频；两端都只接受 `audio/mpeg`、`audio/mp4`、`audio/wav`、`audio/ogg`、`audio/webm` 或对应扩展名，单次不超过 20 MiB，成功只返回清洗后的文件名、MIME、base64 内容和字节数，不暴露设备本地 URI 或本机绝对路径，也不开放通用文件系统能力。H5 的通用音频输入面板 `CreativeAudioInputPanel` 在 `native_app` 且声明 `file.importAudio` 时优先调用宿主导入，并把结果转换成现有 `File` 后继续复用 `readFileAsAsset(file, 'uploaded')` 音频处理链路；普通浏览器、小程序和未声明能力的裁剪壳继续使用浏览器文件输入。
## 迁移顺序

1
packages/shared/src/contracts/hostBridge.test.ts
View File

@@ -60,6 +60,7 @@ describe('HostBridge shared contract helpers', () => {
expect(isHostBridgeCapability('clipboard.readText')).toBe(true);
expect(isHostBridgeCapability('file.importText')).toBe(true);
expect(isHostBridgeCapability('file.importImage')).toBe(true);
expect(isHostBridgeCapability('file.importAudio')).toBe(true);
expect(isHostBridgeCapability('file.imageDropped')).toBe(true);
expect(isHostBridgeCapability('app.setBadgeCount')).toBe(true);
expect(isHostBridgeCapability('notification.showLocal')).toBe(true);

16
packages/shared/src/contracts/hostBridge.ts
View File

@@ -32,6 +32,7 @@ export const HOST_BRIDGE_METHODS = [
'file.importText',
'file.exportImage',
'file.importImage',
'file.importAudio',
'haptics.impact',
'notification.showLocal',
] as const;
@@ -336,6 +337,21 @@ export type FileImportImageResult = {
};
};
export type HostBridgeAudioMimeType =
| 'audio/mpeg'
| 'audio/mp4'
| 'audio/wav'
| 'audio/ogg'
| 'audio/webm';
export type FileImportAudioResult = {
action: 'selected';
fileName: string;
base64Data: string;
mimeType: HostBridgeAudioMimeType;
bytes: number;
};
export type HapticsImpactPayload = {
style?: 'light' | 'medium' | 'heavy';
};

58
src/components/common/CreativeAudioInputPanel.test.tsx
View File

@@ -4,6 +4,8 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import type { ComponentProps } from 'react';
import { afterEach, expect, test, vi } from 'vitest';
import { resetHostRuntimeCacheForTest } from '../../services/host-bridge/hostBridge';
import { resetNativeAppHostBridgeForTest } from '../../services/host-bridge/nativeAppHostBridge';
import type { CreativeAudioAsset } from './creativeAudioFileAsset';
import { CreativeAudioInputPanel } from './CreativeAudioInputPanel';
@@ -18,6 +20,10 @@ afterEach(() => {
configurable: true,
value: originalMediaDevices,
});
window.history.replaceState(null, '', '/');
delete window.__TAURI__;
resetNativeAppHostBridgeForTest();
resetHostRuntimeCacheForTest();
vi.restoreAllMocks();
});
@@ -159,6 +165,58 @@ test('上传音频成功后清空错误并写入资产', async () => {
expect(onError).toHaveBeenCalledWith(null);
});
test('原生 App 宿主可用时上传按钮走 HostBridge 音频导入', async () => {
const invoke = vi.fn(
async (_command: string, args?: Record<string, unknown>) => {
const request = (args as { request: { id: string } }).request;
return {
bridge: 'GenarrativeHostBridge',
version: 1,
id: request.id,
ok: true,
result: {
action: 'selected',
fileName: 'hit.webm',
base64Data: 'YXVkaW8=',
mimeType: 'audio/webm',
bytes: 5,
},
};
},
);
window.history.replaceState(
null,
'',
'/?clientRuntime=native_app&hostCapabilities=file.importAudio',
);
window.__TAURI__ = {
core: {
invoke: async <Result,>(
command: string,
args?: Record<string, unknown>,
) => (await invoke(command, args)) as Result,
},
};
const { readFileAsAsset, onAssetChange, onError } = renderPanel();
fireEvent.click(screen.getByText('上传'));
await waitFor(() => expect(readFileAsAsset).toHaveBeenCalledTimes(1));
const [file, source] = readFileAsAsset.mock.calls[0]!;
expect(file).toBeInstanceOf(File);
expect((file as File).name).toBe('hit.webm');
expect((file as File).type).toBe('audio/webm');
expect(source).toBe('uploaded');
await waitFor(() => expect(onAssetChange).toHaveBeenCalledTimes(1));
expect(onError).toHaveBeenCalledWith(null);
expect(invoke).toHaveBeenCalledWith('host_bridge_request', {
request: expect.objectContaining({
method: 'file.importAudio',
timeoutMs: 30000,
}),
});
});
test('上传音频失败时提示错误且不写入资产', async () => {
const readFileAsAsset = vi.fn(async () => {
throw new Error('音频最长 1 秒。');

49
src/components/common/CreativeAudioInputPanel.tsx
View File

@@ -1,6 +1,11 @@
import { Mic, Pause, Upload } from 'lucide-react';
import { useRef, useState } from 'react';
import {
canUseNativeHostCapability,
type HostFileImportAudioResult,
importHostAudioFile,
} from '../../services/host-bridge/hostBridge';
import {
type CreativeAudioAsset,
readCreativeAudioFileAsAsset,
@@ -36,8 +41,34 @@ export function CreativeAudioInputPanel<TAsset extends CreativeAudioAsset>({
readFileAsAsset = readCreativeAudioFileAsAsset,
}: CreativeAudioInputPanelProps<TAsset>) {
const [isRecording, setIsRecording] = useState(false);
const uploadInputRef = useRef<HTMLInputElement | null>(null);
const recorderRef = useRef<MediaRecorder | null>(null);
const chunksRef = useRef<BlobPart[]>([]);
const canImportHostAudio = canUseNativeHostCapability('file.importAudio');
const hostAudioImportResultToFile = (result: HostFileImportAudioResult) => {
const binary = atob(result.base64Data);
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) {
bytes[index] = binary.charCodeAt(index);
}
return new File([bytes], result.fileName, {
type: result.mimeType,
});
};
const importHostAudioAsUploadedAsset = async () => {
const result = await importHostAudioFile();
if (!result) {
return;
}
const file = hostAudioImportResultToFile(result);
const nextAsset = await readFileAsAsset(file, 'uploaded');
onError(null);
onAssetChange(nextAsset);
};
const startRecording = async () => {
if (disabled || isRecording) {
@@ -129,10 +160,28 @@ export function CreativeAudioInputPanel<TAsset extends CreativeAudioAsset>({
className={`min-h-10 cursor-pointer gap-2 px-3 ${
disabled ? 'pointer-events-none opacity-55' : ''
}`}
onClick={(event) => {
if (disabled) {
return;
}
if (!canImportHostAudio) {
return;
}
event.preventDefault();
void importHostAudioAsUploadedAsset().catch((caughtError) => {
onError(
caughtError instanceof Error
? caughtError.message
: '音频读取失败。',
);
});
}}
>
<Upload className="h-4 w-4" />
上传
<input
ref={uploadInputRef}
type="file"
accept="audio/*"
disabled={disabled}

101
src/services/host-bridge/hostBridge.test.ts
View File

@@ -12,6 +12,7 @@ import {
getHostNetworkStatus,
getHostRuntime,
getNativeAppHostRuntime,
importHostAudioFile,
importHostImageFile,
importHostTextFile,
isWechatMiniProgramWebViewRuntime,
@@ -617,6 +618,14 @@ describe('hostBridge', () => {
mimeType: 'text/markdown',
bytes: 12,
}
: request.method === 'file.importAudio'
? {
action: 'selected',
fileName: '敲击音效.webm',
base64Data: 'YXVkaW8=',
mimeType: 'audio/webm',
bytes: 5,
}
: true,
};
});
@@ -641,6 +650,7 @@ describe('hostBridge', () => {
'file.importText',
'file.exportImage',
'file.importImage',
'file.importAudio',
'file.imageDropped',
'notification.showLocal',
]),
@@ -717,6 +727,13 @@ describe('hostBridge', () => {
mimeType: 'text/markdown',
bytes: 12,
});
await expect(importHostAudioFile()).resolves.toEqual({
action: 'selected',
fileName: '敲击音效.webm',
base64Data: 'YXVkaW8=',
mimeType: 'audio/webm',
bytes: 5,
});
await expect(
showHostLocalNotification({
title: ' 生成完成 ',
@@ -837,6 +854,12 @@ describe('hostBridge', () => {
timeoutMs: 30000,
}),
});
expect(invoke).toHaveBeenCalledWith('host_bridge_request', {
request: expect.objectContaining({
method: 'file.importAudio',
timeoutMs: 30000,
}),
});
expect(invoke).toHaveBeenCalledWith('host_bridge_request', {
request: expect.objectContaining({
method: 'notification.showLocal',
@@ -915,6 +938,7 @@ describe('hostBridge', () => {
).resolves.toBe(false);
await expect(importHostImageFile()).resolves.toBe(false);
await expect(importHostTextFile()).resolves.toBe(false);
await expect(importHostAudioFile()).resolves.toBe(false);
await expect(
showHostLocalNotification({
title: '生成完成',
@@ -941,6 +965,7 @@ describe('hostBridge', () => {
).resolves.toBe(false);
await expect(importHostImageFile()).resolves.toBe(false);
await expect(importHostTextFile()).resolves.toBe(false);
await expect(importHostAudioFile()).resolves.toBe(false);
await expect(
showHostLocalNotification({
title: '生成完成',
@@ -1236,6 +1261,82 @@ describe('hostBridge', () => {
});
});
test('原生 App 宿主通过 HostBridge 导入音频文件', async () => {
const invoke = vi.fn(
async (_command: string, args?: Record<string, unknown>) => {
const request = (args as { request: { id: string } }).request;
return {
bridge: 'GenarrativeHostBridge',
version: 1,
id: request.id,
ok: true,
result: {
action: 'selected',
fileName: ' 敲击音效.webm ',
base64Data: 'YXVkaW8=',
mimeType: 'audio/webm',
bytes: 5,
},
};
},
);
window.history.replaceState(
null,
'',
nativeAppPath(['file.importAudio']),
);
window.__TAURI__ = {
core: {
invoke: asTauriInvoke(invoke),
},
};
await expect(importHostAudioFile()).resolves.toEqual({
action: 'selected',
fileName: '敲击音效.webm',
base64Data: 'YXVkaW8=',
mimeType: 'audio/webm',
bytes: 5,
});
expect(invoke).toHaveBeenCalledWith('host_bridge_request', {
request: expect.objectContaining({
method: 'file.importAudio',
timeoutMs: 30000,
}),
});
});
test('原生 App 宿主取消导入音频时回退为 false', async () => {
const invoke = vi.fn(
async (_command: string, args?: Record<string, unknown>) => {
const request = (args as { request: { id: string } }).request;
return {
bridge: 'GenarrativeHostBridge',
version: 1,
id: request.id,
ok: false,
error: {
code: 'cancelled',
message: 'file import cancelled',
},
};
},
);
window.history.replaceState(
null,
'',
nativeAppPath(['file.importAudio']),
);
window.__TAURI__ = {
core: {
invoke: asTauriInvoke(invoke),
},
};
await expect(importHostAudioFile()).resolves.toBe(false);
});
test('原生 App 宿主取消导入文本时回退为 false', async () => {
const invoke = vi.fn(
async (_command: string, args?: Record<string, unknown>) => {

61
src/services/host-bridge/hostBridge.ts
View File

@@ -6,9 +6,11 @@ import type {
FileExportImageResult,
FileExportTextPayload,
FileExportTextResult,
FileImportAudioResult,
FileImportImageResult,
FileImportTextResult,
HapticsImpactPayload,
HostBridgeAudioMimeType,
HostBridgeCapability,
HostBridgeImageMimeType,
HostBridgeMethod,
@@ -98,6 +100,8 @@ export type HostFileImportImageResult = FileImportImageResult;
export type HostFileImportTextResult = FileImportTextResult;
export type HostFileImportAudioResult = FileImportAudioResult;
export type HostClipboardWriteTextRequest = {
text: string;
};
@@ -901,6 +905,63 @@ export async function importHostTextFile() {
}
}
const HOST_BRIDGE_AUDIO_MIME_TYPES = new Set<HostBridgeAudioMimeType>([
'audio/mpeg',
'audio/mp4',
'audio/wav',
'audio/ogg',
'audio/webm',
]);
function normalizeHostAudioImportResult(
payload: FileImportAudioResult | null | undefined,
): FileImportAudioResult | false {
if (
!payload ||
typeof payload !== 'object' ||
payload.action !== 'selected' ||
!HOST_BRIDGE_AUDIO_MIME_TYPES.has(payload.mimeType) ||
typeof payload.fileName !== 'string' ||
!payload.fileName.trim() ||
typeof payload.base64Data !== 'string' ||
!payload.base64Data.trim() ||
typeof payload.bytes !== 'number' ||
!Number.isInteger(payload.bytes) ||
payload.bytes <= 0
) {
return false;
}
return {
action: 'selected',
fileName: payload.fileName.trim(),
base64Data: payload.base64Data,
mimeType: payload.mimeType,
bytes: payload.bytes,
};
}
export async function importHostAudioFile() {
if (!canUseNativeHostCapability('file.importAudio')) {
return false;
}
try {
return normalizeHostAudioImportResult(
await requestNativeAppHostBridge<FileImportAudioResult>(
'file.importAudio',
undefined,
{ timeoutMs: 30000 },
),
);
} catch (error) {
if (isUnsupportedHostBridgeError(error)) {
return false;
}
throw error;
}
}
export async function requestHostHapticsImpact(
params: HostHapticsImpactRequest = {},
) {