From 6d4033f0443b8381315ab51db84e9c8c45e271ac Mon Sep 17 00:00:00 2001 From: kdletters Date: Thu, 2 Jul 2026 20:28:12 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AE=A9=E9=9F=B3=E9=A2=91=E7=94=9F=E6=88=90?= =?UTF-8?q?=E8=BF=9B=E5=85=A5=E7=B4=A0=E6=9D=90=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 音效和背景音乐生成复用媒体素材持久化路径 前端提交音频生成时携带素材库上下文并消费返回素材快照 更新音频生成契约、OpenAPI 和相关文档 补充音频素材库链路回归测试 --- .../genarrative-external-v1.openapi.json | 50 +++++++++ ...架构】图片画布编辑器MVP接入方案-2026-06-11.md | 3 +- ...构】外部OpenAPI与APIKey接入方案-2026-06-19.md | 10 +- .../generation.rs | 96 ++++++++++++++-- .../vector_engine_audio_generation/tests.rs | 12 ++ .../crates/shared-contracts/src/assets.rs | 35 ++++++ .../ImageCanvasGenerationLayerModel.test.ts | 27 ++++- .../ImageCanvasGenerationLayerModel.ts | 15 ++- ...anvasGenerationSubmissionWorkflow.test.tsx | 106 ++++++++++++++++++ ...ImageCanvasGenerationSubmissionWorkflow.ts | 7 ++ .../image-editor/editorProjectClient.test.ts | 19 ++++ .../image-editor/editorProjectClient.ts | 10 ++ 12 files changed, 371 insertions(+), 19 deletions(-) diff --git a/docs/openapi/genarrative-external-v1.openapi.json b/docs/openapi/genarrative-external-v1.openapi.json index f0bd895b4..3fab957cf 100644 --- a/docs/openapi/genarrative-external-v1.openapi.json +++ b/docs/openapi/genarrative-external-v1.openapi.json @@ -3643,6 +3643,20 @@ }, "generationInputs": { "$ref": "#/components/schemas/JsonValue" + }, + "assetFolderId": { + "type": [ + "string", + "null" + ], + "description": "传 project 时写入默认项目素材文件夹；传具体 folderId 时写入该文件夹。" + }, + "assetLabel": { + "type": [ + "string", + "null" + ], + "description": "写入素材库时使用的素材名称。" } }, "additionalProperties": false @@ -3681,6 +3695,20 @@ }, "generationInputs": { "$ref": "#/components/schemas/JsonValue" + }, + "assetFolderId": { + "type": [ + "string", + "null" + ], + "description": "传 project 时写入默认项目素材文件夹；传具体 folderId 时写入该文件夹。" + }, + "assetLabel": { + "type": [ + "string", + "null" + ], + "description": "写入素材库时使用的素材名称。" } }, "additionalProperties": false @@ -3771,6 +3799,28 @@ ], "description": "当请求携带 canvasCompletion 且服务端成功写入画布布局时返回最新项目快照。" }, + "resource": { + "anyOf": [ + { + "$ref": "#/components/schemas/EditorProjectResource" + }, + { + "type": "null" + } + ], + "description": "请求携带 projectId 时返回写入的画布资源快照。" + }, + "asset": { + "anyOf": [ + { + "$ref": "#/components/schemas/EditorAsset" + }, + { + "type": "null" + } + ], + "description": "请求携带 assetFolderId 时返回写入的账号素材快照。" + }, "queueState": { "anyOf": [ { diff --git a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md index 5f55669f2..83a8e5e54 100644 --- a/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md +++ b/docs/technical/【前端架构】图片画布编辑器MVP接入方案-2026-06-11.md @@ -52,7 +52,7 @@ - 图片、音频、视频和角色动画帧文件本体继续走 OSS / asset object；浏览器读取私有 generated 对象统一经 `/api/assets/read-url` 换签，签名 URL 可在 session 内复用，但不得作为持久化真相。`/api/assets/read-url` 属于页面展示层高频后台请求，前端统一在 `assetReadUrlService` 内做同 key pending 去重、session 缓存和跨组件节流；UI 设计切片、角色动画帧或大量素材恢复时不得绕过该服务并发换签，否则单页可在同一秒内打满发布入口 `genarrative_api_rps` burst。 - 登录态上传和生成结果必须先落 OSS / asset object，再向 `editor_project_resource` / `editor_asset` 写入轻量 `imageSrc: "/"`、`objectKey` 和 `assetObjectId`；未登录演示态可以在内存里使用 Data URL 预览，但项目、素材库、项目资源和 `editor_canvas.layers_json` 不得写入 `data:image/*`、`data:video/*`、`data:audio/*` 或 `blob:`。旧数据读取时如果已有 `objectKey`，`imageSrc` 归一成 `/`；没有 `objectKey` 的旧 Data URL 需要走修复上传并回写轻量引用。上传到生成面板参考图槽位的图片必须先创建 `editor_project_resource` 行；没有当前工程 ID 时才创建账号级 `editor_asset` 行，随后把对应 `resourceId` 或 `assetId` 写入参考图临时状态，生成请求仍使用临时状态中的图片源或 `objectKey`。 - 资源表保存资源和素材级元数据；图层位置、层级、分组选中所需 ID 和 groupId 保存在 `editor_canvas` 的布局 JSON。布局 JSON 是混合数组：普通图层按 `layerId/resourceId` 保存，生成器占位和生成器对话框按 `itemType: "generation-dialog"` 保存，不新增单独表。普通图层的新保存不再把 `assetKind/generationInputs` 写入布局 JSON；刷新时优先从 `editor_project_resource` 恢复，旧布局中的同名字段只作为兼容兜底。生成器快照必须包含生成器 ID、模式、提示词、参数、参考图、状态、占位框位置和可选 `generatedLayerId`；宣发素材生成器还必须保存并恢复 `publicationWorkflowId`、`publicationGameInfo` 和 `publicationReferences`，避免刷新后生成卡片字段或参考图丢失。生成器快照中的参考图同样只保存 `resourceId/sourceAssetId` 行引用和展示所需 label，不保存图片 Data URL、signed URL 或 `objectKey`；刷新时用 `editor_project_resource` / `editor_asset` 行恢复临时生成请求所需图片源。生成成功后仍保存该快照，只是渲染时由 `generatedLayerId` 锚定到成品图层而不重复显示灰色占位框。`generationInputs.references` 是用户可见输入快照中的行级索引，只允许保存 `{ title, label, refType, refId }`；生成接口所需的图片 Data URL、signed URL 或 `objectKey` 只存在于提交前的临时参考图状态和请求体字段，不进入资源 / 素材元数据。图层展示尺寸不再作为独立 `Size` 真相保存，刷新与新建图层均按 `Resolution`（`originalWidth/originalHeight`）原分辨率显示。图层组第一版是画布内布局语义，不单独建表。 -- 图片类和生成视频结果除作为 `editor_project_resource` 和画布图层保存外，还要写入账号级 `editor_asset` 素材库；该写入由生成 BFF 在请求携带 `assetFolderId` 时完成。`GENARRATIVE_EXTERNAL_GENERATION_MODE=queue` 下，画布图片、改图、图标素材、UI 素材提取、角色动作、视频、音效和背景音乐生成都先返回 `queueState`，前端轮询 `/api/runtime/external-generation/jobs/{jobId}` 到完成后重新读取项目快照；`inline` 或无项目上下文时才使用响应中的 resource / asset 快照做本地落画布兜底，不再把同一生成结果二次调用素材创建接口。生成请求失败、inline 完成或 queue 任务终态完成 / 失败后，右上角泥点 chip 必须通过 `/profile/dashboard` 回读余额，不做本地乐观扣减。生成视频会单独抽取首帧封面并写入 `thumbnailSrc`，素材栏和拖回画布时沿用该封面作为 poster。 +- 图片类、生成视频和音频结果除作为 `editor_project_resource` 和画布图层保存外，还要写入账号级 `editor_asset` 素材库；该写入由生成 BFF 在请求携带 `assetFolderId` 时完成。`GENARRATIVE_EXTERNAL_GENERATION_MODE=queue` 下，画布图片、改图、图标素材、UI 素材提取、角色动作、视频、音效和背景音乐生成都先返回 `queueState`，前端轮询 `/api/runtime/external-generation/jobs/{jobId}` 到完成后重新读取项目快照；`inline` 或无项目上下文时才使用响应中的 resource / asset 快照做本地落画布兜底，不再把同一生成结果二次调用素材创建接口。生成请求失败、inline 完成或 queue 任务终态完成 / 失败后，右上角泥点 chip 必须通过 `/profile/dashboard` 回读余额，不做本地乐观扣减。生成视频会单独抽取首帧封面并写入 `thumbnailSrc`，素材栏和拖回画布时沿用该封面作为 poster。 - 前端不直接订阅 SpacetimeDB，统一通过 api-server 的 `/api/editor/projects*` BFF 读写。 - 工程刷新恢复可先应用 session 级轻量项目快照缓存，让画布和素材 chrome 尽快显示；缓存快照必须排除 `data:*` / `blob:` 内联媒体，且在后端项目快照返回前不得触发自动保存。后端快照回来后覆盖本地缓存显示并恢复正常保存队列。 - 未登录用户可以使用本地演示态，但不触发工程自动保存；真实图片生成 / 修改需要登录。编辑器 API 请求允许使用 refresh cookie 静默补 access token，但 401 / 403 只在编辑器局部提示登录，不清空整站登录态，也不把后端 requestId 直接作为生图弹窗主文案。 @@ -80,6 +80,7 @@ - `POST /api/editor/ui-designs/assets/extractions`：以前端已绘入红色框选轮廓的 UI 设计图 Data URL 作为参考图，固定 `gpt-image-2` 和 `editor_green_screen` 组装的标准绿幕素材提取提示词生成素材 spritesheet，生成成功后 api-server 先保存带绿幕 spritesheet 源图，再走 `editor_green_screen` 绿幕透明化后处理，并按连通域自动拆分为 `素材 1..N`，返回结构复用图标 spritesheet 响应。请求必须携带 `aspectRatio: "1:1"`、`imageSize: "1K" | "2K"` 和 `priceMudPoints`；框选数量不超过 6 个时前端按 `1:1·1K` 与 gpt-image-2 1K 价格提交，超过 6 个时按 `1:1·2K` 与 2K 价格提交。后端必须在调用上游前校验比例、尺寸和泥点价格，只允许 `1:1 / 1K / 2K`。请求可携带 `projectId`、`assetFolderId`、`generationInputs` 和 `spritesheetLabel`，后端保存 spritesheet / 拆分素材并返回对应 resource / asset 快照；前端必须把 spritesheet 原图与拆分素材都加入画布。 - `POST /api/editor/images/edits`：按提示词、当前图片 Data URL 和最多 8 张额外参考图调用 VectorEngine edits，返回新的生成图片元数据；请求携带 project / asset 上下文时由后端创建新 resource / asset，前端只消费响应快照。 - `POST /api/editor/videos/generations`：按视频描述、模型、比例、时长、分辨率、模式、声音、默认联网搜索标记和泥点价格生成视频。前端可选模型为 `seedance2.0-fast`、`seedance2.0`、`kling3.0`、`kling3.0-omni`，默认 `seedance2.0-fast`；后端必须将 `seedance2.0-fast` 映射到 `doubao-seedance-2-0-fast-260128`，将 `seedance2.0` 映射到 `doubao-seedance-2-0-260128`，两者不得混用。后端允许 6 类比例、4 到 15 秒整数、`480p / 720p / 1080p`，并拒绝 `seedance2.0-fast + 1080p`；`sound=on/off` 映射 Ark `generate_audio=true/false`。后端复用 Ark / VectorEngine content generation task 轮询链路，下载最终视频并持久化到 OSS；请求携带 `projectId` / `assetFolderId` 时同步创建 project resource / 账号素材并返回 `project` / `asset` 快照，基础响应返回 `videoSrc`、尺寸、prompt、model、provider、taskId、durationSeconds、resolution 和 `priceMudPoints`。 +- `POST /api/editor/audios/sound-effects/generations` 与 `POST /api/editor/audios/background-music/generations`：按音效 / 背景音乐参数生成音频并持久化到 OSS；请求携带 `projectId` / `assetFolderId` 时同步创建 project resource / 账号素材并返回 `project` / `resource` / `asset` 快照，基础响应返回 `audioSrc`、prompt、model、provider、taskId、duration、歌词和 `priceMudPoints`。 所有写接口都必须校验 Bearer 登录态和 owner；接口只返回当前用户有权读取的工程与资源。 diff --git a/docs/【后端架构】外部OpenAPI与APIKey接入方案-2026-06-19.md b/docs/【后端架构】外部OpenAPI与APIKey接入方案-2026-06-19.md index f0a8aa06a..b7159d938 100644 --- a/docs/【后端架构】外部OpenAPI与APIKey接入方案-2026-06-19.md +++ b/docs/【后端架构】外部OpenAPI与APIKey接入方案-2026-06-19.md @@ -99,13 +99,13 @@ SpacetimeDB procedure： - 图片生成 / 重绘 / 规范图 / 宣发图 / UI 设计图复用 `/api/editor/images/generations` 与 `/api/editor/images/edits` 的校验、模型归一、计费和持久化规则。 - 图标 spritesheet 和 UI 设计图素材提取复用站内拆分逻辑，生成图集后按连通域切片，并把图集与切片都按请求写入项目资源和素材库。 -- 角色动画、视频、音效和背景音乐复用站内编辑器生成链路；音频类外部调用使用 API Key 所属账号作为 asset owner。 +- 角色动画、视频、音效和背景音乐复用站内编辑器生成链路；请求携带 `assetFolderId` 时按站内规则写入素材库，音频类外部调用使用 API Key 所属账号作为 asset owner。 - API Key 管理接口仍只属于登录态个人中心，不进入外部 OpenAPI JSON。 -图片类外部生成成功后，后端拿到素材后： +素材外部生成成功后，后端拿到素材后： -1. 通过 OSS / asset object adapter 持久化图片。 -2. 写入 `editor_asset`，让生成图进入账号级素材库。 +1. 通过 OSS / asset object adapter 持久化媒体文件。 +2. 写入 `editor_asset`，让生成素材进入账号级素材库。 3. 如果请求带 `projectId`，写入 `editor_project_resource`。 4. 返回图片读取地址、素材 ID、资源 ID、尺寸、prompt、model、provider 和 taskId。 @@ -136,7 +136,7 @@ docs/openapi/genarrative-external-v1.openapi.json - API Key 创建只返回一次明文，列表不返回明文。 - 撤销后的 API Key 调用外部接口返回 `401`。 -- 外部图片生成、重绘、图标拆分和 UI 素材拆分成功后，生成结果按请求同时出现在画布资源和账号级素材库。 +- 外部图片生成、重绘、图标拆分、UI 素材拆分、视频、音效和音乐生成成功后，生成结果按请求同时出现在画布资源和账号级素材库。 - 外部视频、角色动画、音效和音乐接口使用站内编辑器相同的请求校验、模型限制和价格校验。 - OpenAPI JSON 能被 `serde_json` 解析，且 security scheme 为 Bearer API Key。 - OpenAPI JSON 不包含 `/api/profile/api-keys`、`UserAccessToken` 或 API Key 管理 schema。 diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs index 2141c3cfe..717cec731 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/generation.rs @@ -20,8 +20,9 @@ use crate::{ editor_generation_source_entity_id, enqueue_editor_generation_job, }, editor_project::{ - EditorCanvasGeneratedLayerInput, build_editor_canvas_generated_layer_item, - complete_editor_canvas_generation_with_items, + EditorCanvasGeneratedLayerInput, PersistEditorGeneratedAssetRequest, + build_editor_canvas_generated_layer_item, complete_editor_canvas_generation_with_items, + persist_editor_generated_media_asset, }, http_error::AppError, request_context::RequestContext, @@ -191,6 +192,8 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( let project_id = payload.project_id.clone(); let canvas_completion = payload.canvas_completion.clone(); let generation_inputs = payload.generation_inputs.clone(); + let asset_folder_id = payload.asset_folder_id.clone(); + let asset_label = payload.asset_label.clone(); let pricing = state .editor_generation_pricing() .map_err(|error| { @@ -239,13 +242,47 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( .audio_src .ok_or_else(|| vector_engine_bad_gateway("音效生成完成但缺少播放地址")) .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + let (resource, asset) = persist_editor_generated_media_asset( + &state, + PersistEditorGeneratedAssetRequest { + project_id: project_id.clone(), + owner_user_id: owner_user_id.clone(), + folder_id: asset_folder_id, + label: asset_label + .and_then(|value| { + let trimmed = value.trim().to_string(); + (!trimmed.is_empty()).then_some(trimmed) + }) + .unwrap_or_else(|| "生成音效".to_string()), + image_src: audio_src.clone(), + object_key: generated.object_key.clone(), + asset_object_id: generated.asset_object_id.clone(), + width: EDITOR_AUDIO_WIDTH, + height: EDITOR_AUDIO_HEIGHT, + prompt: normalized.prompt.clone(), + actual_prompt: Some(normalized.prompt.clone()), + model: normalized.model.clone(), + provider: generated.provider.clone(), + task_id: generated.task_id.clone(), + source_resource_id: None, + asset_kind: Some("sound-effect".to_string()), + generation_inputs: generation_inputs.clone(), + thumbnail_src: None, + }, + ) + .await + .map_err(|error| error.into_response_with_context(Some(&request_context)))?; let completed_project = if let Some(completion) = canvas_completion.as_ref() { let layer_id = format!("layer-editor-sound-effect-{}", generated.task_id); + let resource_id = resource + .as_ref() + .map(|resource| resource.resource_id.clone()) + .unwrap_or_else(|| format!("local-resource-editor-sound-effect-{}", generated.task_id)); let item = build_editor_canvas_generated_layer_item( completion, EditorCanvasGeneratedLayerInput { layer_id: layer_id.clone(), - resource_id: format!("local-resource-editor-sound-effect-{}", generated.task_id), + resource_id, title: completion.title.trim().to_string(), src: audio_src.clone(), media_type: Some("audio".to_string()), @@ -285,6 +322,8 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( }; let completed_project = completed_project.and_then(|project| serde_json::to_value(project).ok()); + let resource = resource.and_then(|resource| serde_json::to_value(resource).ok()); + let asset = asset.and_then(|asset| serde_json::to_value(asset).ok()); Ok(json_success_body( Some(&request_context), @@ -304,6 +343,8 @@ pub(crate) async fn generate_editor_sound_effect_for_owner( price_mud_points: normalized.price_mud_points, audio_kind: "sound-effect".to_string(), project: completed_project, + resource, + asset, queue_state: None, }, )) @@ -371,6 +412,8 @@ pub(crate) async fn generate_editor_background_music_for_owner( let project_id = payload.project_id.clone(); let canvas_completion = payload.canvas_completion.clone(); let generation_inputs = payload.generation_inputs.clone(); + let asset_folder_id = payload.asset_folder_id.clone(); + let asset_label = payload.asset_label.clone(); let pricing = state .editor_generation_pricing() .map_err(|error| { @@ -417,16 +460,51 @@ pub(crate) async fn generate_editor_background_music_for_owner( .audio_src .ok_or_else(|| vector_engine_bad_gateway("背景音乐生成完成但缺少播放地址")) .map_err(|error| error.into_response_with_context(Some(&request_context)))?; + let (resource, asset) = persist_editor_generated_media_asset( + &state, + PersistEditorGeneratedAssetRequest { + project_id: project_id.clone(), + owner_user_id: owner_user_id.clone(), + folder_id: asset_folder_id, + label: asset_label + .and_then(|value| { + let trimmed = value.trim().to_string(); + (!trimmed.is_empty()).then_some(trimmed) + }) + .unwrap_or_else(|| "生成背景音乐".to_string()), + image_src: audio_src.clone(), + object_key: generated.object_key.clone(), + asset_object_id: generated.asset_object_id.clone(), + width: EDITOR_AUDIO_WIDTH, + height: EDITOR_AUDIO_HEIGHT, + prompt: normalized.gpt_description_prompt.clone(), + actual_prompt: Some(normalized.gpt_description_prompt.clone()), + model: platform_audio::SUNO_DEFAULT_MODEL.to_string(), + provider: generated.provider.clone(), + task_id: generated.task_id.clone(), + source_resource_id: None, + asset_kind: Some("background-music".to_string()), + generation_inputs: generation_inputs.clone(), + thumbnail_src: None, + }, + ) + .await + .map_err(|error| error.into_response_with_context(Some(&request_context)))?; let completed_project = if let Some(completion) = canvas_completion.as_ref() { let layer_id = format!("layer-editor-background-music-{}", generated.task_id); + let resource_id = resource + .as_ref() + .map(|resource| resource.resource_id.clone()); let item = build_editor_canvas_generated_layer_item( completion, EditorCanvasGeneratedLayerInput { layer_id: layer_id.clone(), - resource_id: format!( - "local-resource-editor-background-music-{}", - generated.task_id - ), + resource_id: resource_id.unwrap_or_else(|| { + format!( + "local-resource-editor-background-music-{}", + generated.task_id + ) + }), title: completion.title.trim().to_string(), src: audio_src.clone(), media_type: Some("audio".to_string()), @@ -466,6 +544,8 @@ pub(crate) async fn generate_editor_background_music_for_owner( }; let completed_project = completed_project.and_then(|project| serde_json::to_value(project).ok()); + let resource = resource.and_then(|resource| serde_json::to_value(resource).ok()); + let asset = asset.and_then(|asset| serde_json::to_value(asset).ok()); Ok(json_success_body( Some(&request_context), @@ -485,6 +565,8 @@ pub(crate) async fn generate_editor_background_music_for_owner( price_mud_points: normalized.price_mud_points, audio_kind: "background-music".to_string(), project: completed_project, + resource, + asset, queue_state: None, }, )) diff --git a/server-rs/crates/api-server/src/vector_engine_audio_generation/tests.rs b/server-rs/crates/api-server/src/vector_engine_audio_generation/tests.rs index 53913ee9d..711b638a4 100644 --- a/server-rs/crates/api-server/src/vector_engine_audio_generation/tests.rs +++ b/server-rs/crates/api-server/src/vector_engine_audio_generation/tests.rs @@ -112,6 +112,8 @@ fn editor_sound_effect_request_normalizes_prompt_duration_and_resolves_price() { project_id: None, canvas_completion: None, generation_inputs: None, + asset_folder_id: None, + asset_label: None, }) .expect("editor sound effect request should normalize"); @@ -131,6 +133,8 @@ fn editor_sound_effect_request_accepts_only_vidu_audio_model() { project_id: None, canvas_completion: None, generation_inputs: None, + asset_folder_id: None, + asset_label: None, }) .expect("Vidu audio model should be accepted"); @@ -143,6 +147,8 @@ fn editor_sound_effect_request_accepts_only_vidu_audio_model() { project_id: None, canvas_completion: None, generation_inputs: None, + asset_folder_id: None, + asset_label: None, }) .expect_err("Suno text-to-sound should be disabled for editor sound effects"); @@ -160,6 +166,8 @@ fn editor_sound_effect_request_rejects_duration_outside_vidu_range() { project_id: None, canvas_completion: None, generation_inputs: None, + asset_folder_id: None, + asset_label: None, }) .expect_err("duration outside 2-10 seconds should fail"); @@ -198,6 +206,8 @@ fn editor_background_music_request_forces_instrumental_and_resolves_price() { project_id: None, canvas_completion: None, generation_inputs: None, + asset_folder_id: None, + asset_label: None, }) .expect("editor background music request should normalize"); @@ -215,6 +225,8 @@ fn editor_background_music_request_rejects_prompt_over_documented_limit() { project_id: None, canvas_completion: None, generation_inputs: None, + asset_folder_id: None, + asset_label: None, }) .expect_err("Suno gpt_description_prompt should follow Apifox 200 char limit"); diff --git a/server-rs/crates/shared-contracts/src/assets.rs b/server-rs/crates/shared-contracts/src/assets.rs index 2ab13de63..90a03a889 100644 --- a/server-rs/crates/shared-contracts/src/assets.rs +++ b/server-rs/crates/shared-contracts/src/assets.rs @@ -503,6 +503,10 @@ pub struct EditorSoundEffectGenerateRequest { pub canvas_completion: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub generation_inputs: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub asset_folder_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub asset_label: Option, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] @@ -516,6 +520,10 @@ pub struct EditorBackgroundMusicGenerateRequest { pub canvas_completion: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub generation_inputs: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub asset_folder_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub asset_label: Option, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] @@ -541,6 +549,10 @@ pub struct EditorAudioGenerateResponse { #[serde(default, skip_serializing_if = "Option::is_none")] pub project: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub resource: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub asset: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub queue_state: Option, } @@ -1299,11 +1311,15 @@ mod tests { project_id: None, canvas_completion: None, generation_inputs: None, + asset_folder_id: Some("project".to_string()), + asset_label: Some("游戏音效 1".to_string()), }) .expect("sound request should serialize"); assert_eq!(sound_payload["prompt"], json!("金币掉落叮当声")); assert_eq!(sound_payload["model"], json!("audio1.0")); assert_eq!(sound_payload["duration"], json!(7)); + assert_eq!(sound_payload["assetFolderId"], json!("project")); + assert_eq!(sound_payload["assetLabel"], json!("游戏音效 1")); assert!(sound_payload.get("priceMudPoints").is_none()); assert!(sound_payload.get("sound").is_none()); assert!(sound_payload.get("type").is_none()); @@ -1322,6 +1338,8 @@ mod tests { project_id: None, canvas_completion: None, generation_inputs: None, + asset_folder_id: None, + asset_label: None, }) .expect("sound request with unset model should serialize"); assert!(unset_model_payload.get("model").is_none()); @@ -1333,6 +1351,8 @@ mod tests { project_id: None, canvas_completion: None, generation_inputs: None, + asset_folder_id: None, + asset_label: None, }) .expect("background music request should serialize"); assert_eq!( @@ -1357,6 +1377,17 @@ mod tests { price_mud_points: 10, audio_kind: "sound-effect".to_string(), project: None, + resource: None, + asset: Some(json!({ + "assetId": "asset-audio-1", + "folderId": "user-1:asset-folder:project", + "label": "游戏音效 1", + "imageSrc": "/generated-character-drafts/editor-audios/sfx.mp3", + "width": 420, + "height": 120, + "sourceType": "generated", + "assetKind": "sound-effect" + })), queue_state: None, }) .expect("audio response should serialize"); @@ -1370,6 +1401,10 @@ mod tests { ); assert_eq!(response_payload["assetObjectId"], json!("assetobj_audio_1")); assert_eq!(response_payload["audioKind"], json!("sound-effect")); + assert_eq!( + response_payload["asset"]["assetKind"], + json!("sound-effect") + ); } #[test] diff --git a/src/components/image-editor/ImageCanvasGenerationLayerModel.test.ts b/src/components/image-editor/ImageCanvasGenerationLayerModel.test.ts index 611a0cb2d..dfa3993aa 100644 --- a/src/components/image-editor/ImageCanvasGenerationLayerModel.test.ts +++ b/src/components/image-editor/ImageCanvasGenerationLayerModel.test.ts @@ -461,6 +461,27 @@ describe('ImageCanvasGenerationLayerModel', () => { priceMudPoints: 10, audioKind: 'sound-effect', durationSeconds: 75, + resource: { + resourceId: 'resource-audio-bff', + projectId: 'editor-project-1', + imageSrc: '/generated-character-drafts/editor-audios/sfx.mp3', + objectKey: 'generated-character-drafts/editor-audios/sfx.mp3', + assetObjectId: 'assetobj-audio-1', + width: 420, + height: 120, + sourceType: 'generated', + assetKind: 'sound-effect', + }, + asset: { + assetId: 'asset-audio-bff', + folderId: 'user-1:asset-folder:project', + label: '游戏音效 14', + imageSrc: '/generated-character-drafts/editor-audios/sfx.mp3', + width: 420, + height: 120, + sourceType: 'generated', + assetKind: 'sound-effect', + }, }, generatedIndex: 14, title: '游戏音效 14', @@ -482,7 +503,7 @@ describe('ImageCanvasGenerationLayerModel', () => { expect(layer).toMatchObject({ id: 'layer-audio-14', - resourceId: 'local-resource-audio-14', + resourceId: 'resource-audio-bff', title: '游戏音效 14', src: '/generated-character-drafts/editor-audios/sfx.mp3', mediaType: 'audio', @@ -500,7 +521,11 @@ describe('ImageCanvasGenerationLayerModel', () => { taskId: 'sound-task-1', objectKey: 'generated-character-drafts/editor-audios/sfx.mp3', assetObjectId: 'assetobj-audio-1', + sourceAssetId: 'asset-audio-bff', durationSeconds: 75, + generatedAssetSnapshot: expect.objectContaining({ + assetId: 'asset-audio-bff', + }), }); }); diff --git a/src/components/image-editor/ImageCanvasGenerationLayerModel.ts b/src/components/image-editor/ImageCanvasGenerationLayerModel.ts index 225090dbf..d13b56521 100644 --- a/src/components/image-editor/ImageCanvasGenerationLayerModel.ts +++ b/src/components/image-editor/ImageCanvasGenerationLayerModel.ts @@ -459,6 +459,8 @@ export function createAudioResultLayer({ generationInputs, frame, }: AudioResultLayerOptions): CanvasLayer { + const resource = generated.resource; + const asset = generated.asset; const originalWidth = generated.width || 420; const originalHeight = generated.height || 120; const { width, height } = resolveLayerResolutionSize( @@ -478,9 +480,10 @@ export function createAudioResultLayer({ return { id: `layer-audio-${generatedIndex}`, - resourceId: `local-resource-audio-${generatedIndex}`, + resourceId: + resource?.resourceId ?? `local-resource-audio-${generatedIndex}`, title, - src: generated.audioSrc, + src: resource?.imageSrc ?? generated.audioSrc, mediaType: 'audio', assetKind: generated.audioKind, x: frameX ?? worldCenter.x - width / 2, @@ -496,10 +499,12 @@ export function createAudioResultLayer({ model: generated.model, provider: generated.provider, taskId: generated.taskId, - objectKey: generated.objectKey, - assetObjectId: generated.assetObjectId, + objectKey: resource?.objectKey ?? generated.objectKey, + assetObjectId: resource?.assetObjectId ?? generated.assetObjectId, + sourceAssetId: asset?.assetId, durationSeconds: generated.durationSeconds ?? undefined, - generationInputs, + generationInputs: resource?.generationInputs ?? generationInputs, + generatedAssetSnapshot: asset ?? undefined, }; } diff --git a/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx b/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx index 5149fc0d1..ff5203247 100644 --- a/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx +++ b/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.test.tsx @@ -122,6 +122,24 @@ function createVideoGenerated(overrides = {}) { }; } +function createAudioGenerated(overrides = {}) { + return { + audioSrc: 'https://assets.example.test/generated-audio.mp3', + width: 420, + height: 120, + sourceType: 'generated' as const, + prompt: '金币掉落叮当声', + actualPrompt: '金币掉落叮当声', + model: 'audio1.0', + provider: 'VectorEngine', + taskId: 'task-audio-generated', + priceMudPoints: 5, + audioKind: 'sound-effect' as const, + durationSeconds: 7, + ...overrides, + }; +} + function createIconResult(name: string, imageSrc: string) { return { name, @@ -1110,6 +1128,94 @@ describe('useImageCanvasGenerationSubmissionWorkflow', () => { ); }); + it('submits audio generations with asset library context and upserts the returned asset', async () => { + const applyProjectSnapshot = vi.fn(); + const upsertGeneratedAsset = vi.fn(); + generateEditorSoundEffectMock.mockResolvedValueOnce( + createAudioGenerated({ + project: { + projectId: 'editor-project-audio', + title: '音频项目', + viewport: { x: 0, y: 0, scale: 1 }, + layers: [], + resources: [], + updatedAt: '2026-06-27T00:00:00.000Z', + }, + asset: { + assetId: 'asset-sound-1', + folderId: 'user-1:asset-folder:project', + label: '游戏音效 1', + imageSrc: 'https://assets.example.test/generated-audio.mp3', + width: 420, + height: 120, + sourceType: 'generated', + assetKind: 'sound-effect', + }, + }), + ); + render( + , + ); + + fireEvent.click(screen.getByRole('button', { name: '设置初始对话' })); + fireEvent.click(screen.getByRole('button', { name: '提交当前生成' })); + + await waitFor(() => { + expect(generateEditorSoundEffectMock).toHaveBeenCalledWith( + expect.objectContaining({ + prompt: '金币掉落叮当声', + duration: 7, + projectId: 'editor-project-audio', + assetFolderId: 'project', + assetLabel: '游戏音效 1', + canvasCompletion: { + dialogId: 'dialog-sound', + title: '游戏音效 1', + placeholder: { + x: 200, + y: 160, + width: 420, + height: 120, + originalWidth: 420, + originalHeight: 120, + }, + }, + }), + ); + }); + await waitFor(() => { + expect(applyProjectSnapshot).toHaveBeenCalledWith( + expect.objectContaining({ projectId: 'editor-project-audio' }), + ); + expect(upsertGeneratedAsset).toHaveBeenCalledWith( + expect.objectContaining({ + assetId: 'asset-sound-1', + assetKind: 'sound-effect', + }), + ); + }); + }); + it('resolves uploaded objectKey references before submitting image generation', async () => { resolveEditorImageReferenceDataUrlForGenerationMock.mockResolvedValueOnce( 'data:image/jpeg;base64,normalized-ref', diff --git a/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.ts b/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.ts index b07c543aa..cf4797680 100644 --- a/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.ts +++ b/src/components/image-editor/useImageCanvasGenerationSubmissionWorkflow.ts @@ -1547,6 +1547,8 @@ export function useImageCanvasGenerationSubmissionWorkflow({ ...submissionPlan.input, projectId, generationInputs: submissionPlan.result.generationInputs, + assetFolderId, + assetLabel: submissionPlan.result.title, ...(projectId && canvasCompletionPlaceholder ? { canvasCompletion: { @@ -1561,6 +1563,8 @@ export function useImageCanvasGenerationSubmissionWorkflow({ ...submissionPlan.input, projectId, generationInputs: submissionPlan.result.generationInputs, + assetFolderId, + assetLabel: submissionPlan.result.title, ...(projectId && canvasCompletionPlaceholder ? { canvasCompletion: { @@ -1586,6 +1590,9 @@ export function useImageCanvasGenerationSubmissionWorkflow({ } if (generated.project && applyProjectSnapshot) { applyProjectSnapshot(generated.project); + if (generated.asset) { + upsertGeneratedAsset?.(generated.asset); + } return; } addAudioResultLayer( diff --git a/src/services/image-editor/editorProjectClient.test.ts b/src/services/image-editor/editorProjectClient.test.ts index 8a2598cd5..14b0fdb73 100644 --- a/src/services/image-editor/editorProjectClient.test.ts +++ b/src/services/image-editor/editorProjectClient.test.ts @@ -1255,15 +1255,28 @@ describe('editorProjectClient', () => { provider: 'VectorEngine', taskId: 'sound-task-1', audioKind: 'sound-effect', + asset: { + assetId: 'asset-sound-1', + folderId: 'user-1:asset-folder:project', + label: '游戏音效 1', + imageSrc: '/generated-character-drafts/editor-audios/sfx.mp3', + width: 420, + height: 120, + sourceType: 'generated', + assetKind: 'sound-effect', + }, }); const result = await generateEditorSoundEffect({ prompt: '金币掉落叮当声', model: 'audio1.0', duration: 7, + assetFolderId: 'project', + assetLabel: '游戏音效 1', }); expect(result.taskId).toBe('sound-task-1'); + expect(result.asset?.assetId).toBe('asset-sound-1'); expect(requestJsonMock).toHaveBeenCalledWith( '/api/editor/audios/sound-effects/generations', expect.objectContaining({ @@ -1273,6 +1286,8 @@ describe('editorProjectClient', () => { prompt: '金币掉落叮当声', model: 'audio1.0', duration: 7, + assetFolderId: 'project', + assetLabel: '游戏音效 1', }), }), '生成游戏音效失败', @@ -1334,6 +1349,8 @@ describe('editorProjectClient', () => { const result = await generateEditorBackgroundMusic({ gptDescriptionPrompt: '森林冒险背景音乐', makeInstrumental: true, + assetFolderId: 'project', + assetLabel: '游戏背景音乐 1', }); expect(result.taskId).toBe('music-task-1'); @@ -1345,6 +1362,8 @@ describe('editorProjectClient', () => { body: JSON.stringify({ gptDescriptionPrompt: '森林冒险背景音乐', makeInstrumental: true, + assetFolderId: 'project', + assetLabel: '游戏背景音乐 1', }), }), '生成游戏背景音乐失败', diff --git a/src/services/image-editor/editorProjectClient.ts b/src/services/image-editor/editorProjectClient.ts index 77c5eee7c..072b75804 100644 --- a/src/services/image-editor/editorProjectClient.ts +++ b/src/services/image-editor/editorProjectClient.ts @@ -393,6 +393,8 @@ export type EditorSoundEffectGenerationInput = { projectId?: string | null; canvasCompletion?: EditorCanvasGenerationCompletionInput | null; generationInputs?: EditorAssetGenerationInputs | null; + assetFolderId?: string | null; + assetLabel?: string | null; }; export type EditorBackgroundMusicGenerationInput = { @@ -401,6 +403,8 @@ export type EditorBackgroundMusicGenerationInput = { projectId?: string | null; canvasCompletion?: EditorCanvasGenerationCompletionInput | null; generationInputs?: EditorAssetGenerationInputs | null; + assetFolderId?: string | null; + assetLabel?: string | null; }; export type EditorAudioGenerationResult = { @@ -418,6 +422,8 @@ export type EditorAudioGenerationResult = { priceMudPoints: number; audioKind: 'sound-effect' | 'background-music'; durationSeconds?: number | null; + resource?: EditorProjectResourceSnapshot | null; + asset?: EditorAssetSnapshot | null; project?: EditorProjectSnapshot | null; queueState?: ExternalGenerationJobStatusRecord | null; }; @@ -1032,6 +1038,8 @@ export async function generateEditorSoundEffect( ...(input.generationInputs ? { generationInputs: input.generationInputs } : {}), + ...(input.assetFolderId ? { assetFolderId: input.assetFolderId } : {}), + ...(input.assetLabel ? { assetLabel: input.assetLabel } : {}), }), '生成游戏音效失败', { @@ -1056,6 +1064,8 @@ export async function generateEditorBackgroundMusic( ...(input.generationInputs ? { generationInputs: input.generationInputs } : {}), + ...(input.assetFolderId ? { assetFolderId: input.assetFolderId } : {}), + ...(input.assetLabel ? { assetLabel: input.assetLabel } : {}), }), '生成游戏背景音乐失败', {