diff --git a/apps/ai-game-creator-shell/src-tauri/src/assets.rs b/apps/ai-game-creator-shell/src-tauri/src/assets.rs index c6df5a8a7..d15eaadc1 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/assets.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/assets.rs @@ -1638,6 +1638,8 @@ pub(crate) fn register_local_asset_entry( kind: kind.to_string(), media_type: media_type.to_string(), local_path: normalized_path.clone(), + image_sequence_frames: None, + image_sequence_duration_ms: None, source, }); Ok((id, "asset.register")) diff --git a/apps/ai-game-creator-shell/src-tauri/src/commands.rs b/apps/ai-game-creator-shell/src-tauri/src/commands.rs index 048986210..d1d5f7a57 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/commands.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/commands.rs @@ -2312,6 +2312,8 @@ mod ui_editor_font_tests { kind: "font".to_string(), media_type: "font/ttf".to_string(), local_path: relative_path.to_string(), + image_sequence_frames: None, + image_sequence_duration_ms: None, source: GameCreationAppAssetSource { kind: GameCreationAppAssetSourceKind::Uploaded, canvas_project_id: None, diff --git a/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas.rs b/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas.rs index 94a07ba68..467de49c3 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas.rs @@ -2762,6 +2762,8 @@ fn commit_asset_canvas_at_internal( kind: asset_kind, media_type: staged.media_type.clone(), local_path: final_relative_path.clone(), + image_sequence_frames: None, + image_sequence_duration_ms: None, source, }; manifest.assets.push(asset.clone()); diff --git a/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas/generation.rs b/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas/generation.rs index edc0b8b95..f36ceb74f 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas/generation.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas/generation.rs @@ -2958,6 +2958,8 @@ mod tests { kind: "illustration".to_string(), media_type: "image/png".to_string(), local_path: "assets/source.png".to_string(), + image_sequence_frames: None, + image_sequence_duration_ms: None, source: GameCreationAppAssetSource { kind: GameCreationAppAssetSourceKind::Uploaded, canvas_project_id: None, @@ -4108,6 +4110,8 @@ mod tests { kind: "illustration".to_string(), media_type: "image/png".to_string(), local_path: "assets/source.png".to_string(), + image_sequence_frames: None, + image_sequence_duration_ms: None, source: GameCreationAppAssetSource { kind: GameCreationAppAssetSourceKind::Canvas, canvas_project_id: None, diff --git a/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas_tests.rs b/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas_tests.rs index ab7ec8fc9..be228ae29 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas_tests.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project/asset_canvas_tests.rs @@ -408,6 +408,8 @@ fn refine_preserves_source_and_records_non_destructive_lineage() { kind: "illustration".to_string(), media_type: "image/png".to_string(), local_path: "assets/source.png".to_string(), + image_sequence_frames: None, + image_sequence_duration_ms: None, source: GameCreationAppAssetSource { kind: GameCreationAppAssetSourceKind::Uploaded, canvas_project_id: None, @@ -488,6 +490,8 @@ fn discovers_unique_active_refine_draft_and_rejects_ambiguous_candidates() { kind: "illustration".to_string(), media_type: "image/png".to_string(), local_path: "assets/source.png".to_string(), + image_sequence_frames: None, + image_sequence_duration_ms: None, source: GameCreationAppAssetSource { kind: GameCreationAppAssetSourceKind::Uploaded, canvas_project_id: None, @@ -979,6 +983,8 @@ fn rejects_symlinked_or_hardlinked_media_at_source_staging_and_final_paths() { kind: "illustration".to_string(), media_type: "image/png".to_string(), local_path: "assets/source.png".to_string(), + image_sequence_frames: None, + image_sequence_duration_ms: None, source: GameCreationAppAssetSource { kind: GameCreationAppAssetSourceKind::Uploaded, canvas_project_id: None, diff --git a/apps/ai-game-creator-shell/src-tauri/src/project/manifest/recovery_tests.rs b/apps/ai-game-creator-shell/src-tauri/src/project/manifest/recovery_tests.rs index e72d29970..65d8a1737 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project/manifest/recovery_tests.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project/manifest/recovery_tests.rs @@ -183,6 +183,8 @@ fn successful_first_playable_registration_creates_one_initial_version_with_asset kind: "character".to_string(), media_type: "image/png".to_string(), local_path: "assets/player.png".to_string(), + image_sequence_frames: None, + image_sequence_duration_ms: None, source: GameCreationAppAssetSource { kind: GameCreationAppAssetSourceKind::Generated, canvas_project_id: None, diff --git a/apps/ai-game-creator-shell/src-tauri/src/project/resource_dependency_graph.rs b/apps/ai-game-creator-shell/src-tauri/src/project/resource_dependency_graph.rs index 5a4a36fb1..82ec4561c 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project/resource_dependency_graph.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project/resource_dependency_graph.rs @@ -633,6 +633,8 @@ mod tests { kind: "test".to_string(), media_type: "image/png".to_string(), local_path: format!("assets/{id}.png"), + image_sequence_frames: None, + image_sequence_duration_ms: None, source: GameCreationAppAssetSource { kind: GameCreationAppAssetSourceKind::Canvas, canvas_project_id: None, diff --git a/apps/ai-game-creator-shell/src-tauri/src/project/resource_editor.rs b/apps/ai-game-creator-shell/src-tauri/src/project/resource_editor.rs index 91972015a..6b5fe501c 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project/resource_editor.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project/resource_editor.rs @@ -80,6 +80,7 @@ fn resource_edit_project_mutation_lock(root: &Path) -> Result bool { matches!( *self, - Self::ImageReference | Self::Video | Self::SoundEffect | Self::BackgroundMusic + Self::ImageReference + | Self::CharacterAnimation + | Self::Video + | Self::SoundEffect + | Self::BackgroundMusic ) } @@ -104,6 +109,7 @@ impl LocalProjectResourceEditKind { match self { Self::ImageReference => "image-reference", Self::Svg => "svg", + Self::CharacterAnimation => "character-animation", Self::Video => "video", Self::SoundEffect => "sound-effect", Self::BackgroundMusic => "background-music", @@ -114,6 +120,14 @@ impl LocalProjectResourceEditKind { } } +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum LocalProjectResourceGenerationMode { + Create, + #[default] + Derive, +} + #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub(crate) struct DeriveLocalProjectResourceInput { @@ -123,6 +137,8 @@ pub(crate) struct DeriveLocalProjectResourceInput { pub(crate) operation_id: String, pub(crate) idempotency_key: String, pub(crate) edit_kind: LocalProjectResourceEditKind, + #[serde(default)] + pub(crate) generation_mode: LocalProjectResourceGenerationMode, pub(crate) source_resource_id: String, #[serde(default)] pub(crate) source_asset_id: Option, @@ -145,6 +161,8 @@ pub(crate) struct DeriveLocalProjectResourceInput { pub(crate) struct DeriveLocalProjectResourceResult { pub(crate) operation_id: String, pub(crate) edit_kind: LocalProjectResourceEditKind, + #[serde(default)] + pub(crate) generation_mode: LocalProjectResourceGenerationMode, pub(crate) source_resource_id: String, pub(crate) committed_project_revision: u64, pub(crate) asset: Option, @@ -262,6 +280,9 @@ struct ResourceEditSourceSnapshot { asset_kind: String, source_sha256: String, bytes: Option>, + generation_mode: LocalProjectResourceGenerationMode, + source_width: Option, + source_height: Option, text: Option, source_asset: Option, source_version: Option, @@ -303,6 +324,7 @@ struct ResourceEditLedger { idempotency_key: String, request_fingerprint: String, edit_kind: LocalProjectResourceEditKind, + generation_mode: LocalProjectResourceGenerationMode, project_id: String, expected_project_revision: u64, source_resource_id: String, @@ -338,6 +360,12 @@ struct ResourceEditLedger { remote_operation_id: Option, remote_resource_id: Option, remote_object_key: Option, + #[serde(default)] + remote_legacy_public_path: Option, + #[serde(default)] + remote_sequence_frames_json: Option, + #[serde(default)] + remote_sequence_duration_ms: Option, remote_asset_object_id: Option, remote_model: Option, #[serde(default)] @@ -694,7 +722,9 @@ fn resource_edit_prompt_max_chars(edit_kind: &LocalProjectResourceEditKind) -> u match edit_kind { LocalProjectResourceEditKind::BackgroundMusic => 140, LocalProjectResourceEditKind::SoundEffect => 1_900, - LocalProjectResourceEditKind::Video => 4_000, + LocalProjectResourceEditKind::Video | LocalProjectResourceEditKind::CharacterAnimation => { + 4_000 + } _ => 32_000, } } @@ -712,6 +742,7 @@ fn normalize_resource_edit_prompt( LocalProjectResourceEditKind::BackgroundMusic => "背景音乐", LocalProjectResourceEditKind::SoundEffect => "音效", LocalProjectResourceEditKind::Video => "视频", + LocalProjectResourceEditKind::CharacterAnimation => "角色动画", _ => "", } )); @@ -775,6 +806,7 @@ fn resource_edit_request_fingerprint( "projectId": input.expected_project_id, "operationId": input.operation_id, "editKind": input.edit_kind, + "generationMode": input.generation_mode, "sourceResourceId": source.canonical_resource_id, "sourcePath": source.source_path, "sourceSha256": source.source_sha256, @@ -952,6 +984,16 @@ fn infer_resource_edit_source_media_type( edit_kind: &LocalProjectResourceEditKind, source_path: Option<&str>, ) -> Option { + if source_path.is_none() { + return match edit_kind { + LocalProjectResourceEditKind::CharacterAnimation => Some("image/png"), + LocalProjectResourceEditKind::Video => Some("video/mp4"), + LocalProjectResourceEditKind::SoundEffect + | LocalProjectResourceEditKind::BackgroundMusic => Some("audio/mpeg"), + _ => None, + } + .map(str::to_string); + } let extension = source_path .and_then(|path| Path::new(path).extension()) .and_then(|value| value.to_str()) @@ -998,6 +1040,7 @@ fn infer_resource_edit_source_asset_kind(edit_kind: &LocalProjectResourceEditKin match edit_kind { LocalProjectResourceEditKind::ImageReference => "art-image", LocalProjectResourceEditKind::Svg => "svg", + LocalProjectResourceEditKind::CharacterAnimation => "character-animation", LocalProjectResourceEditKind::Video => "video", LocalProjectResourceEditKind::SoundEffect => "sound-effect", LocalProjectResourceEditKind::BackgroundMusic => "background-music", @@ -1057,6 +1100,9 @@ fn resolve_resource_edit_source( .map_err(|error| format!("序列化源项目版本失败：{error}"))?, ), bytes: None, + generation_mode: input.generation_mode, + source_width: None, + source_height: None, text: None, source_asset: None, source_version: Some(version), @@ -1076,12 +1122,45 @@ fn resolve_resource_edit_source( asset_kind: "agent-result-derivative".to_string(), source_sha256: sha256_hex(content.as_bytes()), bytes: None, + generation_mode: input.generation_mode, + source_width: None, + source_height: None, text: Some(content), source_asset: None, source_version: None, }); } + if input.generation_mode == LocalProjectResourceGenerationMode::Create { + if !matches!( + input.edit_kind, + LocalProjectResourceEditKind::Video + | LocalProjectResourceEditKind::SoundEffect + | LocalProjectResourceEditKind::BackgroundMusic + ) { + return Err("当前资源类型不支持无源生成".to_string()); + } + let asset_kind = infer_resource_edit_source_asset_kind(&input.edit_kind); + return Ok(ResourceEditSourceSnapshot { + canonical_resource_id: input.source_resource_id.trim().to_string(), + source_path: None, + media_type: infer_resource_edit_source_media_type( + &input.edit_kind, + input.source_path.as_deref(), + ) + .unwrap_or_else(|| "application/octet-stream".to_string()), + asset_kind, + source_sha256: sha256_hex(input.prompt.trim().as_bytes()), + bytes: None, + generation_mode: input.generation_mode, + source_width: None, + source_height: None, + text: None, + source_asset: None, + source_version: None, + }); + } + let requested_path = input .source_path .as_deref() @@ -1167,6 +1246,9 @@ fn resolve_resource_edit_source( asset_kind, source_sha256: sha256_hex(text.as_bytes()), bytes: None, + generation_mode: input.generation_mode, + source_width: None, + source_height: None, text: Some(text), source_asset, source_version: None, @@ -1202,6 +1284,14 @@ fn resolve_resource_edit_source( media_read_limit(&input.edit_kind), "源媒体资源", )?; + let (source_width, source_height) = + if input.edit_kind == LocalProjectResourceEditKind::CharacterAnimation { + let decoded = image::load_from_memory(&bytes) + .map_err(|_| "生成角色动画前无法解析源图片尺寸".to_string())?; + (Some(decoded.width()), Some(decoded.height())) + } else { + (None, None) + }; Ok(ResourceEditSourceSnapshot { canonical_resource_id, source_path: Some(path), @@ -1209,6 +1299,9 @@ fn resolve_resource_edit_source( asset_kind, source_sha256: sha256_hex(&bytes), bytes: Some(bytes), + generation_mode: input.generation_mode, + source_width, + source_height, text: None, source_asset, source_version: None, @@ -1792,7 +1885,13 @@ fn build_resource_edit_audio_prompt( .and_then(|value| value.to_str()) .map(str::trim) .filter(|value| !value.is_empty()) - .unwrap_or("现有音频"); + .unwrap_or( + if source.generation_mode == LocalProjectResourceGenerationMode::Create { + "新音频" + } else { + "现有音频" + }, + ); let original_prompt = source .source_asset .as_ref() @@ -1804,8 +1903,17 @@ fn build_resource_edit_audio_prompt( source_context.push_str("，原始描述 "); source_context.push_str(original_prompt); } - let prefix = "基于现有音频资源语义派生重制；源信息："; - let instruction_prefix = "；编辑要求："; + let prefix = if source.generation_mode == LocalProjectResourceGenerationMode::Create { + "生成新音频；目标：" + } else { + "基于现有音频资源语义派生重制；源信息：" + }; + let instruction_prefix = if source.generation_mode == LocalProjectResourceGenerationMode::Create + { + "；要求：" + } else { + "；编辑要求：" + }; let fixed_chars = prefix.chars().count() + instruction_prefix.chars().count() + prompt.chars().count(); if fixed_chars >= max_chars { @@ -1825,6 +1933,7 @@ fn resource_edit_remote_request( prompt: &str, asset_name: &str, source_reference: Option<&str>, + canvas_context: Option<&ExternalCanvasGenerationContext>, ) -> Result<(&'static str, serde_json::Value), String> { let generation_inputs = serde_json::json!({ "source": RESOURCE_EDIT_QUEUE_SOURCE, @@ -1840,9 +1949,25 @@ fn resource_edit_remote_request( "generationInputs": generation_inputs, }), )), - LocalProjectResourceEditKind::Video => Ok(( - "/api/external/v1/editor/videos/generations", + LocalProjectResourceEditKind::CharacterAnimation => Ok(( + "/api/external/v1/editor/character-animations/generations", serde_json::json!({ + "sourceLayerId": format!("resource-{}", input.operation_id), + "sourceImageSrc": source_reference.ok_or_else(|| "角色动画缺少稳定源图片引用".to_string())?, + "sourceWidth": source.source_width.ok_or_else(|| "角色动画缺少源图片宽度".to_string())?, + "sourceHeight": source.source_height.ok_or_else(|| "角色动画缺少源图片高度".to_string())?, + "promptText": prompt, + "resolution": "720p", + "ratio": "same", + "frameCount": 32, + "durationSeconds": 4, + "model": "seedance2.0-fast", + "assetLabel": asset_name, + "generationInputs": generation_inputs, + }), + )), + LocalProjectResourceEditKind::Video => { + let mut body = serde_json::json!({ "prompt": prompt, "model": "seedance2.0-fast", "aspectRatio": "16:9", @@ -1851,31 +1976,66 @@ fn resource_edit_remote_request( "mode": "std", "sound": "on", "webSearchEnabled": false, - "referenceVideoSrcs": [source_reference.ok_or_else(|| "视频编辑缺少稳定源引用".to_string())?], "assetKind": "video", "assetLabel": asset_name, "generationInputs": generation_inputs, - }), - )), - LocalProjectResourceEditKind::SoundEffect => Ok(( - "/api/external/v1/editor/audios/sound-effects/generations", - serde_json::json!({ + }); + if input.generation_mode == LocalProjectResourceGenerationMode::Derive { + body["referenceVideoSrcs"] = serde_json::json!([ + source_reference.ok_or_else(|| "视频编辑缺少稳定源引用".to_string())? + ]); + } + if let Some(context) = canvas_context { + body["projectId"] = serde_json::json!(context.project_id); + body["assetFolderId"] = serde_json::json!(context.asset_folder_id); + body["canvasCompletion"] = serde_json::json!({ + "title": asset_name, + "placeholder": external_canvas_placeholder("16:9"), + }); + } + Ok(("/api/external/v1/editor/videos/generations", body)) + } + LocalProjectResourceEditKind::SoundEffect => { + let mut body = serde_json::json!({ "prompt": build_resource_edit_audio_prompt(source, prompt, 2_048)?, "model": "eleven_text_to_sound_v2", "loop": false, "assetLabel": asset_name, "generationInputs": generation_inputs, - }), - )), - LocalProjectResourceEditKind::BackgroundMusic => Ok(( - "/api/external/v1/editor/audios/background-music/generations", - serde_json::json!({ + }); + if let Some(context) = canvas_context { + body["projectId"] = serde_json::json!(context.project_id); + body["assetFolderId"] = serde_json::json!(context.asset_folder_id); + body["canvasCompletion"] = serde_json::json!({ + "title": asset_name, + "placeholder": external_canvas_placeholder("1:1"), + }); + } + Ok(( + "/api/external/v1/editor/audios/sound-effects/generations", + body, + )) + } + LocalProjectResourceEditKind::BackgroundMusic => { + let mut body = serde_json::json!({ "gptDescriptionPrompt": build_resource_edit_audio_prompt(source, prompt, 200)?, "makeInstrumental": true, "assetLabel": asset_name, "generationInputs": generation_inputs, - }), - )), + }); + if let Some(context) = canvas_context { + body["projectId"] = serde_json::json!(context.project_id); + body["assetFolderId"] = serde_json::json!(context.asset_folder_id); + body["canvasCompletion"] = serde_json::json!({ + "title": asset_name, + "placeholder": external_canvas_placeholder("1:1"), + }); + } + Ok(( + "/api/external/v1/editor/audios/background-music/generations", + body, + )) + } _ => Err("当前资源类型不是远端媒体派生".to_string()), } } @@ -1896,12 +2056,14 @@ fn resource_edit_result_has_download(payload: &serde_json::Value) -> bool { .unwrap_or(data); json_string_field(resource, "objectKey").is_some() || json_string_field(data, "objectKey").is_some() + || json_string_field(data, "previewVideoPath").is_some() } fn is_external_resource_edit_endpoint(endpoint: &str) -> bool { matches!( endpoint, "/api/external/v1/editor/images/edits" + | "/api/external/v1/editor/character-animations/generations" | "/api/external/v1/editor/videos/generations" | "/api/external/v1/editor/audios/sound-effects/generations" | "/api/external/v1/editor/audios/background-music/generations" @@ -2078,9 +2240,21 @@ async fn wait_for_resource_edit_remote( } } +struct ResourceEditRemoteIdentity { + resource_id: Option, + object_key: Option, + legacy_public_path: Option, + sequence_frames: + Option>, + sequence_duration_ms: Option, + asset_object_id: Option, + model: Option, +} + fn extract_resource_edit_remote_identity( generated: &serde_json::Value, -) -> Result<(Option, String, Option, Option), String> { + edit_kind: LocalProjectResourceEditKind, +) -> Result { let data = external_editor_response_data(generated); let null = serde_json::Value::Null; let resource = data @@ -2093,14 +2267,74 @@ fn extract_resource_edit_remote_identity( .unwrap_or(&null); let object_key = json_string_field(resource, "objectKey") .or_else(|| json_string_field(data, "objectKey")) - .ok_or_else(|| "资源编辑结果缺少稳定 objectKey".to_string())?; + .or_else(|| extract_character_animation_object_key(generated)) + .filter(|value| !value.is_empty()); + let legacy_public_path = if edit_kind == LocalProjectResourceEditKind::CharacterAnimation { + json_string_field(data, "previewVideoPath") + } else { + None + }; + let sequence_frames = if edit_kind == LocalProjectResourceEditKind::CharacterAnimation { + data.get("frames") + .and_then(serde_json::Value::as_array) + .map(|frames| { + frames + .iter() + .filter_map(|frame| { + let image_src = json_string_field(frame, "imageSrc")?; + let width = frame.get("width").and_then(serde_json::Value::as_u64)? as u32; + let height = + frame.get("height").and_then(serde_json::Value::as_u64)? as u32; + Some(shared_contracts::game_creation_app::GameCreationAppImageSequenceFrame { + image_src, + object_key: json_string_field(frame, "objectKey"), + asset_object_id: json_string_field(frame, "assetObjectId"), + width, + height, + }) + }) + .collect::>() + }) + } else { + None + }; + let sequence_duration_ms = if edit_kind == LocalProjectResourceEditKind::CharacterAnimation { + data.get("durationSeconds") + .and_then(serde_json::Value::as_u64) + .map(|seconds| seconds.saturating_mul(1_000)) + .or_else(|| { + data.get("imageSequenceDurationMs") + .and_then(serde_json::Value::as_u64) + }) + } else { + None + }; + if object_key.is_none() && legacy_public_path.is_none() { + return Err("资源编辑结果缺少稳定媒体引用".to_string()); + } let resource_id = json_string_field(resource, "resourceId").or_else(|| json_string_field(data, "resourceId")); let asset_object_id = json_string_field(resource, "assetObjectId") .or_else(|| json_string_field(asset, "assetObjectId")) .or_else(|| json_string_field(data, "assetObjectId")); let model = json_string_field(data, "model"); - Ok((resource_id, object_key, asset_object_id, model)) + Ok(ResourceEditRemoteIdentity { + resource_id, + object_key, + legacy_public_path, + sequence_frames, + sequence_duration_ms, + asset_object_id, + model, + }) +} + +fn extract_character_animation_object_key(generated: &serde_json::Value) -> Option { + let data = external_editor_response_data(generated); + data.get("frames") + .and_then(|frames| frames.as_array()) + .and_then(|frames| frames.first()) + .and_then(|frame| json_string_field(frame, "objectKey")) } fn validate_downloaded_media( @@ -2140,6 +2374,15 @@ fn validate_downloaded_media( Err("派生图片不是受支持的 PNG、JPEG 或 WebP".to_string()) } } + LocalProjectResourceEditKind::CharacterAnimation => { + if is_mp4 { + Ok(("video/mp4".to_string(), "mp4".to_string())) + } else if is_webm { + Ok(("video/webm".to_string(), "webm".to_string())) + } else { + Err("角色动画预览不是受支持的 MP4 或 WebM 视频".to_string()) + } + } LocalProjectResourceEditKind::Video => { if is_mp4 { Ok(("video/mp4".to_string(), "mp4".to_string())) @@ -2348,11 +2591,17 @@ async fn prepare_remote_resource_edit( .timeout(Duration::from_secs(35 * 60)) .build() .map_err(|_| "无法创建资源编辑 HTTP 客户端".to_string())?; - let object_key = if ledger.phase == ResourceEditLedgerPhase::RemoteCompleted { - ledger - .remote_object_key - .clone() - .ok_or_else(|| "远端已完成的资源编辑缺少稳定 objectKey".to_string())? + let download_source = if ledger.phase == ResourceEditLedgerPhase::RemoteCompleted { + if let Some(legacy_public_path) = ledger.remote_legacy_public_path.clone() { + serde_json::json!({ "imageSrc": legacy_public_path }) + } else { + serde_json::json!({ + "objectKey": ledger + .remote_object_key + .clone() + .ok_or_else(|| "远端已完成的资源编辑缺少稳定 objectKey".to_string())? + }) + } } else { let generated = if ledger.remote_operation_id.is_some() { let result = @@ -2363,8 +2612,11 @@ async fn prepare_remote_resource_edit( } else { let source_reference = if matches!( input.edit_kind, - LocalProjectResourceEditKind::ImageReference | LocalProjectResourceEditKind::Video - ) { + LocalProjectResourceEditKind::ImageReference + | LocalProjectResourceEditKind::CharacterAnimation + ) || (input.edit_kind == LocalProjectResourceEditKind::Video + && input.generation_mode == LocalProjectResourceGenerationMode::Derive) + { Some( ensure_resource_edit_source_reference( root, @@ -2398,12 +2650,27 @@ async fn prepare_remote_resource_edit( write_resource_edit_ledger(root, ledger)?; } if ledger.endpoint.is_none() || ledger.request_body_json.is_none() { + let canvas_context = + if input.generation_mode == LocalProjectResourceGenerationMode::Create { + Some( + prepare_external_canvas_generation_context( + root, + &client, + &api_base_url, + &api_key, + ) + .await?, + ) + } else { + None + }; let (endpoint, body) = resource_edit_remote_request( input, source, prompt, asset_name, source_reference.as_deref(), + canvas_context.as_ref(), )?; ledger.endpoint = Some(endpoint.to_string()); ledger.request_body_json = Some( @@ -2426,16 +2693,32 @@ async fn prepare_remote_resource_edit( ensure_frozen_resource_edit_platform_session(frozen_platform_session)?; result }; - let (resource_id, object_key, asset_object_id, model) = - extract_resource_edit_remote_identity(&generated)?; - ledger.remote_resource_id = resource_id; - ledger.remote_object_key = Some(object_key.clone()); - ledger.remote_asset_object_id = asset_object_id; - ledger.remote_model = model; + let identity = extract_resource_edit_remote_identity(&generated, input.edit_kind)?; + ledger.remote_resource_id = identity.resource_id; + ledger.remote_object_key = identity.object_key.clone(); + ledger.remote_legacy_public_path = identity.legacy_public_path.clone(); + ledger.remote_sequence_frames_json = identity + .sequence_frames + .as_ref() + .map(|frames| { + serde_json::to_string(frames) + .map_err(|error| format!("序列化角色动画序列帧失败：{error}")) + }) + .transpose()?; + ledger.remote_sequence_duration_ms = identity.sequence_duration_ms; + ledger.remote_asset_object_id = identity.asset_object_id; + ledger.remote_model = identity.model; update_resource_edit_phase(root, ledger, ResourceEditLedgerPhase::RemoteCompleted)?; - object_key + if let Some(legacy_public_path) = identity.legacy_public_path { + serde_json::json!({ "imageSrc": legacy_public_path }) + } else { + serde_json::json!({ + "objectKey": identity + .object_key + .ok_or_else(|| "资源编辑结果缺少稳定 objectKey".to_string())? + }) + } }; - let download_source = serde_json::json!({ "objectKey": object_key }); let download = resolve_canvas_resource_download_with_limit( &client, &api_base_url, @@ -2579,6 +2862,7 @@ fn committed_resource_edit_result( Ok(DeriveLocalProjectResourceResult { operation_id: input.operation_id.clone(), edit_kind: input.edit_kind.clone(), + generation_mode: input.generation_mode, source_resource_id: source_resource_id.to_string(), committed_project_revision: revision, asset, @@ -2906,6 +3190,8 @@ pub(crate) fn normalize_local_project_raster_resource_at( kind: source_subtype.to_string(), media_type: verified_media_type, local_path: source_path, + image_sequence_frames: None, + image_sequence_duration_ms: None, source: GameCreationAppAssetSource { kind: GameCreationAppAssetSourceKind::Generated, canvas_project_id: None, @@ -3116,12 +3402,25 @@ fn commit_resource_edit_asset_internal( derivative_file_stem(asset_name), staged_extension ); - let remote = ledger.remote_object_key.is_some(); + let remote = ledger.remote_object_key.is_some() || ledger.remote_legacy_public_path.is_some(); + let image_sequence_frames = ledger + .remote_sequence_frames_json + .as_deref() + .map(|json| { + serde_json::from_str::< + Vec, + >(json) + .map_err(|error| format!("解析角色动画序列帧失败：{error}")) + }) + .transpose()?; + let image_sequence_duration_ms = ledger.remote_sequence_duration_ms; let asset = GameCreationAppAssetManifestEntry { id: asset_id.clone(), kind: source.asset_kind.clone(), media_type: staged_media_type.to_string(), local_path: relative_path.clone(), + image_sequence_frames, + image_sequence_duration_ms, source: GameCreationAppAssetSource { kind: if remote { GameCreationAppAssetSourceKind::Canvas @@ -3419,6 +3718,7 @@ fn commit_resource_edit_asset_internal( Ok(DeriveLocalProjectResourceResult { operation_id: input.operation_id.clone(), edit_kind: input.edit_kind.clone(), + generation_mode: input.generation_mode, source_resource_id: source.canonical_resource_id.clone(), committed_project_revision: revision.revision, asset: Some(asset), @@ -3607,6 +3907,7 @@ fn commit_resource_edit_version( return Ok(DeriveLocalProjectResourceResult { operation_id: input.operation_id.clone(), edit_kind: input.edit_kind.clone(), + generation_mode: input.generation_mode, source_resource_id: source.canonical_resource_id.clone(), committed_project_revision: current_revision.revision, asset: None, @@ -3716,6 +4017,7 @@ fn commit_resource_edit_version( Ok(DeriveLocalProjectResourceResult { operation_id: input.operation_id.clone(), edit_kind: input.edit_kind.clone(), + generation_mode: input.generation_mode, source_resource_id: source.canonical_resource_id.clone(), committed_project_revision: current_revision.revision, asset: None, @@ -4003,6 +4305,7 @@ pub(crate) async fn resume_local_project_resource_edit_at( operation_id: ledger.operation_id, idempotency_key: ledger.idempotency_key, edit_kind: ledger.edit_kind, + generation_mode: ledger.generation_mode, source_resource_id: ledger.source_resource_id, source_asset_id: ledger .source_asset_id @@ -4069,6 +4372,7 @@ pub(crate) async fn derive_local_project_resource_at( idempotency_key: input.idempotency_key.clone(), request_fingerprint, edit_kind: input.edit_kind.clone(), + generation_mode: input.generation_mode, project_id: input.expected_project_id.clone(), expected_project_revision: input.expected_project_revision, source_resource_id: source.canonical_resource_id.clone(), @@ -4103,6 +4407,9 @@ pub(crate) async fn derive_local_project_resource_at( remote_operation_id: None, remote_resource_id: None, remote_object_key: None, + remote_legacy_public_path: None, + remote_sequence_frames_json: None, + remote_sequence_duration_ms: None, remote_asset_object_id: None, remote_model: None, terminal_failure_code: None, @@ -4329,6 +4636,7 @@ mod tests { operation_id, idempotency_key: Uuid::new_v4().to_string(), edit_kind, + generation_mode: LocalProjectResourceGenerationMode::Derive, source_resource_id, source_asset_id: None, source_path: None, @@ -4352,6 +4660,7 @@ mod tests { idempotency_key: input.idempotency_key.clone(), request_fingerprint: "test-fingerprint".to_string(), edit_kind: input.edit_kind.clone(), + generation_mode: input.generation_mode, project_id: input.expected_project_id.clone(), expected_project_revision: input.expected_project_revision, source_resource_id: source.canonical_resource_id.clone(), @@ -4386,6 +4695,9 @@ mod tests { remote_operation_id: None, remote_resource_id: None, remote_object_key: None, + remote_legacy_public_path: None, + remote_sequence_frames_json: None, + remote_sequence_duration_ms: None, remote_asset_object_id: None, remote_model: None, terminal_failure_code: None, @@ -4610,6 +4922,9 @@ mod tests { asset_kind: "video".to_string(), source_sha256: "a".repeat(64), bytes: Some(vec![1, 2, 3]), + generation_mode: LocalProjectResourceGenerationMode::Derive, + source_width: None, + source_height: None, text: None, source_asset: None, source_version: None, @@ -4724,6 +5039,9 @@ mod tests { asset_kind: "video".to_string(), source_sha256: "a".repeat(64), bytes: Some(b"\0\0\0\x18ftypisom".to_vec()), + generation_mode: LocalProjectResourceGenerationMode::Derive, + source_width: None, + source_height: None, text: None, source_asset: None, source_version: None, @@ -4857,6 +5175,9 @@ mod tests { asset_kind: "video".to_string(), source_sha256: "a".repeat(64), bytes: Some(b"\0\0\0\x18ftypisom".to_vec()), + generation_mode: LocalProjectResourceGenerationMode::Derive, + source_width: None, + source_height: None, text: None, source_asset: None, source_version: None, @@ -5025,6 +5346,9 @@ mod tests { asset_kind: "video".to_string(), source_sha256: "a".repeat(64), bytes: Some(b"\0\0\0\x18ftypisom".to_vec()), + generation_mode: LocalProjectResourceGenerationMode::Derive, + source_width: None, + source_height: None, text: None, source_asset: None, source_version: None, @@ -5152,6 +5476,9 @@ mod tests { asset_kind: "video".to_string(), source_sha256: "a".repeat(64), bytes: Some(b"\0\0\0\x18ftypisom".to_vec()), + generation_mode: LocalProjectResourceGenerationMode::Derive, + source_width: None, + source_height: None, text: None, source_asset: None, source_version: None, @@ -5320,6 +5647,9 @@ mod tests { asset_kind: "video".to_string(), source_sha256: "a".repeat(64), bytes: Some(b"\0\0\0\x18ftypisom".to_vec()), + generation_mode: LocalProjectResourceGenerationMode::Derive, + source_width: None, + source_height: None, text: None, source_asset: None, source_version: None, @@ -5464,6 +5794,9 @@ mod tests { asset_kind: "background-music".to_string(), source_sha256: "a".repeat(64), bytes: Some(vec![1]), + generation_mode: LocalProjectResourceGenerationMode::Derive, + source_width: None, + source_height: None, text: None, source_asset: None, source_version: None, @@ -5478,7 +5811,7 @@ mod tests { ); bgm.prompt = prompt.to_string(); let (bgm_endpoint, bgm_body) = - resource_edit_remote_request(&bgm, &source, prompt, "主题音乐编辑版", None) + resource_edit_remote_request(&bgm, &source, prompt, "主题音乐编辑版", None, None) .expect("build BGM request"); assert_eq!( bgm_endpoint, @@ -5505,6 +5838,7 @@ mod tests { prompt, "视频编辑版", Some("stable-video-reference"), + None, ) .expect("build video request"); assert_eq!(video_endpoint, "/api/external/v1/editor/videos/generations"); @@ -5517,7 +5851,7 @@ mod tests { let mut sound_effect = bgm.clone(); sound_effect.edit_kind = LocalProjectResourceEditKind::SoundEffect; let (sound_effect_endpoint, _) = - resource_edit_remote_request(&sound_effect, &source, prompt, "音效编辑版", None) + resource_edit_remote_request(&sound_effect, &source, prompt, "音效编辑版", None, None) .expect("build sound effect request"); assert_eq!( sound_effect_endpoint, @@ -5532,6 +5866,7 @@ mod tests { prompt, "图片编辑版", Some("stable-image-reference"), + None, ) .expect("build image request"); assert_eq!(image_endpoint, "/api/external/v1/editor/images/edits"); @@ -5545,6 +5880,150 @@ mod tests { )); } + #[test] + fn resource_edit_requests_support_create_media_and_character_animation() { + let source = ResourceEditSourceSnapshot { + canonical_resource_id: "local-asset:media-1".to_string(), + source_path: Some("assets/source.mp4".to_string()), + media_type: "video/mp4".to_string(), + asset_kind: "video".to_string(), + source_sha256: "a".repeat(64), + bytes: Some(b"\0\0\0\x18ftypisom".to_vec()), + generation_mode: LocalProjectResourceGenerationMode::Derive, + source_width: Some(720), + source_height: Some(1280), + text: None, + source_asset: None, + source_version: None, + }; + let prompt = "全新的森林场景"; + let canvas_context = ExternalCanvasGenerationContext { + project_id: "project-1".to_string(), + asset_folder_id: "folder-1".to_string(), + canvas_name: "测试画板".to_string(), + }; + + let mut create_video = input( + std::path::Path::new("."), + Uuid::new_v4().to_string(), + LocalProjectResourceEditKind::Video, + "create:video".to_string(), + ); + create_video.generation_mode = LocalProjectResourceGenerationMode::Create; + let (create_video_endpoint, create_video_body) = resource_edit_remote_request( + &create_video, + &source, + prompt, + "新视频", + None, + Some(&canvas_context), + ) + .expect("build create video request"); + assert_eq!( + create_video_endpoint, + "/api/external/v1/editor/videos/generations" + ); + assert!(create_video_body.get("referenceVideoSrcs").is_none()); + assert_eq!( + create_video_body["projectId"], + serde_json::json!("project-1") + ); + assert_eq!( + create_video_body["assetFolderId"], + serde_json::json!("folder-1") + ); + assert!(create_video_body.get("canvasCompletion").is_some()); + + let mut create_sound_effect = create_video.clone(); + create_sound_effect.edit_kind = LocalProjectResourceEditKind::SoundEffect; + let (create_sound_effect_endpoint, create_sound_effect_body) = + resource_edit_remote_request( + &create_sound_effect, + &source, + "清脆的点击声", + "新音效", + None, + Some(&canvas_context), + ) + .expect("build create sound effect request"); + assert_eq!( + create_sound_effect_endpoint, + "/api/external/v1/editor/audios/sound-effects/generations" + ); + assert_eq!( + create_sound_effect_body["projectId"], + serde_json::json!("project-1") + ); + assert!(create_sound_effect_body.get("canvasCompletion").is_some()); + + let mut animation = create_video.clone(); + animation.edit_kind = LocalProjectResourceEditKind::CharacterAnimation; + animation.generation_mode = LocalProjectResourceGenerationMode::Derive; + let (animation_endpoint, animation_body) = resource_edit_remote_request( + &animation, + &source, + "让角色自然呼吸", + "角色动画", + Some("stable-image-object-key"), + None, + ) + .expect("build character animation request"); + assert_eq!( + animation_endpoint, + "/api/external/v1/editor/character-animations/generations" + ); + assert_eq!( + animation_body["sourceImageSrc"], + serde_json::json!("stable-image-object-key") + ); + assert_eq!(animation_body["sourceWidth"], serde_json::json!(720)); + assert_eq!(animation_body["sourceHeight"], serde_json::json!(1280)); + + let generated = serde_json::json!({ + "data": { + "previewVideoPath": "/generated-character-drafts/layer/animation/preview.mp4", + "objectKey": "generated-character-drafts/layer/animation/frame-01.png", + "durationSeconds": 4, + "frames": [ + { + "imageSrc": "generated-character-drafts/layer/animation/frame-01.png", + "objectKey": "generated-character-drafts/layer/animation/frame-01.png", + "assetObjectId": "animation-frame-object", + "width": 720, + "height": 1280 + } + ], + "resource": { + "resourceId": "editor-resource-animation", + "objectKey": "generated-character-drafts/layer/animation/frame-01.png" + } + } + }); + let identity = extract_resource_edit_remote_identity( + &generated, + LocalProjectResourceEditKind::CharacterAnimation, + ) + .expect("extract character animation identity"); + assert_eq!( + identity.legacy_public_path.as_deref(), + Some("/generated-character-drafts/layer/animation/preview.mp4") + ); + assert_eq!( + identity.object_key.as_deref(), + Some("generated-character-drafts/layer/animation/frame-01.png") + ); + assert_eq!(identity.sequence_duration_ms, Some(4_000)); + assert_eq!(identity.sequence_frames.as_ref().map(Vec::len), Some(1)); + assert_eq!( + identity + .sequence_frames + .as_ref() + .and_then(|frames| frames.first()) + .map(|frame| frame.width), + Some(720) + ); + } + #[tokio::test] async fn video_edit_uses_external_upload_generation_poll_and_read_url_without_mutating_source() { diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs index bd9fc0443..9375e913f 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs @@ -29,7 +29,7 @@ pub enum NodeSource { pub enum StageStatus { NoProblem, NeedReview(String), // reason inside - Blocked(String), // reason inside + Blocked(String), // reason inside } #[cfg(test)] diff --git a/apps/ai-game-creator-shell/src/view/project-development/ResourceEditSurface.tsx b/apps/ai-game-creator-shell/src/view/project-development/ResourceEditSurface.tsx index d61075181..bd1a89b66 100644 --- a/apps/ai-game-creator-shell/src/view/project-development/ResourceEditSurface.tsx +++ b/apps/ai-game-creator-shell/src/view/project-development/ResourceEditSurface.tsx @@ -14,6 +14,7 @@ export type ResourceEditSubmitInput = { export function ResourceEditSurface({ resourceLabel, editKind, + generationMode = 'derive', initialAssetName, semanticNotice, onCancel, @@ -21,6 +22,7 @@ export function ResourceEditSurface({ }: { resourceLabel: string; editKind: LocalProjectResourceEditKind; + generationMode?: 'create' | 'derive'; initialAssetName: string; semanticNotice: string | null; onCancel: () => void; @@ -33,6 +35,7 @@ export function ResourceEditSurface({ const [error, setError] = useState(''); const inputLocked = attempted || submitting; const promptMaxLength = resourceEditPromptMaxLength(editKind); + const isCreate = generationMode === 'create'; async function submit(event: FormEvent) { event.preventDefault(); @@ -72,7 +75,7 @@ export function ResourceEditSurface({ 返回资源 - 编辑现有资源 + {isCreate ? '新建媒体资源' : '编辑现有资源'} {resourceLabel} @@ -96,7 +99,11 @@ export function ResourceEditSurface({ rows={7} autoFocus disabled={inputLocked} - placeholder="描述希望在现有资源基础上发生的变化" + placeholder={ + isCreate + ? '描述希望生成的视频、音效或背景音乐' + : '描述希望在现有资源基础上发生的变化' + } onChange={(event) => setPrompt(event.currentTarget.value)} /> @@ -132,7 +139,9 @@ export function ResourceEditSurface({ ? '正在派生…' : editKind === 'version' ? '创建子版本' - : '生成派生资源'} + : isCreate + ? '生成资源' + : '生成派生资源'} )} diff --git a/apps/ai-game-creator-shell/src/view/project-development/index.tsx b/apps/ai-game-creator-shell/src/view/project-development/index.tsx index 411bb5d18..fefac6bc0 100644 --- a/apps/ai-game-creator-shell/src/view/project-development/index.tsx +++ b/apps/ai-game-creator-shell/src/view/project-development/index.tsx @@ -89,7 +89,10 @@ import { ResourceDependencyOverlay } from './ResourceDependencyOverlay'; import { defaultDerivedResourceName, type ProjectResourceEditCapability, + resolveProjectCharacterAnimationCapability, resolveProjectResourceEditCapability, + type ResourceGenerationKind, + resourceGenerationOptions, } from './resourceEditModel'; import { type ResourceEditSubmitInput, @@ -176,8 +179,9 @@ type ResourceEditorRoute = { flowId: string; operationId: string; idempotencyKey: string; - resource: ProjectResource; + resource: ProjectResource | null; capability: Extract; + generationMode: 'create' | 'derive'; }; type UiEditorRoute = { @@ -2324,11 +2328,70 @@ export default function ProjectDevelopmentView({ idempotencyKey: crypto.randomUUID(), resource, capability, + generationMode: 'derive', }); }, [advanceFocusGeneration, normalizeRasterSourceAndOpen, openAssetCanvas], ); + const openCharacterAnimation = useCallback( + (resource: ProjectResource) => { + const capability = resolveProjectCharacterAnimationCapability(resource); + if (!capability) { + return; + } + canvasOpenEpochRef.current += 1; + advanceFocusGeneration(); + const flowId = crypto.randomUUID(); + activeFocusFlowIdRef.current = flowId; + pendingResourceFocusRef.current = null; + setAssetCanvasRoute(null); + setAssetCanvasNotice(''); + setResourceEditorRoute({ + flowId, + operationId: crypto.randomUUID(), + idempotencyKey: crypto.randomUUID(), + resource, + capability, + generationMode: 'derive', + }); + }, + [advanceFocusGeneration], + ); + + const openCreateResource = useCallback( + (kind: ResourceGenerationKind) => { + canvasOpenEpochRef.current += 1; + advanceFocusGeneration(); + const flowId = crypto.randomUUID(); + activeFocusFlowIdRef.current = flowId; + pendingResourceFocusRef.current = null; + setAssetCanvasRoute(null); + setAssetCanvasNotice(''); + const sourceMediaType = kind === 'video' ? 'video/mp4' : 'audio/mpeg'; + setResourceEditorRoute({ + flowId, + operationId: crypto.randomUUID(), + idempotencyKey: crypto.randomUUID(), + resource: null, + capability: { + route: 'derive', + editKind: kind, + sourceMediaType, + semanticNotice: + kind === 'background-music' + ? '将生成全新的背景音乐。' + : kind === 'sound-effect' + ? '将生成全新的音效。' + : '将生成全新的视频。', + generationMode: 'create', + }, + generationMode: 'create', + }); + }, + [advanceFocusGeneration], + ); + const cancelResourceEditor = useCallback(() => { if (resourceEditorRoute) { resourceEditorRevisionRef.current.delete(resourceEditorRoute.operationId); @@ -2690,6 +2753,7 @@ export default function ProjectDevelopmentView({ ); } const resource = route.resource; + const isCreate = route.generationMode === 'create'; const result = await withPlatformSessionRefresh(() => invoke( 'derive_local_project_resource', @@ -2701,16 +2765,27 @@ export default function ProjectDevelopmentView({ operationId: route.operationId, idempotencyKey: route.idempotencyKey, editKind: route.capability.editKind, - sourceResourceId: resource.id, - sourceAssetId: resource.manifestAssetId, + generationMode: route.generationMode, + sourceResourceId: isCreate + ? `create:${route.operationId}` + : (resource?.id ?? ''), + sourceAssetId: isCreate + ? null + : (resource?.manifestAssetId ?? null), sourcePath: - resource.version || resource.subtype === 'agent-result' + isCreate || + resource?.version || + resource?.subtype === 'agent-result' ? null - : resource.path, + : (resource?.path ?? null), sourceMediaType: route.capability.sourceMediaType, - sourceSubtype: resource.subtype, - producerTaskId: resource.producerTaskId, - sourceVersionId: resource.version?.versionId ?? null, + sourceSubtype: isCreate ? null : (resource?.subtype ?? null), + producerTaskId: isCreate + ? null + : (resource?.producerTaskId ?? null), + sourceVersionId: isCreate + ? null + : (resource?.version?.versionId ?? null), prompt, assetName, }, @@ -2759,7 +2834,11 @@ export default function ProjectDevelopmentView({ setResourceEditorRoute(null); setFocusedResourceId(null); setMode('resources'); - setAssetCanvasNotice('派生资源已保存，正在同步资源与布局…'); + setAssetCanvasNotice( + isCreate + ? '生成资源已保存，正在同步资源与布局…' + : '派生资源已保存，正在同步资源与布局…', + ); resourceEditorRevisionRef.current.delete(route.operationId); }, [manifest.projectId, onManifestChange, projectPath, resourceEditorRoute], @@ -3116,14 +3195,16 @@ export default function ProjectDevelopmentView({ : `管理未完成编辑 (${pendingResourceEdits.length})`} ) : null} - + {resourceGenerationOptions.map((option) => ( + + ))} + ) : null}