diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs index c7e48d309..6b8545876 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs @@ -254,11 +254,32 @@ mod tests { #[test] fn next_extract_prompt_contains_previous_rework_notes() { - let note = SeparationNote { - description: "图标".to_string(), - rework_notes: vec!["不要带父背景".to_string()], + let node = SeparationNode { + id: NodeId::new("image").unwrap(), + kind: SeparationNodeKind::ImageTarget, + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 1, + height_px: 1, + note: SeparationNote { + description: "图标".to_string(), + rework_notes: vec!["不要带父背景".to_string()], + }, + children: vec![], + rework_count: 0, }; - let prompt = super::prompt::gen_extract_prompt(vec![note]); + let tree = SeparationTree { + src_ui_design: UIDesignImageId::new("page").unwrap(), + root: node.clone(), + root_extractable: true, + }; + let separation = SeparationState { + schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(), + trees: vec![tree.clone()], + bound: vec![], + problematic_nodes: vec![], + }; + let prompt = super::prompt::gen_extract_prompt(&separation, &tree, &[&node]); assert!(prompt.contains("previous rework notes:\n- 不要带父背景")); } diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs deleted file mode 100644 index a91a94889..000000000 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs +++ /dev/null @@ -1,70 +0,0 @@ -use crate::ui_editor::commands::separation::{SeparationNode, SeparationNote}; - -const SHARED_SEPARATION_REQ: &str = r#" - - MUST hard edges; preserve no glow/blur beyond the exact visible shape. - NEVER keep its parent's background with it. - NEVER include any text unless requested. - - UI elements that needs to extract has been marked with GREEN line frames box with crossline inside. (only for mark purpose, NEVER wrap a frame in your extraction). - On some UI elements, there is some PURPLE filled area, they were removed UI elements, reconstruct the background under where they were. - - MUST extract exactly these marked UI elements area. -"#; -pub(super) fn gen_extract_prompt(separation_notes: Vec) -> String { - let extract_system_prompt = format!( - r#" - This is a UI design image, not a normal photo/illustration. Extract it strictly as UI elements/layers, not as a generic foreground/background extraction. - Treat distinct UI element as its own layer with hard, clean, pixel-accurate edges and full transparency outside the element. - - MUST keep each element at its original position on a transparent canvas. - {SHARED_SEPARATION_REQ} - here are UI elements to extract: - - "# - ); - let mut result = extract_system_prompt; - result.reserve(512); - for elem in separation_notes { - result.push_str(&elem.as_prompt()); - result.push('\n'); - } - result -} -pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String { - let binding_system_prompt = format!( - r#" - You will be given a src UI design image and a processed image, where some ui elements are separated. - You need to recognize and review the separation using the given tool. - field notes: - * extracted_area MUST be the recognized area from the processed image, INSTEAD OF from the src image. - The processed image is the only authoritative image for extracted_area. - Return the pixel bounding box of the extracted element as it appears in the processed image. - Do not copy, infer, or reuse the source node rectangle. - The src image is only for identifying which semantic UI element belongs to to_node. - - Here were the separation requirements: - ``` - {SHARED_SEPARATION_REQ} - ``` - And you should also review if the extracted's successfully meet the src image: - * shape - * color - * style - * edge process - ... - - if not, use `NeedRework` data structure in the tool to indicate the (it should be from which) node id and advice. - your advice (less than 20 words) will be used to improve the separation in the next time. - - these node need handle: - "# - ); - let mut result = binding_system_prompt; - result.reserve(512); - for elem in nodes { - result.push_str(&elem.as_prompt()); - result.push('\n'); - } - result -} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/binding.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/binding.rs new file mode 100644 index 000000000..0f621c248 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/binding.rs @@ -0,0 +1,38 @@ +use crate::ui_editor::commands::separation::SeparationNode; + +pub(crate) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String { + let binding_system_prompt = r#" + You will be given a src UI design image and a processed image, where some ui elements are separated. + You need to recognize and review the separation using the given tool. + field notes: + * extracted_area MUST be the recognized area from the processed image, INSTEAD OF from the src image. + The processed image is the only authoritative image for extracted_area. + Return the pixel bounding box of the extracted element as it appears in the processed image. + Do not copy, infer, or reuse the source node rectangle. + The src image is only for identifying which semantic UI element belongs to to_node. + + Here were the separation requirements: + Preserve hard edges and the exact visible shape. + The processed image is a transparent atlas containing the requested image layers. + Do not use the source node rectangle as the extracted area. + And you should also review if the extracted's successfully meet the src image: + * shape + * color + * style + * edge process + ... + + if not, use `NeedRework` data structure in the tool to indicate the (it should be from which) node id and advice. + your advice (less than 20 words) will be used to improve the separation in the next time. + + these node need handle: + "# + .to_string(); + let mut result = binding_system_prompt; + result.reserve(512); + for elem in nodes { + result.push_str(&elem.as_prompt()); + result.push('\n'); + } + result +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/extract.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/extract.rs new file mode 100644 index 000000000..be839b24b --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/extract.rs @@ -0,0 +1,101 @@ +use crate::ui_editor::commands::separation::{SeparationNode, SeparationNodeKind}; +use std::collections::HashSet; + +use crate::ui_editor::commands::separation::model::{SeparationState, SeparationTree}; +use crate::ui_editor::utils::NodeId; + +pub(crate) fn gen_extract_prompt( + state: &SeparationState, + tree: &SeparationTree, + batch: &[&SeparationNode], +) -> String { + let mut result = r#" + This is a UI design image, not a normal photo/illustration. Extract it strictly as UI elements/layers, not as a generic foreground/background extraction. + Treat distinct UI element as its own layer with hard, clean, pixel-accurate edges and full transparency outside the element. + Generate one transparent atlas at the requested canvas size. You may move or scale output layers so they do not cover one another. + Preserve hard edges and the exact visible shape. Never split a scene/background into multiple scene layers. + Ordinary text is editable UI text: remove it from its parent image/background and do not generate a text raster layer. + Extract only the image nodes marked OUTPUT_THIS_TURN. Reconstruct every child/text layer that is listed under a parent but is not an output target. + + UI layer tree: +"# + .to_string(); + result.reserve(2048); + let target_ids = batch + .iter() + .map(|node| node.id.clone()) + .collect::>(); + let terminal_ids = state + .bound + .iter() + .map(|node| node.node_id.clone()) + .chain( + state + .problematic_nodes + .iter() + .map(|node| node.node_id.clone()), + ) + .collect::>(); + let mut lines = Vec::new(); + let mut index = 1usize; + append_context_lines( + &tree.root, + 0, + true, + &target_ids, + &terminal_ids, + &mut index, + &mut lines, + ); + result.push_str(&lines.join("\n")); + result +} + +fn append_context_lines( + node: &SeparationNode, + depth: usize, + is_root: bool, + target_ids: &HashSet, + terminal_ids: &HashSet, + index: &mut usize, + lines: &mut Vec, +) { + let status = match node.kind { + SeparationNodeKind::TextRemovalOnly => "REMOVE_ONLY", + SeparationNodeKind::PureContainer => "CONTEXT_ONLY", + SeparationNodeKind::ImageTarget => { + if target_ids.contains(&node.id) { + "OUTPUT_THIS_TURN" + } else if terminal_ids.contains(&node.id) { + "DONE" + } else { + "CONTEXT_ONLY" + } + } + }; + let role = if is_root { "root" } else { "child" }; + lines.push(format!( + "{}. [{}] depth={} role={} rect=({}, {}, {}, {}) {}", + *index, + status, + depth, + role, + node.global_pos_x_px, + node.global_pos_y_px, + node.width_px, + node.height_px, + node.note.as_prompt() + )); + *index += 1; + for child in &node.children { + append_context_lines( + child, + depth + 1, + false, + target_ids, + terminal_ids, + index, + lines, + ); + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/mod.rs new file mode 100644 index 000000000..219a1164c --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/mod.rs @@ -0,0 +1,5 @@ +mod binding; +mod extract; + +pub(super) use binding::gen_binding_prompt; +pub(super) use extract::gen_extract_prompt; diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs deleted file mode 100644 index 08f85b7d8..000000000 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs +++ /dev/null @@ -1,711 +0,0 @@ -use super::area::normalize_binding_area; -use super::model::*; -use super::prompt::{gen_binding_prompt, gen_extract_prompt}; -use crate::config::{build_game_creator_llm_client_from_llm_config, load_game_creator_app_config}; -use crate::platform_session::current_platform_session; -use crate::ui_editor::commands::separation::*; -use crate::ui_editor::commands::utils::{ - parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm, - run_with_repair_history, strict_json_schema, -}; -use crate::ui_editor::state::State; -use crate::ui_editor::utils::NodeId; -use base64::Engine as _; -use image::ImageFormat; -use platform_llm::{ - LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice, -}; -use serde::Deserialize; -use std::fs; -use std::path::{Path, PathBuf}; -use std::time::Instant; - -pub fn apply_batch_patch( - state: &mut SeparationState, - tree_index: usize, - decisions: &[BindingDecision], - cut_paths: &std::collections::HashMap, -) -> Result<(), String> { - app_log!( - "ui_separation.batch_patch.start tree_index={} decisions={} cut_paths={}", - tree_index, - decisions.len(), - cut_paths.len() - ); - let batch_nodes = { - let tree = state - .trees - .get(tree_index) - .ok_or_else(|| "separation tree 索引无效".to_string())?; - next_image_batch(state, tree) - .into_iter() - .cloned() - .collect::>() - }; - let batch = batch_nodes.iter().collect::>(); - validate_binding_response( - &BindingResp { - decisions: decisions.to_vec(), - }, - &batch, - )?; - let rework_counts = batch_nodes - .iter() - .map(|node| (node.id.clone(), node.rework_count)) - .collect::>(); - let tree = state.trees.get_mut(tree_index).expect("tree index checked"); - for decision in decisions { - match decision { - BindingDecision::Ok { to_node, .. } => { - let path = cut_paths - .get(to_node) - .ok_or_else(|| format!("缺少节点 {} 的 cut 图片", to_node.as_str()))?; - state.bound.push(BoundNode { - node_id: to_node.clone(), - cut_image_path: path.clone(), - }); - } - BindingDecision::NeedRework { - to_node, - advice: problem_description, - } => { - append_rework_note(&mut tree.root, to_node, problem_description); - let count = rework_counts.get(to_node).copied().unwrap_or(0) + 1; - increment_rework_count(&mut tree.root, to_node, count); - if count >= MAX_REWORK_COUNT { - state.problematic_nodes.push(ProblematicNode { - node_id: to_node.clone(), - problem_description: problem_description.clone(), - rework_count: count, - }); - } - } - } - } - app_log!( - "ui_separation.batch_patch.completed tree_index={} bound={} problematic={} pending_root_children={}", - tree_index, - state.bound.len(), - state.problematic_nodes.len(), - tree.root.children.len() - ); - Ok(()) -} - -fn append_rework_note(node: &mut SeparationNode, id: &NodeId, note: &str) -> bool { - if node.id == *id { - node.note.rework_notes.push(note.to_string()); - return true; - } - node.children - .iter_mut() - .any(|child| append_rework_note(child, id, note)) -} - -fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) { - if node.id == *id { - node.rework_count = count; - return; - } - for child in &mut node.children { - increment_rework_count(child, id, count); - } -} - -#[derive(Deserialize)] -struct RawEditResponse { - data: Vec, -} -#[derive(Deserialize)] -struct RawEditItem { - b64_json: String, -} - -async fn raw_image_edit( - session: &crate::platform_session::PlatformSessionSnapshot, - image_data_url: &str, - prompt: &str, - width: u32, - height: u32, -) -> Result { - let started = Instant::now(); - let result = raw_image_edit_inner(session, image_data_url, prompt, width, height).await; - app_log!( - "ui_separation.image_edit.timing outcome={} elapsed_ms={} width={} height={}", - if result.is_ok() { "ok" } else { "error" }, - started.elapsed().as_millis(), - width, - height - ); - result -} - -async fn raw_image_edit_inner( - session: &crate::platform_session::PlatformSessionSnapshot, - image_data_url: &str, - prompt: &str, - width: u32, - height: u32, -) -> Result { - app_log!( - "ui_separation.image_edit.start width={} height={} prompt_chars={}", - width, - height, - prompt.chars().count() - ); - let (mime, data) = image_data_url - .split_once(",") - .ok_or_else(|| "界面图 data URL 无效".to_string())?; - let mime = mime - .strip_prefix("data:") - .and_then(|v| v.strip_suffix(";base64")) - .unwrap_or("image/png"); - if !mime.eq_ignore_ascii_case("image/png") { - return Err("图片分离请求只支持 PNG 源图".to_string()); - } - let image_bytes = base64::engine::general_purpose::STANDARD - .decode(data.trim()) - .map_err(|error| format!("解码源图失败:{error}"))?; - if image_bytes.is_empty() { - return Err("源图不能为空".to_string()); - } - let client = crate::http_client::agc_main_site_client_builder() - .build() - .map_err(|e| format!("创建图片编辑客户端失败:{e}"))?; - let url = format!( - "{}/api/raw/v1/images/edit", - session.api_base_url.trim_end_matches('/') - ); - let image_part = reqwest::multipart::Part::bytes(image_bytes) - .file_name("image.png") - .mime_str("image/png") - .map_err(|error| format!("构造图片编辑文件部件失败:{error}"))?; - let body = reqwest::multipart::Form::new() - .part("image", image_part) - .text("prompt", prompt.to_string()) - .text("width", width.to_string()) - .text("height", height.to_string()) - .text("output_format", "png") - .text("background", "transparent"); - let response = crate::http_client::with_agc_main_site_marker( - client - .post(url) - .bearer_auth(&session.access_token) - .multipart(body), - ) - .send() - .await - .map_err(|e| { - app_log!("ui_separation.error stage=image_edit reason=send error={e}"); - format!("图片分离请求失败:{e}") - })?; - if !response.status().is_success() { - app_log!( - "ui_separation.error stage=image_edit reason=http_status status={}", - response.status() - ); - return Err(format!("图片分离请求失败(HTTP {})", response.status())); - } - let payload = response.json::().await.map_err(|e| { - app_log!("ui_separation.error stage=image_edit reason=parse_response error={e}"); - format!("解析图片分离响应失败:{e}") - })?; - let result = payload - .data - .into_iter() - .next() - .map(|item| format!("data:image/png;base64,{}", item.b64_json)) - .ok_or_else(|| "图片分离响应没有图像".to_string()); - match &result { - Ok(value) => app_log!( - "ui_separation.image_edit.completed data_url_chars={}", - value.chars().count() - ), - Err(error) => { - app_log!("ui_separation.error stage=image_edit reason=empty_result error={error}") - } - } - result -} - -async fn write_processed_image(processed_url: String, target: PathBuf) -> Result<(), String> { - let started = Instant::now(); - let result = write_processed_image_inner(processed_url, target).await; - app_log!( - "ui_separation.processed_image.write.timing outcome={} elapsed_ms={}", - if result.is_ok() { "ok" } else { "error" }, - started.elapsed().as_millis() - ); - result -} - -async fn write_processed_image_inner(processed_url: String, target: PathBuf) -> Result<(), String> { - app_log!( - "ui_separation.processed_image.write.start target_file={} data_url_chars={}", - target - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or(""), - processed_url.chars().count() - ); - tokio::task::spawn_blocking(move || { - let encoded = processed_url - .split_once(',') - .map(|(_, data)| data) - .ok_or_else(|| "处理图 data URL 无效".to_string())?; - let processed_bytes = base64::engine::general_purpose::STANDARD - .decode(encoded) - .map_err(|error| format!("解析处理图失败:{error}"))?; - let byte_len = processed_bytes.len(); - fs::write(&target, processed_bytes) - .map_err(|error| format!("写入处理图失败:{}: {error}", target.display())) - .map(|_| { - app_log!( - "ui_separation.processed_image.write.completed target_file={} bytes={}", - target - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or(""), - byte_len - ); - }) - }) - .await - .map_err(|error| format!("写入处理图任务失败:{error}"))? -} - -async fn visual_binding( - source_url: String, - processed_url: String, - nodes: &[&SeparationNode], -) -> Result { - let started = Instant::now(); - let result = visual_binding_inner(source_url, processed_url, nodes).await; - app_log!( - "ui_separation.visual_binding.timing outcome={} elapsed_ms={} nodes={}", - if result.is_ok() { "ok" } else { "error" }, - started.elapsed().as_millis(), - nodes.len() - ); - result -} - -async fn visual_binding_inner( - source_url: String, - processed_url: String, - nodes: &[&SeparationNode], -) -> Result { - app_log!( - "ui_separation.visual_binding.start nodes={} source_url_chars={} processed_url_chars={}", - nodes.len(), - source_url.chars().count(), - processed_url.chars().count() - ); - let llm_config = load_game_creator_app_config() - .map_err(|e| { - app_log!("ui_separation.error stage=visual_binding reason=load_config error={e}"); - e.to_string() - })? - .llm; - - let client = - build_game_creator_llm_client_from_llm_config(&llm_config, "llm").map_err(|e| { - app_log!("ui_separation.error stage=visual_binding reason=build_client error={e}"); - e.to_string() - })?; - let schema = strict_json_schema::().map_err(|error| { - app_log!("ui_separation.error stage=visual_binding reason=build_schema error={error}"); - error - })?; - let tool = LlmFunctionTool::new( - "bind_ui_elements", - "确认处理图中的区域对应哪些 UI 节点", - schema, - ) - .with_strict(true); - let initial_history = vec![ - LlmMessage::system(gen_binding_prompt(nodes.to_vec())), - LlmMessage::user_multimodal(vec![ - LlmMessageContentPart::InputText { - text: "processed image:".to_string(), - }, - LlmMessageContentPart::InputImage { - image_url: processed_url.clone(), - }, - LlmMessageContentPart::InputText { - text: "src image:".to_string(), - }, - LlmMessageContentPart::InputImage { - image_url: source_url.clone(), - }, - ]), - ]; - let result = run_with_repair_history( - 2, - initial_history, - |history| { - let tool = tool.clone(); - let client = client.clone(); - let llm_config = llm_config.clone(); - async move { - let request = LlmRunRequest::new(history) - .with_function_tools(vec![tool.clone()]) - .with_tool_choice(LlmToolChoice::Required); - let request_started = Instant::now(); - let response = request_ui_editor_llm(&client, &llm_config, request).await; - app_log!( - "ui_separation.llm.timing outcome={} elapsed_ms={}", - if response.is_ok() { "ok" } else { "error" }, - request_started.elapsed().as_millis() - ); - response - .map_err(|e| e.to_string()) - .and_then(|response| { - response - .tool_calls - .into_iter() - .find(|call| call.name == "bind_ui_elements") - .map(|call| call.arguments) - .ok_or_else(|| "视觉绑定模型未返回工具调用".to_string()) - }) - .and_then(|arguments| parse_limited_llm_tool_arguments(&arguments)) - .and_then(|args| { - serde_json::from_value::(args) - .map_err(|e| format!("视觉绑定结果无效:{e}")) - }) - .and_then(|parsed| Ok(parsed)) - } - }, - |value: &BindingResp| validate_binding_response(value, nodes), - ) - .await; - match &result { - Ok(value) => app_log!( - "ui_separation.visual_binding.completed nodes={} decisions={}", - nodes.len(), - value.decisions.len() - ), - Err(error) => app_log!( - "ui_separation.error stage=visual_binding reason=failed nodes={} error={error}", - nodes.len() - ), - } - result -} - -pub(crate) async fn separate_ui_impl( - project_path: String, - asset_id: String, - state: State, -) -> Result { - app_log!( - "ui_separation.start asset_id={} ui_trees={} ui_images={} sprites={}", - asset_id, - state.ui_trees.len(), - state.ui_design_images.len(), - state.sprite_assets.len() - ); - let session = current_platform_session().ok_or_else(|| "请先登录平台账号".to_string())?; - let root = Path::new(project_path.trim()); - let sidecar = separation_sidecar_dir(root, &asset_id).map_err(|error| { - app_log!( - "ui_separation.error stage=sidecar_dir asset_id={} error={error}", - asset_id - ); - error - })?; - fs::create_dir_all(&sidecar).map_err(|e| { - app_log!( - "ui_separation.error stage=sidecar_create asset_id={} error={e}", - asset_id - ); - format!("创建 separation sidecar 失败:{e}") - })?; - let state_path = separation_state_path(root, &asset_id)?; - let restored = state_path.exists(); - let mut separation = if restored { - app_log!("ui_separation.state_restore.start asset_id={}", asset_id); - read_separation_state(&state_path).map_err(|error| { - app_log!( - "ui_separation.error stage=state_restore asset_id={} error={error}", - asset_id - ); - error - })? - } else { - app_log!("ui_separation.state_construct.start asset_id={}", asset_id); - construct_separation_state(&state) - }; - app_log!( - "ui_separation.state_ready asset_id={} restored={} trees={} bound={} problematic={}", - asset_id, - restored, - separation.trees.len(), - separation.bound.len(), - separation.problematic_nodes.len() - ); - for tree_index in 0..separation.trees.len() { - let tree = &separation.trees[tree_index]; - let image_id = tree.src_ui_design.clone(); - let image = state - .ui_design_images - .get(&image_id) - .ok_or_else(|| "缺少源界面图".to_string())?; - let source_path = crate::project::resolve_local_project_path(root, &image.path)?; - let source_url = read_ui_reference_image_data_url(source_path) - .await - .map_err(|error| { - app_log!( - "ui_separation.error stage=read_source tree_index={} image_id={} error={error}", - tree_index, - image_id.as_str() - ); - error - })?; - app_log!( - "ui_separation.tree_start tree_index={} image_id={} width={} height={}", - tree_index, - image_id.as_str(), - image.pixel_size.x.round() as u32, - image.pixel_size.y.round() as u32 - ); - write_separation_state(&state_path, &separation).map_err(|error| { - app_log!( - "ui_separation.error stage=state_checkpoint tree_index={} error={error}", - tree_index - ); - error - })?; - let mut batch_index = 0usize; - loop { - let batch_started = Instant::now(); - let Some(current_tree) = separation.trees.get(tree_index) else { - break; - }; - let batch_nodes = next_image_batch(&separation, current_tree) - .into_iter() - .cloned() - .collect::>(); - if batch_nodes.is_empty() { - app_log!( - "ui_separation.tree_completed tree_index={} image_id={} bound={} problematic={}", - tree_index, - image_id.as_str(), - separation.bound.len(), - separation.problematic_nodes.len() - ); - break; - } - let batch = batch_nodes.iter().collect::>(); - let prompt = gen_extract_prompt(&separation, current_tree, &batch); - app_log!( - "ui_separation.batch_start tree_index={} batch_index={} nodes={} prompt_chars={} rework_total={}", - tree_index, - batch_index, - batch.len(), - prompt.chars().count(), - batch.iter().map(|node| node.rework_count).sum::() - ); - let processed_url = match raw_image_edit( - &session, - &source_url, - &prompt, - image.pixel_size.x as u32, - image.pixel_size.y as u32, - ) - .await - { - Ok(value) => value, - Err(error) => { - app_log!( - "ui_separation.error stage=image_edit tree_index={} batch_index={} error={error}", - tree_index, - batch_index - ); - write_separation_state(&state_path, &separation)?; - return Err(error); - } - }; - let processed_path = sidecar.join(format!("processed-{}.png", separation.bound.len())); - if let Err(error) = - write_processed_image(processed_url.clone(), processed_path.clone()).await - { - app_log!( - "ui_separation.error stage=processed_image_write tree_index={} batch_index={} error={error}", - tree_index, - batch_index - ); - write_separation_state(&state_path, &separation)?; - return Err(error); - } - let binding = match visual_binding(source_url.clone(), processed_url, &batch).await { - Ok(value) => value, - Err(error) => { - app_log!( - "ui_separation.error stage=visual_binding tree_index={} batch_index={} error={error}", - tree_index, - batch_index - ); - write_separation_state(&state_path, &separation)?; - return Err(error); - } - }; - app_log!( - "ui_separation.binding_decisions tree_index={} batch_index={} decisions={}", - tree_index, - batch_index, - binding.decisions.len() - ); - let mut cut_paths = std::collections::HashMap::new(); - let mut cut_error = None; - for decision in &binding.decisions { - if let BindingDecision::Ok { - to_node, - extracted_area, - } = decision - { - let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str())); - match cut_processed_image( - processed_path.clone(), - *extracted_area, - cut_path.clone(), - ) - .await - { - Ok(()) => { - cut_paths - .insert(to_node.clone(), project_relative_path(root, &cut_path)?); - } - Err(error) => { - app_log!( - "ui_separation.error stage=cut_image tree_index={} batch_index={} node_id={} error={error}", - tree_index, - batch_index, - to_node.as_str() - ); - cut_error = - Some(format!("节点 {} 的分离区域无效:{error}", to_node.as_str())); - break; - } - } - } - } - if let Some(error) = cut_error { - app_log!( - "ui_separation.error stage=cut_batch tree_index={} batch_index={} error={error}", - tree_index, - batch_index - ); - write_separation_state(&state_path, &separation)?; - return Err(error); - } - apply_batch_patch(&mut separation, tree_index, &binding.decisions, &cut_paths)?; - write_separation_state(&state_path, &separation)?; - app_log!( - "ui_separation.batch_completed tree_index={} batch_index={} cuts={} bound={} problematic={} elapsed_ms={}", - tree_index, - batch_index, - cut_paths.len(), - separation.bound.len(), - separation.problematic_nodes.len(), - batch_started.elapsed().as_millis() - ); - batch_index += 1; - } - } - app_log!( - "ui_separation.completed asset_id={} bound_nodes={} problematic_nodes={}", - asset_id, - separation.bound.len(), - separation.problematic_nodes.len() - ); - Ok(separation_dto(&separation)) -} - -async fn cut_processed_image( - source: PathBuf, - area: BindingArea, - target: PathBuf, -) -> Result<(), String> { - let started = Instant::now(); - let result = cut_processed_image_inner(source, area, target).await; - app_log!( - "ui_separation.cut_image.timing outcome={} elapsed_ms={}", - if result.is_ok() { "ok" } else { "error" }, - started.elapsed().as_millis() - ); - result -} - -async fn cut_processed_image_inner( - source: PathBuf, - area: BindingArea, - target: PathBuf, -) -> Result<(), String> { - app_log!( - "ui_separation.cut_image.start source_file={} target_file={} area=({}, {}, {}, {})", - source - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or(""), - target - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or(""), - area.global_pos_x_px, - area.global_pos_y_px, - area.width_px, - area.height_px - ); - tokio::task::spawn_blocking(move || cut_processed_image_blocking(&source, &area, &target)) - .await - .map_err(|error| format!("裁切处理图任务失败:{error}"))? -} - -fn cut_processed_image_blocking( - source: &Path, - area: &BindingArea, - target: &Path, -) -> Result<(), String> { - let image = image::open(source) - .map_err(|e| format!("读取处理图失败:{e}"))? - .to_rgba8(); - let normalized = normalize_binding_area(&image, *area)?; - let original_area = *area; - let normalized_area = normalized.area; - app_log!( - "ui_separation.cut_image.normalized changed={} clamped={} transparent={} original_area=({}, {}, {}, {}) normalized_area=({}, {}, {}, {})", - normalized.changed, - normalized.clamped, - normalized.transparent, - original_area.global_pos_x_px, - original_area.global_pos_y_px, - original_area.width_px, - original_area.height_px, - normalized_area.global_pos_x_px, - normalized_area.global_pos_y_px, - normalized_area.width_px, - normalized_area.height_px - ); - let cropped = image::imageops::crop_imm( - &image, - normalized_area.global_pos_x_px, - normalized_area.global_pos_y_px, - normalized_area.width_px, - normalized_area.height_px, - ) - .to_image(); - cropped - .save_with_format(target, ImageFormat::Png) - .map_err(|e| format!("写入 cut 图片失败:{e}"))?; - app_log!( - "ui_separation.cut_image.completed target_file={} width={} height={}", - target - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or(""), - normalized_area.width_px, - normalized_area.height_px - ); - Ok(()) -} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/binding.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/binding.rs new file mode 100644 index 000000000..a2c614930 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/binding.rs @@ -0,0 +1,130 @@ +use crate::config::{build_game_creator_llm_client_from_llm_config, load_game_creator_app_config}; +use crate::ui_editor::commands::separation::prompt::gen_binding_prompt; +use crate::ui_editor::commands::separation::{ + validate_binding_response, BindingResp, SeparationNode, +}; +use crate::ui_editor::commands::utils::{ + parse_limited_llm_tool_arguments, request_ui_editor_llm, run_with_repair_history, + strict_json_schema, +}; +use platform_llm::{ + LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice, +}; +use std::time::Instant; + +pub(super) async fn visual_binding( + source_url: String, + processed_url: String, + nodes: &[&SeparationNode], +) -> Result { + let started = Instant::now(); + let result = visual_binding_inner(source_url, processed_url, nodes).await; + app_log!( + "ui_separation.visual_binding.timing outcome={} elapsed_ms={} nodes={}", + if result.is_ok() { "ok" } else { "error" }, + started.elapsed().as_millis(), + nodes.len() + ); + result +} + +async fn visual_binding_inner( + source_url: String, + processed_url: String, + nodes: &[&SeparationNode], +) -> Result { + app_log!( + "ui_separation.visual_binding.start nodes={} source_url_chars={} processed_url_chars={}", + nodes.len(), + source_url.chars().count(), + processed_url.chars().count() + ); + let llm_config = load_game_creator_app_config() + .map_err(|e| { + app_log!("ui_separation.error stage=visual_binding reason=load_config error={e}"); + e.to_string() + })? + .llm; + let client = + build_game_creator_llm_client_from_llm_config(&llm_config, "llm").map_err(|e| { + app_log!("ui_separation.error stage=visual_binding reason=build_client error={e}"); + e.to_string() + })?; + let schema = strict_json_schema::().map_err(|error| { + app_log!("ui_separation.error stage=visual_binding reason=build_schema error={error}"); + error + })?; + let tool = LlmFunctionTool::new( + "bind_ui_elements", + "确认处理图中的区域对应哪些 UI 节点", + schema, + ) + .with_strict(true); + let initial_history = vec![ + LlmMessage::system(gen_binding_prompt(nodes.to_vec())), + LlmMessage::user_multimodal(vec![ + LlmMessageContentPart::InputText { + text: "processed image:".to_string(), + }, + LlmMessageContentPart::InputImage { + image_url: processed_url.clone(), + }, + LlmMessageContentPart::InputText { + text: "src image:".to_string(), + }, + LlmMessageContentPart::InputImage { + image_url: source_url.clone(), + }, + ]), + ]; + let result = run_with_repair_history( + 2, + initial_history, + |history| { + let tool = tool.clone(); + let client = client.clone(); + let llm_config = llm_config.clone(); + async move { + let request = LlmRunRequest::new(history) + .with_function_tools(vec![tool.clone()]) + .with_tool_choice(LlmToolChoice::Required); + let request_started = Instant::now(); + let response = request_ui_editor_llm(&client, &llm_config, request).await; + app_log!( + "ui_separation.llm.timing outcome={} elapsed_ms={}", + if response.is_ok() { "ok" } else { "error" }, + request_started.elapsed().as_millis() + ); + response + .map_err(|e| e.to_string()) + .and_then(|response| { + response + .tool_calls + .into_iter() + .find(|call| call.name == "bind_ui_elements") + .map(|call| call.arguments) + .ok_or_else(|| "视觉绑定模型未返回工具调用".to_string()) + }) + .and_then(|arguments| parse_limited_llm_tool_arguments(&arguments)) + .and_then(|args| { + serde_json::from_value::(args) + .map_err(|e| format!("视觉绑定结果无效:{e}")) + }) + } + }, + |value: &BindingResp| validate_binding_response(value, nodes), + ) + .await; + match &result { + Ok(value) => app_log!( + "ui_separation.visual_binding.completed nodes={} decisions={}", + nodes.len(), + value.decisions.len() + ), + Err(error) => app_log!( + "ui_separation.error stage=visual_binding reason=failed nodes={} error={error}", + nodes.len() + ), + } + result +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/cut.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/cut.rs new file mode 100644 index 000000000..8b083c8b5 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/cut.rs @@ -0,0 +1,93 @@ +use crate::ui_editor::commands::separation::area::normalize_binding_area; +use crate::ui_editor::commands::separation::model::BindingArea; +use image::ImageFormat; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +pub(super) async fn cut_processed_image( + source: PathBuf, + area: BindingArea, + target: PathBuf, +) -> Result<(), String> { + let started = Instant::now(); + let result = cut_processed_image_inner(source, area, target).await; + app_log!( + "ui_separation.cut_image.timing outcome={} elapsed_ms={}", + if result.is_ok() { "ok" } else { "error" }, + started.elapsed().as_millis() + ); + result +} + +async fn cut_processed_image_inner( + source: PathBuf, + area: BindingArea, + target: PathBuf, +) -> Result<(), String> { + app_log!( + "ui_separation.cut_image.start source_file={} target_file={} area=({}, {}, {}, {})", + source + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + area.global_pos_x_px, + area.global_pos_y_px, + area.width_px, + area.height_px + ); + tokio::task::spawn_blocking(move || cut_processed_image_blocking(&source, &area, &target)) + .await + .map_err(|error| format!("裁切处理图任务失败:{error}"))? +} + +fn cut_processed_image_blocking( + source: &Path, + area: &BindingArea, + target: &Path, +) -> Result<(), String> { + let image = image::open(source) + .map_err(|e| format!("读取处理图失败:{e}"))? + .to_rgba8(); + let normalized = normalize_binding_area(&image, *area)?; + let original_area = *area; + let normalized_area = normalized.area; + app_log!( + "ui_separation.cut_image.normalized changed={} clamped={} transparent={} original_area=({}, {}, {}, {}) normalized_area=({}, {}, {}, {})", + normalized.changed, + normalized.clamped, + normalized.transparent, + original_area.global_pos_x_px, + original_area.global_pos_y_px, + original_area.width_px, + original_area.height_px, + normalized_area.global_pos_x_px, + normalized_area.global_pos_y_px, + normalized_area.width_px, + normalized_area.height_px + ); + let cropped = image::imageops::crop_imm( + &image, + normalized_area.global_pos_x_px, + normalized_area.global_pos_y_px, + normalized_area.width_px, + normalized_area.height_px, + ) + .to_image(); + cropped + .save_with_format(target, ImageFormat::Png) + .map_err(|e| format!("写入 cut 图片失败:{e}"))?; + app_log!( + "ui_separation.cut_image.completed target_file={} width={} height={}", + target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + normalized_area.width_px, + normalized_area.height_px + ); + Ok(()) +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/image_edit.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/image_edit.rs new file mode 100644 index 000000000..88a48d5f7 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/image_edit.rs @@ -0,0 +1,172 @@ +use crate::platform_session::PlatformSessionSnapshot; +use base64::Engine as _; +use serde::Deserialize; +use std::fs; +use std::path::PathBuf; +use std::time::Instant; + +#[derive(Deserialize)] +struct RawEditResponse { + data: Vec, +} + +#[derive(Deserialize)] +struct RawEditItem { + b64_json: String, +} + +pub(super) async fn raw_image_edit( + session: &PlatformSessionSnapshot, + image_data_url: &str, + prompt: &str, + width: u32, + height: u32, +) -> Result { + let started = Instant::now(); + let result = raw_image_edit_inner(session, image_data_url, prompt, width, height).await; + app_log!( + "ui_separation.image_edit.timing outcome={} elapsed_ms={} width={} height={}", + if result.is_ok() { "ok" } else { "error" }, + started.elapsed().as_millis(), + width, + height + ); + result +} + +async fn raw_image_edit_inner( + session: &PlatformSessionSnapshot, + image_data_url: &str, + prompt: &str, + width: u32, + height: u32, +) -> Result { + app_log!( + "ui_separation.image_edit.start width={} height={} prompt_chars={}", + width, + height, + prompt.chars().count() + ); + let (mime, data) = image_data_url + .split_once(',') + .ok_or_else(|| "界面图 data URL 无效".to_string())?; + let mime = mime + .strip_prefix("data:") + .and_then(|value| value.strip_suffix(";base64")) + .unwrap_or("image/png"); + if !mime.eq_ignore_ascii_case("image/png") { + return Err("图片分离请求只支持 PNG 源图".to_string()); + } + let image_bytes = base64::engine::general_purpose::STANDARD + .decode(data.trim()) + .map_err(|error| format!("解码源图失败:{error}"))?; + if image_bytes.is_empty() { + return Err("源图不能为空".to_string()); + } + let client = crate::http_client::agc_main_site_client_builder() + .build() + .map_err(|error| format!("创建图片编辑客户端失败:{error}"))?; + let url = format!( + "{}/api/raw/v1/images/edit", + session.api_base_url.trim_end_matches('/') + ); + let image_part = reqwest::multipart::Part::bytes(image_bytes) + .file_name("image.png") + .mime_str("image/png") + .map_err(|error| format!("构造图片编辑文件部件失败:{error}"))?; + let body = reqwest::multipart::Form::new() + .part("image", image_part) + .text("prompt", prompt.to_string()) + .text("width", width.to_string()) + .text("height", height.to_string()) + .text("output_format", "png") + .text("background", "transparent"); + let response = crate::http_client::with_agc_main_site_marker( + client + .post(url) + .bearer_auth(&session.access_token) + .multipart(body), + ) + .send() + .await + .map_err(|error| { + app_log!("ui_separation.error stage=image_edit reason=send error={error}"); + format!("图片分离请求失败:{error}") + })?; + if !response.status().is_success() { + app_log!( + "ui_separation.error stage=image_edit reason=http_status status={}", + response.status() + ); + return Err(format!("图片分离请求失败(HTTP {})", response.status())); + } + let payload = response.json::().await.map_err(|error| { + app_log!("ui_separation.error stage=image_edit reason=parse_response error={error}"); + format!("解析图片分离响应失败:{error}") + })?; + let result = payload + .data + .into_iter() + .next() + .map(|item| format!("data:image/png;base64,{}", item.b64_json)) + .ok_or_else(|| "图片分离响应没有图像".to_string()); + match &result { + Ok(value) => app_log!( + "ui_separation.image_edit.completed data_url_chars={}", + value.chars().count() + ), + Err(error) => { + app_log!("ui_separation.error stage=image_edit reason=empty_result error={error}") + } + } + result +} + +pub(super) async fn write_processed_image( + processed_url: String, + target: PathBuf, +) -> Result<(), String> { + let started = Instant::now(); + let result = write_processed_image_inner(processed_url, target).await; + app_log!( + "ui_separation.processed_image.write.timing outcome={} elapsed_ms={}", + if result.is_ok() { "ok" } else { "error" }, + started.elapsed().as_millis() + ); + result +} + +async fn write_processed_image_inner(processed_url: String, target: PathBuf) -> Result<(), String> { + app_log!( + "ui_separation.processed_image.write.start target_file={} data_url_chars={}", + target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + processed_url.chars().count() + ); + tokio::task::spawn_blocking(move || { + let encoded = processed_url + .split_once(',') + .map(|(_, data)| data) + .ok_or_else(|| "处理图 data URL 无效".to_string())?; + let processed_bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|error| format!("解析处理图失败:{error}"))?; + let byte_len = processed_bytes.len(); + fs::write(&target, processed_bytes) + .map_err(|error| format!("写入处理图失败:{}: {error}", target.display())) + .map(|_| { + app_log!( + "ui_separation.processed_image.write.completed target_file={} bytes={}", + target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""), + byte_len + ); + }) + }) + .await + .map_err(|error| format!("写入处理图任务失败:{error}"))? +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/mod.rs new file mode 100644 index 000000000..0a9a5c652 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/mod.rs @@ -0,0 +1,222 @@ +mod binding; +mod cut; +mod image_edit; +mod patch; + +pub use patch::apply_batch_patch; + +use super::model::*; +use super::persistence::{ + project_relative_path, read_separation_state, separation_dto, separation_sidecar_dir, + separation_state_path, write_separation_state, +}; +use super::prompt::gen_extract_prompt; +use super::tree::next_image_batch; +use crate::platform_session::current_platform_session; +use crate::ui_editor::commands::utils::read_ui_reference_image_data_url; +use crate::ui_editor::state::State; +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use std::time::Instant; + +pub(crate) async fn separate_ui_impl( + project_path: String, + asset_id: String, + state: State, +) -> Result { + app_log!( + "ui_separation.start asset_id={} ui_trees={} ui_images={} sprites={}", + asset_id, + state.ui_trees.len(), + state.ui_design_images.len(), + state.sprite_assets.len() + ); + let session = current_platform_session().ok_or_else(|| "请先登录平台账号".to_string())?; + let root = Path::new(project_path.trim()); + let sidecar = separation_sidecar_dir(root, &asset_id).map_err(|error| { + app_log!( + "ui_separation.error stage=sidecar_dir asset_id={} error={error}", + asset_id + ); + error + })?; + fs::create_dir_all(&sidecar).map_err(|error| { + app_log!( + "ui_separation.error stage=sidecar_create asset_id={} error={error}", + asset_id + ); + format!("创建 separation sidecar 失败:{error}") + })?; + let state_path = separation_state_path(root, &asset_id)?; + let restored = state_path.exists(); + let mut separation = if restored { + app_log!("ui_separation.state_restore.start asset_id={asset_id}"); + read_separation_state(&state_path).map_err(|error| { + app_log!( + "ui_separation.error stage=state_restore asset_id={} error={error}", + asset_id + ); + error + })? + } else { + app_log!("ui_separation.state_construct.start asset_id={asset_id}"); + super::tree::construct_separation_state(&state) + }; + app_log!( + "ui_separation.state_ready asset_id={} restored={} trees={} bound={} problematic={}", + asset_id, + restored, + separation.trees.len(), + separation.bound.len(), + separation.problematic_nodes.len() + ); + + for tree_index in 0..separation.trees.len() { + let tree = &separation.trees[tree_index]; + let image_id = tree.src_ui_design.clone(); + let image = state + .ui_design_images + .get(&image_id) + .ok_or_else(|| "缺少源界面图".to_string())?; + let source_path = crate::project::resolve_local_project_path(root, &image.path)?; + let source_url = read_ui_reference_image_data_url(source_path) + .await + .map_err(|error| { + app_log!( + "ui_separation.error stage=read_source tree_index={} image_id={} error={error}", + tree_index, + image_id.as_str() + ); + error + })?; + app_log!( + "ui_separation.tree_start tree_index={} image_id={} width={} height={}", + tree_index, + image_id.as_str(), + image.pixel_size.x.round() as u32, + image.pixel_size.y.round() as u32 + ); + write_separation_state(&state_path, &separation).map_err(|error| { + app_log!( + "ui_separation.error stage=state_checkpoint tree_index={} error={error}", + tree_index + ); + error + })?; + let mut batch_index = 0usize; + loop { + let batch_started = Instant::now(); + let Some(current_tree) = separation.trees.get(tree_index) else { + break; + }; + let batch_nodes = next_image_batch(&separation, current_tree) + .into_iter() + .cloned() + .collect::>(); + if batch_nodes.is_empty() { + app_log!( + "ui_separation.tree_completed tree_index={} image_id={} bound={} problematic={}", + tree_index, image_id.as_str(), separation.bound.len(), separation.problematic_nodes.len() + ); + break; + } + let batch = batch_nodes.iter().collect::>(); + let prompt = gen_extract_prompt(&separation, current_tree, &batch); + app_log!( + "ui_separation.batch_start tree_index={} batch_index={} nodes={} prompt_chars={} rework_total={}", + tree_index, batch_index, batch.len(), prompt.chars().count(), + batch.iter().map(|node| node.rework_count).sum::() + ); + let processed_url = match image_edit::raw_image_edit( + &session, + &source_url, + &prompt, + image.pixel_size.x as u32, + image.pixel_size.y as u32, + ) + .await + { + Ok(value) => value, + Err(error) => { + app_log!("ui_separation.error stage=image_edit tree_index={} batch_index={} error={error}", tree_index, batch_index); + write_separation_state(&state_path, &separation)?; + return Err(error); + } + }; + let processed_path = sidecar.join(format!("processed-{}.png", separation.bound.len())); + if let Err(error) = + image_edit::write_processed_image(processed_url.clone(), processed_path.clone()) + .await + { + app_log!("ui_separation.error stage=processed_image_write tree_index={} batch_index={} error={error}", tree_index, batch_index); + write_separation_state(&state_path, &separation)?; + return Err(error); + } + let binding = match binding::visual_binding(source_url.clone(), processed_url, &batch) + .await + { + Ok(value) => value, + Err(error) => { + app_log!("ui_separation.error stage=visual_binding tree_index={} batch_index={} error={error}", tree_index, batch_index); + write_separation_state(&state_path, &separation)?; + return Err(error); + } + }; + app_log!( + "ui_separation.binding_decisions tree_index={} batch_index={} decisions={}", + tree_index, + batch_index, + binding.decisions.len() + ); + let mut cut_paths = HashMap::new(); + let mut cut_error = None; + for decision in &binding.decisions { + if let BindingDecision::Ok { + to_node, + extracted_area, + } = decision + { + let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str())); + match cut::cut_processed_image( + processed_path.clone(), + *extracted_area, + cut_path.clone(), + ) + .await + { + Ok(()) => { + cut_paths + .insert(to_node.clone(), project_relative_path(root, &cut_path)?); + } + Err(error) => { + app_log!("ui_separation.error stage=cut_image tree_index={} batch_index={} node_id={} error={error}", tree_index, batch_index, to_node.as_str()); + cut_error = + Some(format!("节点 {} 的分离区域无效:{error}", to_node.as_str())); + break; + } + } + } + } + if let Some(error) = cut_error { + app_log!("ui_separation.error stage=cut_batch tree_index={} batch_index={} error={error}", tree_index, batch_index); + write_separation_state(&state_path, &separation)?; + return Err(error); + } + patch::apply_batch_patch(&mut separation, tree_index, &binding.decisions, &cut_paths)?; + write_separation_state(&state_path, &separation)?; + app_log!( + "ui_separation.batch_completed tree_index={} batch_index={} cuts={} bound={} problematic={} elapsed_ms={}", + tree_index, batch_index, cut_paths.len(), separation.bound.len(), separation.problematic_nodes.len(), batch_started.elapsed().as_millis() + ); + batch_index += 1; + } + } + app_log!( + "ui_separation.completed asset_id={} bound_nodes={} problematic_nodes={}", + asset_id, + separation.bound.len(), + separation.problematic_nodes.len() + ); + Ok(separation_dto(&separation)) +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/patch.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/patch.rs new file mode 100644 index 000000000..c2fdd0e84 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow/patch.rs @@ -0,0 +1,98 @@ +use crate::ui_editor::commands::separation::{ + next_image_batch, validate_binding_response, BindingDecision, BindingResp, BoundNode, + ProblematicNode, SeparationNode, SeparationState, MAX_REWORK_COUNT, +}; +use crate::ui_editor::utils::NodeId; +use std::collections::HashMap; + +pub fn apply_batch_patch( + state: &mut SeparationState, + tree_index: usize, + decisions: &[BindingDecision], + cut_paths: &HashMap, +) -> Result<(), String> { + app_log!( + "ui_separation.batch_patch.start tree_index={} decisions={} cut_paths={}", + tree_index, + decisions.len(), + cut_paths.len() + ); + let batch_nodes = { + let tree = state + .trees + .get(tree_index) + .ok_or_else(|| "separation tree 索引无效".to_string())?; + next_image_batch(state, tree) + .into_iter() + .cloned() + .collect::>() + }; + let batch = batch_nodes.iter().collect::>(); + validate_binding_response( + &BindingResp { + decisions: decisions.to_vec(), + }, + &batch, + )?; + let rework_counts = batch_nodes + .iter() + .map(|node| (node.id.clone(), node.rework_count)) + .collect::>(); + let tree = state.trees.get_mut(tree_index).expect("tree index checked"); + for decision in decisions { + match decision { + BindingDecision::Ok { to_node, .. } => { + let path = cut_paths + .get(to_node) + .ok_or_else(|| format!("缺少节点 {} 的 cut 图片", to_node.as_str()))?; + state.bound.push(BoundNode { + node_id: to_node.clone(), + cut_image_path: path.clone(), + }); + } + BindingDecision::NeedRework { + to_node, + advice: problem_description, + } => { + append_rework_note(&mut tree.root, to_node, problem_description); + let count = rework_counts.get(to_node).copied().unwrap_or(0) + 1; + increment_rework_count(&mut tree.root, to_node, count); + if count >= MAX_REWORK_COUNT { + state.problematic_nodes.push(ProblematicNode { + node_id: to_node.clone(), + problem_description: problem_description.clone(), + rework_count: count, + }); + } + } + } + } + app_log!( + "ui_separation.batch_patch.completed tree_index={} bound={} problematic={} pending_root_children={}", + tree_index, + state.bound.len(), + state.problematic_nodes.len(), + tree.root.children.len() + ); + Ok(()) +} + +fn append_rework_note(node: &mut SeparationNode, id: &NodeId, note: &str) -> bool { + if node.id == *id { + node.note.rework_notes.push(note.to_string()); + return true; + } + node.children + .iter_mut() + .any(|child| append_rework_note(child, id, note)) +} + +fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) { + if node.id == *id { + node.rework_count = count; + return; + } + for child in &mut node.children { + increment_rework_count(child, id, count); + } +} diff --git a/docs/technical/【技术方案】UI编辑器自动切分素材工作流-2026-09-08.md b/docs/technical/【技术方案】UI编辑器自动切分素材工作流-2026-09-08.md index 28a48888e..de3f0c774 100644 --- a/docs/technical/【技术方案】UI编辑器自动切分素材工作流-2026-09-08.md +++ b/docs/technical/【技术方案】UI编辑器自动切分素材工作流-2026-09-08.md @@ -102,6 +102,12 @@ Rust 在接收并校验工具结果后,才把这两个枚举变体映射为正 - 图片编辑调用当前 Raw GPT Image 2 multipart 合同:`image`(PNG 文件)、`prompt`、`width`、`height`、`output_format=png`、`background=transparent`;不再发送旧 JSON/base64 请求体。 - 后端 raw operation 的持久状态与扣费后崩溃恢复窗口,遵循 Raw GPT Image 2 方案中的独立 TODO。 +## 实现组织 + +- separation prompt 按职责拆分为 `prompt/extract.rs` 与 `prompt/binding.rs`,由 `prompt/mod.rs` 统一导出。 +- separation workflow 按执行边界拆分为 `workflow/image_edit.rs`(Raw image-edit 与处理图写入)、`workflow/binding.rs`(视觉绑定)、`workflow/cut.rs`(像素归一化与裁切)、`workflow/patch.rs`(批次状态 patch);`workflow/mod.rs` 仅负责批次编排与 sidecar 检查点。 +- 上述拆分只调整 Rust 模块边界,不改变批次选择、重试、sidecar 持久化、绑定校验或错误恢复语义。 + ## TODO - 正在执行 batch 的持久化和恢复。