From d30bc462c59f733ef7fe5180bcf62e6ae1d3f536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Wed, 9 Sep 2026 14:50:19 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E5=8C=96=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E5=88=86=E7=A6=BB=20Rust=20=E6=A8=A1=E5=9D=97=20=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E6=A0=91=E9=80=89=E6=8B=A9=E3=80=81=E6=A0=87=E8=AE=B0?= =?UTF-8?q?=E5=9B=BE=E5=92=8C=E6=89=B9=E6=AC=A1=E6=B5=81=E7=A8=8B=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E9=A3=8E=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ui_editor/commands/separation/marker.rs | 120 +++++++++-- .../src/ui_editor/commands/separation/mod.rs | 19 +- .../src/ui_editor/commands/separation/tree.rs | 189 +++++++++++++----- .../ui_editor/commands/separation/workflow.rs | 42 +++- 4 files changed, 285 insertions(+), 85 deletions(-) diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/marker.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/marker.rs index f39015389..ffd972792 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/marker.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/marker.rs @@ -4,49 +4,129 @@ use image::GenericImage; use std::path::Path; use std::path::PathBuf; -pub async fn build_marked_image(source_url: String, nodes: Vec, target: PathBuf) -> Result { +pub async fn build_marked_image( + source_url: String, + nodes: Vec, + target: PathBuf, +) -> Result { tokio::task::spawn_blocking(move || build_marked_image_blocking(&source_url, &nodes, &target)) .await .map_err(|error| format!("构建标记图任务失败：{error}"))? } -fn build_marked_image_blocking(source_url: &str, nodes: &[SeparationNode], target: &Path) -> Result { - let encoded = source_url.split_once(',').map(|(_, d)| d).ok_or_else(|| "源图 data URL 无效".to_string())?; - let bytes = base64::engine::general_purpose::STANDARD.decode(encoded).map_err(|e| format!("解码源图失败：{e}"))?; - let mut image = image::load_from_memory(&bytes).map_err(|e| format!("读取源图失败：{e}"))?.to_rgba8(); +fn build_marked_image_blocking( + source_url: &str, + nodes: &[SeparationNode], + target: &Path, +) -> Result { + let encoded = source_url + .split_once(',') + .map(|(_, d)| d) + .ok_or_else(|| "源图 data URL 无效".to_string())?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|e| format!("解码源图失败：{e}"))?; + let mut image = image::load_from_memory(&bytes) + .map_err(|e| format!("读取源图失败：{e}"))? + .to_rgba8(); let width = image.width(); let height = image.height(); // Purple reconstruction comes first so the parent green frame remains visible on top. for node in nodes { for child in &node.children { - fill_rect(&mut image, child.global_pos_x_px, child.global_pos_y_px, child.width_px, child.height_px, image::Rgba([180, 0, 180, 120]), width, height); + fill_rect( + &mut image, + child.global_pos_x_px, + child.global_pos_y_px, + child.width_px, + child.height_px, + image::Rgba([180, 0, 180, 120]), + width, + height, + ); } } for node in nodes { - draw_frame(&mut image, node.global_pos_x_px, node.global_pos_y_px, node.width_px, node.height_px, width, height); + draw_frame( + &mut image, + node.global_pos_x_px, + node.global_pos_y_px, + node.width_px, + node.height_px, + width, + height, + ); } - image::DynamicImage::ImageRgba8(image.clone()).save_with_format(target, image::ImageFormat::Png).map_err(|e| format!("写入标记图失败：{e}"))?; + image::DynamicImage::ImageRgba8(image.clone()) + .save_with_format(target, image::ImageFormat::Png) + .map_err(|e| format!("写入标记图失败：{e}"))?; let mut png = Vec::new(); - image::DynamicImage::ImageRgba8(image).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).map_err(|e| format!("编码标记图失败：{e}"))?; - Ok(format!("data:image/png;base64,{}", base64::engine::general_purpose::STANDARD.encode(png))) + image::DynamicImage::ImageRgba8(image) + .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) + .map_err(|e| format!("编码标记图失败：{e}"))?; + Ok(format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(png) + )) } -fn clipped_rect(x: u32, y: u32, w: u32, h: u32, width: u32, height: u32) -> Option<(u32, u32, u32, u32)> { - if width == 0 || height == 0 || w == 0 || h == 0 { return None; } - let x0 = x.min(width - 1); let y0 = y.min(height - 1); +fn clipped_rect( + x: u32, + y: u32, + w: u32, + h: u32, + width: u32, + height: u32, +) -> Option<(u32, u32, u32, u32)> { + if width == 0 || height == 0 || w == 0 || h == 0 { + return None; + } + let x0 = x.min(width - 1); + let y0 = y.min(height - 1); let x1 = x.saturating_add(w).min(width).saturating_sub(1); let y1 = y.saturating_add(h).min(height).saturating_sub(1); (x0 <= x1 && y0 <= y1).then_some((x0, y0, x1, y1)) } -fn fill_rect(image: &mut image::RgbaImage, x: u32, y: u32, w: u32, h: u32, color: image::Rgba, width: u32, height: u32) { - let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else { return; }; - for yy in y0..=y1 { for xx in x0..=x1 { image.put_pixel(xx, yy, color); } } +fn fill_rect( + image: &mut image::RgbaImage, + x: u32, + y: u32, + w: u32, + h: u32, + color: image::Rgba, + width: u32, + height: u32, +) { + let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else { + return; + }; + for yy in y0..=y1 { + for xx in x0..=x1 { + image.put_pixel(xx, yy, color); + } + } } -fn draw_frame(image: &mut image::RgbaImage, x: u32, y: u32, w: u32, h: u32, width: u32, height: u32) { - let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else { return; }; +fn draw_frame( + image: &mut image::RgbaImage, + x: u32, + y: u32, + w: u32, + h: u32, + width: u32, + height: u32, +) { + let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else { + return; + }; let green = image::Rgba([0, 255, 0, 255]); - for xx in x0..=x1 { image.put_pixel(xx, y0, green); image.put_pixel(xx, y1, green); } - for yy in y0..=y1 { image.put_pixel(x0, yy, green); image.put_pixel(x1, yy, green); } + for xx in x0..=x1 { + image.put_pixel(xx, y0, green); + image.put_pixel(xx, y1, green); + } + for yy in y0..=y1 { + image.put_pixel(x0, yy, green); + image.put_pixel(x1, yy, green); + } } diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs index 41f24538a..83a984c2e 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs @@ -1,31 +1,31 @@ -mod model; mod marker; +mod model; mod persistence; mod prompt; mod tree; mod workflow; +pub(crate) use marker::build_marked_image; pub use model::*; pub use persistence::*; pub use tree::*; pub use workflow::apply_batch_patch; pub(crate) use workflow::separate_ui_impl; -pub(crate) use marker::build_marked_image; #[cfg(test)] mod tests { use super::*; - use crate::ui_editor::component::Component; - use crate::ui_editor::layout::node::Node; - use crate::ui_editor::state::{State, UITree}; - use crate::ui_editor::utils::{NodeId, UIDesignImageId}; - use std::path::Path; use crate::ui_editor::component::image::{ImageComponent, ImageType}; + use crate::ui_editor::component::Component; use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode; use crate::ui_editor::layout::control_layout::ControlLayout; + use crate::ui_editor::layout::node::Node; use crate::ui_editor::layout::node::{NodeMetadata, NodeSource, StageStatus}; use crate::ui_editor::resource::ui_design_image::UIDesignImage; + use crate::ui_editor::state::{State, UITree}; + use crate::ui_editor::utils::{NodeId, UIDesignImageId}; use nalgebra::Vector2; use std::collections::HashMap; + use std::path::Path; use typed_floats::tf32::StrictlyPositiveFinite; fn node(id: &str, components: Vec, children: Vec) -> Node { @@ -89,10 +89,7 @@ mod tests { )], ); let result = construct_separation_state(&state(root)); - assert_eq!( - result.trees[0].root.children[0].id.as_str(), - "image" - ); + assert_eq!(result.trees[0].root.children[0].id.as_str(), "image"); } #[test] diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/tree.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/tree.rs index 5c84af6db..dcca538e7 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/tree.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/tree.rs @@ -7,11 +7,21 @@ use std::collections::HashSet; fn is_unbound_image(node: &Node) -> bool { node.components.iter().any(|component| { - matches!(component, Component::Image(ImageComponent { target_graphic: None, .. })) + matches!( + component, + Component::Image(ImageComponent { + target_graphic: None, + .. + }) + ) }) } -fn node_pixel_rect(node: &Node, parent: &crate::ui_editor::layout::dimension::UIRect, ppu: f32) -> (u32, u32, u32, u32) { +fn node_pixel_rect( + node: &Node, + parent: &crate::ui_editor::layout::dimension::UIRect, + ppu: f32, +) -> (u32, u32, u32, u32) { let rect = node.layout.transform.resolve(parent); ( (rect.min.x * ppu).max(0.0).round() as u32, @@ -32,7 +42,12 @@ fn node_description(node: &Node) -> String { } } -fn collect_todo_nodes(node: &Node, parent: &crate::ui_editor::layout::dimension::UIRect, ppu: f32, output: &mut Vec) { +fn collect_todo_nodes( + node: &Node, + parent: &crate::ui_editor::layout::dimension::UIRect, + ppu: f32, + output: &mut Vec, +) { let rect = node.layout.transform.resolve(parent); let mut children = Vec::new(); for child in &node.children { @@ -41,10 +56,17 @@ fn collect_todo_nodes(node: &Node, parent: &crate::ui_editor::layout::dimension: if is_unbound_image(node) { let (x, y, w, h) = node_pixel_rect(node, parent, ppu); output.push(SeparationNode { - id: node.id.clone(), global_pos_x_px: x, global_pos_y_px: y, - width_px: w, height_px: h, - note: SeparationNote { description: node_description(node), text_note: String::new() }, - children, rework_count: 0, + id: node.id.clone(), + global_pos_x_px: x, + global_pos_y_px: y, + width_px: w, + height_px: h, + note: SeparationNote { + description: node_description(node), + text_note: String::new(), + }, + children, + rework_count: 0, }); } else { output.extend(children); @@ -52,40 +74,71 @@ fn collect_todo_nodes(node: &Node, parent: &crate::ui_editor::layout::dimension: } pub fn construct_separation_state(state: &State) -> SeparationState { - let trees = state.ui_trees.iter().filter_map(|tree| { - let image = state.ui_design_images.get(&tree.src_ui_design)?; - let ppu = image.pixels_per_unit.get(); - let size = image.pixel_size / ppu; - let root_rect = crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size); - let mut children = Vec::new(); - for child in &tree.root.children { - collect_todo_nodes(child, &root_rect, ppu, &mut children); - } - let root_extractable = is_unbound_image(&tree.root); - if !root_extractable && children.is_empty() { return None; } - let (x, y, w, h) = node_pixel_rect(&tree.root, &root_rect, ppu); - Some(SeparationTree { - src_ui_design: tree.src_ui_design.clone(), - root: SeparationNode { - id: tree.root.id.clone(), global_pos_x_px: x, global_pos_y_px: y, - width_px: w, height_px: h, - note: SeparationNote { description: node_description(&tree.root), text_note: String::new() }, - children, rework_count: 0, - }, - root_extractable, + let trees = state + .ui_trees + .iter() + .filter_map(|tree| { + let image = state.ui_design_images.get(&tree.src_ui_design)?; + let ppu = image.pixels_per_unit.get(); + let size = image.pixel_size / ppu; + let root_rect = + crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size); + let mut children = Vec::new(); + for child in &tree.root.children { + collect_todo_nodes(child, &root_rect, ppu, &mut children); + } + let root_extractable = is_unbound_image(&tree.root); + if !root_extractable && children.is_empty() { + return None; + } + let (x, y, w, h) = node_pixel_rect(&tree.root, &root_rect, ppu); + Some(SeparationTree { + src_ui_design: tree.src_ui_design.clone(), + root: SeparationNode { + id: tree.root.id.clone(), + global_pos_x_px: x, + global_pos_y_px: y, + width_px: w, + height_px: h, + note: SeparationNote { + description: node_description(&tree.root), + text_note: String::new(), + }, + children, + rework_count: 0, + }, + root_extractable, + }) }) - }).collect(); - SeparationState { schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(), trees, bound: Vec::new(), problematic_nodes: Vec::new() } + .collect(); + SeparationState { + schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(), + trees, + bound: Vec::new(), + problematic_nodes: Vec::new(), + } } fn terminal_ids(state: &SeparationState) -> HashSet { - state.bound.iter().map(|n| n.node_id.clone()) - .chain(state.problematic_nodes.iter().map(|n| n.node_id.clone())).collect() + state + .bound + .iter() + .map(|n| n.node_id.clone()) + .chain(state.problematic_nodes.iter().map(|n| n.node_id.clone())) + .collect() } -fn logical_leaves<'a>(node: &'a SeparationNode, extractable: bool, terminal: &HashSet, output: &mut Vec<&'a SeparationNode>) { +fn logical_leaves<'a>( + node: &'a SeparationNode, + extractable: bool, + terminal: &HashSet, + output: &mut Vec<&'a SeparationNode>, +) { let is_terminal = terminal.contains(&node.id); - let children_terminal = node.children.iter().all(|child| terminal.contains(&child.id)); + let children_terminal = node + .children + .iter() + .all(|child| terminal.contains(&child.id)); if extractable && !is_terminal && children_terminal { output.push(node); return; @@ -100,34 +153,74 @@ fn overlaps(a: &SeparationNode, b: &SeparationNode) -> bool { let ay1 = a.global_pos_y_px as u64 + a.height_px as u64; let bx1 = b.global_pos_x_px as u64 + b.width_px as u64; let by1 = b.global_pos_y_px as u64 + b.height_px as u64; - let width = ax1.min(bx1).saturating_sub(a.global_pos_x_px.max(b.global_pos_x_px) as u64); - let height = ay1.min(by1).saturating_sub(a.global_pos_y_px.max(b.global_pos_y_px) as u64); + let width = ax1 + .min(bx1) + .saturating_sub(a.global_pos_x_px.max(b.global_pos_x_px) as u64); + let height = ay1 + .min(by1) + .saturating_sub(a.global_pos_y_px.max(b.global_pos_y_px) as u64); width > 0 && height > 0 } -pub fn next_leaf_batch<'a>(state: &SeparationState, tree: &'a SeparationTree) -> Vec<&'a SeparationNode> { +pub fn next_leaf_batch<'a>( + state: &SeparationState, + tree: &'a SeparationTree, +) -> Vec<&'a SeparationNode> { let terminal = terminal_ids(state); let mut candidates = Vec::new(); - logical_leaves(&tree.root, tree.root_extractable, &terminal, &mut candidates); + logical_leaves( + &tree.root, + tree.root_extractable, + &terminal, + &mut candidates, + ); let mut selected: Vec<&'a SeparationNode> = Vec::new(); for candidate in candidates { - if selected.iter().all(|other| !overlaps(candidate, other)) { selected.push(candidate); } + if selected.iter().all(|other| !overlaps(candidate, other)) { + selected.push(candidate); + } } - app_log!("ui_separation.batch_selected image_id={} leaf_nodes={}", tree.src_ui_design.as_str(), selected.len()); + app_log!( + "ui_separation.batch_selected image_id={} leaf_nodes={}", + tree.src_ui_design.as_str(), + selected.len() + ); selected } -pub fn validate_binding_response(response: &BindingResp, batch: &[&SeparationNode]) -> Result<(), String> { - let expected = batch.iter().map(|node| node.id.clone()).collect::>(); +pub fn validate_binding_response( + response: &BindingResp, + batch: &[&SeparationNode], +) -> Result<(), String> { + let expected = batch + .iter() + .map(|node| node.id.clone()) + .collect::>(); let mut seen = HashSet::new(); for decision in &response.decisions { - let node_id = match decision { BindingDecision::Ok { to_node, .. } | BindingDecision::NeedRework { to_node, .. } => to_node }; - if !expected.contains(node_id) { return Err(format!("视觉绑定返回了未知节点：{}", node_id.as_str())); } - if !seen.insert(node_id.clone()) { return Err(format!("视觉绑定重复返回节点：{}", node_id.as_str())); } - if let BindingDecision::NeedRework { problem_description, .. } = decision { - if problem_description.trim().is_empty() { return Err("NeedRework 必须包含问题描述".to_string()); } + let node_id = match decision { + BindingDecision::Ok { to_node, .. } | BindingDecision::NeedRework { to_node, .. } => { + to_node + } + }; + if !expected.contains(node_id) { + return Err(format!("视觉绑定返回了未知节点：{}", node_id.as_str())); + } + if !seen.insert(node_id.clone()) { + return Err(format!("视觉绑定重复返回节点：{}", node_id.as_str())); + } + if let BindingDecision::NeedRework { + problem_description, + .. + } = decision + { + if problem_description.trim().is_empty() { + return Err("NeedRework 必须包含问题描述".to_string()); + } } } - if seen.len() != expected.len() { return Err("视觉绑定未覆盖当前 batch 的全部节点".to_string()); } + if seen.len() != expected.len() { + return Err("视觉绑定未覆盖当前 batch 的全部节点".to_string()); + } Ok(()) } diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs index 07221b0e4..521754270 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs @@ -89,6 +89,27 @@ pub fn apply_batch_patch( Ok(()) } +fn mark_batch_problematic( + state: &mut SeparationState, + tree_index: usize, + batch: &[&SeparationNode], + error: String, +) { + app_log!( + "ui_separation.batch_problematic tree_index={} nodes={} error={}", + tree_index, + batch.len(), + error + ); + for node in batch { + state.problematic_nodes.push(ProblematicNode { + node_id: node.id.clone(), + problem_description: error.clone(), + rework_count: node.rework_count, + }); + } +} + fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) { if node.id == *id { node.rework_count = count; @@ -279,7 +300,6 @@ async fn visual_binding( let llm_config = llm_config.clone(); async move { let request = LlmRunRequest::new(vec![ - LlmMessage::system("你是 UI 图片视觉绑定器。只根据图像判断区域，不做 OCR。"), LlmMessage::user_multimodal(vec![ LlmMessageContentPart::InputText { text: prompt }, LlmMessageContentPart::InputImage { @@ -456,8 +476,10 @@ pub(crate) async fn separate_ui_impl( tree_index, batch_index ); + mark_batch_problematic(&mut separation, tree_index, &batch, error); write_separation_state(&state_path, &separation)?; - return Err(error); + batch_index += 1; + continue; } }; let processed_url = match raw_image_edit( @@ -476,8 +498,10 @@ pub(crate) async fn separate_ui_impl( tree_index, batch_index ); + mark_batch_problematic(&mut separation, tree_index, &batch, error); write_separation_state(&state_path, &separation)?; - return Err(error); + batch_index += 1; + continue; } }; let processed_path = sidecar.join(format!("processed-{}.png", separation.bound.len())); @@ -489,8 +513,10 @@ pub(crate) async fn separate_ui_impl( tree_index, batch_index ); + mark_batch_problematic(&mut separation, tree_index, &batch, error); write_separation_state(&state_path, &separation)?; - return Err(error); + batch_index += 1; + continue; } let binding = match visual_binding(source_url.clone(), processed_url, &batch).await { Ok(value) => value, @@ -500,8 +526,10 @@ pub(crate) async fn separate_ui_impl( tree_index, batch_index ); + mark_batch_problematic(&mut separation, tree_index, &batch, error); write_separation_state(&state_path, &separation)?; - return Err(error); + batch_index += 1; + continue; } }; app_log!( @@ -550,8 +578,10 @@ pub(crate) async fn separate_ui_impl( tree_index, batch_index ); + mark_batch_problematic(&mut separation, tree_index, &batch, error); write_separation_state(&state_path, &separation)?; - return Err(error); + batch_index += 1; + continue; } apply_batch_patch(&mut separation, tree_index, &binding.decisions, &cut_paths)?; write_separation_state(&state_path, &separation)?;