diff --git a/apps/ai-game-creator-shell/src-tauri/src/main.rs b/apps/ai-game-creator-shell/src-tauri/src/main.rs index e163cad21..3f4293087 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/main.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/main.rs @@ -328,6 +328,15 @@ async fn recognize_ui( ui_editor::commands::recognize_ui_impl(project_path, state).await } +#[tauri::command] +async fn separate_ui( + project_path: String, + asset_id: String, + state: ui_editor::state::State, +) -> Result { + ui_editor::commands::separate_ui_impl(project_path, asset_id, state).await +} + #[tauri::command] async fn merge_ui(state: ui_editor::state::State) -> Result { ui_editor::commands::merge_ui_impl(state).await @@ -2559,6 +2568,7 @@ fn main() { check_ui_editor_font_glyph_coverage, suggest_ui_design_semantic, recognize_ui, + separate_ui, merge_ui, bind_components, load_ui_design_state, diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs index e2e1a1bc5..181931f2d 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs @@ -1,6 +1,7 @@ pub mod binding; pub mod merge; pub mod recognition; +pub mod separation; pub mod ui_design_suggestion; pub mod utils; @@ -10,5 +11,7 @@ pub use merge::MergeDTO; pub(crate) use merge::{merge_ui_impl, merge_ui_impl_with_provider}; pub use recognition::RecognitionDTO; pub(crate) use recognition::{recognize_ui_impl, recognize_ui_impl_with_provider}; +pub(crate) use separation::separate_ui_impl; +pub use separation::SeparationDTO; pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl; pub use ui_design_suggestion::UIDesignSuggestionTreeNode; diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs index 081db0bb1..ceae0d7ea 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs @@ -4,6 +4,7 @@ use crate::ui_editor::commands::utils::{ parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm, strict_json_schema, }; +use crate::ui_editor::component::Component; use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode; use crate::ui_editor::layout::control_layout::ControlLayout; use crate::ui_editor::layout::dimension::UIRect; @@ -46,6 +47,8 @@ const SYSTEM_PROMPT: &str = r#" * 由于每个截图未必是完整的, 可能是局部的, 每棵树描述清楚每个截图上UI的层次结构即可 * 不同树的共用框架/层次/...请使用使用相同的名称描述. 不同状态/变体名称使用相同的前缀, 用后缀区别 * 粒度要求: 尽可能细致, 最小单元举例: 进度条的底槽、填充和外框; slider的底槽, dragger等 +* 为每个节点直接返回完整 components。纯容器返回空数组；需要从设计图自动分离图片的 Image component 必须令 target_graphic 为 null。文本内容和组件类型完全由视觉判断，不调用或依赖 OCR。 +* 每个节点当前最多返回一个 Image component 和一个 Text component。 "#; @@ -109,6 +112,7 @@ struct RecognitionNode { description: String, children: Vec, confidence: Confidence, + components: Vec, } #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] @@ -312,10 +316,7 @@ fn convert_node( allow_llm_edit_component: true, source: NodeSource::Llm, }, - // V1 有意把识别结果限定为“结构草稿”：组件绑定属于后续独立阶段。 - // 因此空组件不是丢失数据，而是等待 visual-binding 阶段补齐 Image/Text。 - // 约定见 docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md。 - components: Vec::new(), + components: source.components.clone(), children_display_mode: ChildrenDisplayMode::Stack, children, }) @@ -323,6 +324,30 @@ fn convert_node( fn validate_confidence(nodes: &[RecognitionNode]) -> Result<(), String> { for node in nodes { + let image_count = node + .components + .iter() + .filter(|component| matches!(component, Component::Image(_))) + .count(); + let text_count = node + .components + .iter() + .filter(|component| matches!(component, Component::Text(_))) + .count(); + if image_count > 1 || text_count > 1 { + return Err("单个节点当前最多包含一个 Image 和一个 Text component".to_string()); + } + if node.components.iter().any(|component| { + matches!( + component, + Component::Image(crate::ui_editor::component::image::ImageComponent { + target_graphic: Some(_), + .. + }) + ) + }) { + return Err("识别阶段不能返回已绑定的 SpriteAssetId".to_string()); + } if let Confidence::UnSure(reason) = &node.confidence { if reason.trim().is_empty() { return Err("UnSure 必须包含审阅原因".to_string()); @@ -387,6 +412,7 @@ mod tests { description: String::new(), children: Vec::new(), confidence: Confidence::Confident, + components: Vec::new(), } } @@ -490,6 +516,7 @@ mod tests { description: String::new(), children: Vec::new(), confidence: Confidence::UnSure(String::new()), + components: Vec::new(), }; assert!(validate_confidence(&[node]).is_err()); } diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation.rs new file mode 100644 index 000000000..d1217aefa --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation.rs @@ -0,0 +1,861 @@ +use crate::config::{build_game_creator_llm_client_from_llm_config, load_game_creator_app_config}; +use crate::platform_session::current_platform_session; +use crate::ui_editor::commands::utils::{ + parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm, + strict_json_schema, +}; +use crate::ui_editor::component::image::ImageComponent; +use crate::ui_editor::component::Component; +use crate::ui_editor::layout::node::Node; +use crate::ui_editor::state::{State, UITree}; +use crate::ui_editor::utils::{NodeId, UIDesignImageId}; +use base64::Engine as _; +use image::ImageFormat; +use platform_llm::{ + LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice, +}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::fs; +use std::path::{Path, PathBuf}; +use ts_rs::TS; + +pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1"; +pub const MAX_REWORK_COUNT: u32 = 3; + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationNote { + pub description: String, +} + +impl SeparationNote { + pub fn as_prompt(&self) -> String { + format!("- {}", self.description.trim()) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationNode { + pub id: NodeId, + pub global_pos_x_px: u32, + pub global_pos_y_px: u32, + pub width_px: u32, + pub height_px: u32, + pub note: SeparationNote, + pub children: Vec, + pub rework_count: u32, +} + +impl SeparationNode { + pub fn as_prompt(&self) -> String { + format!( + "node_id={} area=({}, {}, {}, {}) {}", + self.id.as_str(), + self.global_pos_x_px, + self.global_pos_y_px, + self.width_px, + self.height_px, + self.note.as_prompt() + ) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationTree { + pub src_ui_design: UIDesignImageId, + pub root: SeparationNode, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct BoundNode { + pub node_id: NodeId, + pub cut_image_path: String, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct ProblematicNode { + pub node_id: NodeId, + pub problem_description: String, + pub rework_count: u32, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationState { + pub schema_version: String, + pub unprocessed_trees: Vec, + pub bound: Vec, + pub problematic_nodes: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationDTO { + pub bound_nodes: Vec, + pub problematic_nodes: Vec, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub struct BindingArea { + pub global_pos_x_px: u32, + pub global_pos_y_px: u32, + pub width_px: u32, + pub height_px: u32, +} + +impl BindingArea { + pub fn validate_in(&self, image_width: u32, image_height: u32) -> Result<(), String> { + if self.width_px == 0 || self.height_px == 0 { + return Err("BindingArea 宽度和高度必须大于 0".to_string()); + } + let max_x = self + .global_pos_x_px + .checked_add(self.width_px) + .ok_or_else(|| "BindingArea 横向范围溢出".to_string())?; + let max_y = self + .global_pos_y_px + .checked_add(self.height_px) + .ok_or_else(|| "BindingArea 纵向范围溢出".to_string())?; + if max_x > image_width || max_y > image_height { + return Err("BindingArea 超出处理图边界".to_string()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub enum BindingDecision { + Ok { + separated_image_area: BindingArea, + to_node: NodeId, + }, + NeedRework { + problem_description: String, + to_node: NodeId, + }, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub struct BindingResp { + pub decisions: Vec, +} + +const SHARED_SEPARATION_REQ: &str = r#" +MUST hard edges; preserve no glow/blur beyond the exact visible shape. +NEVER keep its parent's background with it. +UI elements marked with GREEN line frames are extraction marks only; never include the frame. +PURPLE filled areas represent removed elements; reconstruct the background under them. +"#; + +pub fn gen_extract_prompt(separation_notes: &[SeparationNote]) -> String { + let mut result = format!("This is a UI design image, not a normal photo/illustration.\nExtract distinct UI elements as independent layers with clean edges and full transparency outside each element.\nKeep every element at its original position on a transparent canvas.\n{}\nElements to extract:\n", SHARED_SEPARATION_REQ); + for note in separation_notes { + result.push_str(¬e.as_prompt()); + result.push('\n'); + } + result +} + +pub fn gen_binding_prompt(nodes: &[&SeparationNode]) -> String { + let mut result = format!("You are reviewing a UI elements separation result.\n{}\nThe source and processed images use top-left pixel coordinates.\nReturn one decision for every requested node. For a failed node, provide a short repair description.\nNodes:\n", SHARED_SEPARATION_REQ); + for node in nodes { + result.push_str(&node.as_prompt()); + result.push('\n'); + } + result +} + +fn is_unbound_image(node: &Node) -> bool { + node.components.iter().any(|component| { + matches!( + component, + Component::Image(ImageComponent { + target_graphic: None, + .. + }) + ) + }) +} + +fn node_pixel_rect( + node: &Node, + parent: &crate::ui_editor::layout::dimension::UIRect, + ppu: f32, +) -> (u32, u32, u32, u32) { + let rect = node.layout.transform.resolve(parent); + let x = (rect.min.x * ppu).max(0.0).round() as u32; + let y = (rect.min.y * ppu).max(0.0).round() as u32; + let w = (rect.size.x * ppu).max(0.0).round() as u32; + let h = (rect.size.y * ppu).max(0.0).round() as u32; + (x, y, w, h) +} + +fn node_description(node: &Node) -> String { + let name = node.metadata.name.trim(); + let description = node.metadata.description.trim(); + match (name.is_empty(), description.is_empty()) { + (true, true) => "未命名 UI 图片元素".to_string(), + (false, true) => name.to_string(), + (true, false) => description.to_string(), + (false, false) => format!("{name}：{description}"), + } +} + +fn collect_todo_nodes( + node: &Node, + parent: &crate::ui_editor::layout::dimension::UIRect, + ppu: f32, + output: &mut Vec, +) { + let mut children = Vec::new(); + let rect = node.layout.transform.resolve(parent); + for child in &node.children { + collect_todo_nodes(child, &rect, ppu, &mut children); + } + if is_unbound_image(node) { + let (x, y, w, h) = node_pixel_rect(node, parent, ppu); + output.push(SeparationNode { + id: node.id.clone(), + global_pos_x_px: x, + global_pos_y_px: y, + width_px: w, + height_px: h, + note: SeparationNote { + description: node_description(node), + }, + children, + rework_count: 0, + }); + } else { + output.extend(children); + } +} + +pub fn construct_separation_state(state: &State) -> SeparationState { + let unprocessed_trees = state + .ui_trees + .iter() + .filter_map(|tree| { + let image = state.ui_design_images.get(&tree.src_ui_design)?; + let ppu = image.pixels_per_unit.get(); + let size = image.pixel_size / ppu; + let root_rect = + crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size); + let mut children = Vec::new(); + collect_todo_nodes(&tree.root, &root_rect, ppu, &mut children); + (!children.is_empty()).then(|| SeparationTree { + src_ui_design: tree.src_ui_design.clone(), + root: SeparationNode { + id: tree.root.id.clone(), + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: image.pixel_size.x.max(0.0).round() as u32, + height_px: image.pixel_size.y.max(0.0).round() as u32, + note: SeparationNote::default(), + children, + rework_count: 0, + }, + }) + }) + .collect(); + SeparationState { + schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(), + unprocessed_trees, + bound: Vec::new(), + problematic_nodes: Vec::new(), + } +} + +pub fn next_leaf_batch(tree: &SeparationTree) -> Vec<&SeparationNode> { + fn leaves<'a>(node: &'a SeparationNode, output: &mut Vec<&'a SeparationNode>) { + if node.children.is_empty() { + output.push(node); + } else { + for child in &node.children { + leaves(child, output); + } + } + } + let mut output = Vec::new(); + leaves(&tree.root, &mut output); + output +} + +pub fn validate_binding_response( + response: &BindingResp, + batch: &[&SeparationNode], +) -> Result<(), String> { + let expected = batch + .iter() + .map(|node| node.id.clone()) + .collect::>(); + let mut seen = std::collections::HashSet::new(); + for decision in &response.decisions { + let node_id = match decision { + BindingDecision::Ok { to_node, .. } | BindingDecision::NeedRework { to_node, .. } => { + to_node + } + }; + if !expected.contains(node_id) { + return Err(format!("视觉绑定返回了未知节点：{}", node_id.as_str())); + } + if !seen.insert(node_id.clone()) { + return Err(format!("视觉绑定重复返回节点：{}", node_id.as_str())); + } + if let BindingDecision::NeedRework { + problem_description, + .. + } = decision + { + if problem_description.trim().is_empty() { + return Err("NeedRework 必须包含问题描述".to_string()); + } + } + } + if seen.len() != expected.len() { + return Err("视觉绑定未覆盖当前 batch 的全部节点".to_string()); + } + Ok(()) +} + +pub fn separation_sidecar_dir(root: &Path, asset_id: &str) -> Result { + if asset_id.trim().is_empty() || asset_id.trim() != asset_id { + return Err("UI 资源 ID 无效".to_string()); + } + let mut stem = String::new(); + for character in asset_id.chars() { + if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') { + stem.push(character); + } else { + stem.push('_'); + } + } + let digest = format!("{:x}", Sha256::digest(asset_id.as_bytes())); + let dir = root + .join("ui") + .join(format!(".{stem}-{}-separation", &digest[..16])); + if !dir.starts_with(root) { + return Err("separation sidecar 路径越界".to_string()); + } + Ok(dir) +} + +pub fn write_separation_state(path: &Path, state: &SeparationState) -> Result<(), String> { + if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION { + return Err("不支持的 separation state schema".to_string()); + } + let bytes = serde_json::to_vec_pretty(state) + .map_err(|error| format!("序列化 separation state 失败：{error}"))?; + let parent = path + .parent() + .ok_or_else(|| "separation state 路径缺少父目录".to_string())?; + fs::create_dir_all(parent).map_err(|error| format!("创建 separation sidecar 失败：{error}"))?; + let temporary = path.with_extension("json.tmp"); + fs::write(&temporary, bytes).map_err(|error| format!("写入 separation state 失败：{error}"))?; + fs::rename(&temporary, path).map_err(|error| format!("安装 separation state 失败：{error}")) +} + +pub fn read_separation_state(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|error| format!("读取 separation state 失败：{error}"))?; + let state: SeparationState = serde_json::from_slice(&bytes) + .map_err(|error| format!("解析 separation state 失败：{error}"))?; + if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION { + return Err("不支持的 separation state schema".to_string()); + } + Ok(state) +} + +pub fn separation_dto(state: &SeparationState) -> SeparationDTO { + SeparationDTO { + bound_nodes: state.bound.clone(), + problematic_nodes: state.problematic_nodes.clone(), + } +} + +pub fn apply_batch_patch( + state: &mut SeparationState, + tree_index: usize, + decisions: &[BindingDecision], + cut_paths: &std::collections::HashMap, +) -> Result<(), String> { + let tree = state + .unprocessed_trees + .get_mut(tree_index) + .ok_or_else(|| "separation tree 索引无效".to_string())?; + let batch = next_leaf_batch(tree); + validate_binding_response( + &BindingResp { + decisions: decisions.to_vec(), + }, + &batch, + )?; + let rework_counts = batch + .iter() + .map(|node| (node.id.clone(), node.rework_count)) + .collect::>(); + let mut ids = std::collections::HashSet::new(); + for decision in decisions { + match decision { + BindingDecision::Ok { to_node, .. } => { + let path = cut_paths + .get(to_node) + .ok_or_else(|| format!("缺少节点 {} 的 cut 图片", to_node.as_str()))?; + state.bound.push(BoundNode { + node_id: to_node.clone(), + cut_image_path: path.clone(), + }); + ids.insert(to_node.clone()); + } + BindingDecision::NeedRework { + to_node, + problem_description, + } => { + let count = rework_counts.get(to_node).copied().unwrap_or(0) + 1; + if count >= MAX_REWORK_COUNT { + state.problematic_nodes.push(ProblematicNode { + node_id: to_node.clone(), + problem_description: problem_description.clone(), + rework_count: count, + }); + ids.insert(to_node.clone()); + } else { + increment_rework_count(&mut tree.root, to_node, count); + } + } + } + } + remove_ids(&mut tree.root, &ids); + Ok(()) +} + +fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) { + if node.id == *id { + node.rework_count = count; + return; + } + for child in &mut node.children { + increment_rework_count(child, id, count); + } +} + +#[derive(Deserialize)] +struct RawEditResponse { + data: Vec, +} +#[derive(Deserialize)] +struct RawEditItem { + b64_json: String, +} + +async fn raw_image_edit( + session: &crate::platform_session::PlatformSessionSnapshot, + image_data_url: &str, + prompt: &str, + width: u32, + height: u32, +) -> Result { + let (mime, data) = image_data_url + .split_once(",") + .ok_or_else(|| "界面图 data URL 无效".to_string())?; + let mime = mime + .strip_prefix("data:") + .and_then(|v| v.strip_suffix(";base64")) + .unwrap_or("image/png"); + let client = crate::http_client::agc_main_site_client_builder() + .build() + .map_err(|e| format!("创建图片编辑客户端失败：{e}"))?; + let url = format!( + "{}/api/raw/v1/images/edit", + session.api_base_url.trim_end_matches('/') + ); + let body = serde_json::json!({ + "image": {"data": data, "mimeType": mime}, + "prompt": prompt, + "width": width, + "height": height, + "output_format": "png", + "background": "transparent" + }); + let response = crate::http_client::with_agc_main_site_marker( + client + .post(url) + .bearer_auth(&session.access_token) + .json(&body), + ) + .send() + .await + .map_err(|e| format!("图片分离请求失败：{e}"))?; + if !response.status().is_success() { + return Err(format!("图片分离请求失败（HTTP {}）", response.status())); + } + let payload = response + .json::() + .await + .map_err(|e| format!("解析图片分离响应失败：{e}"))?; + payload + .data + .into_iter() + .next() + .map(|item| format!("data:image/png;base64,{}", item.b64_json)) + .ok_or_else(|| "图片分离响应没有图像".to_string()) +} + +fn build_marked_image( + source_url: &str, + nodes: &[&SeparationNode], + target: &Path, +) -> Result { + let encoded = source_url + .split_once(',') + .map(|(_, d)| d) + .ok_or_else(|| "源图 data URL 无效".to_string())?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|e| format!("解码源图失败：{e}"))?; + let mut image = image::load_from_memory(&bytes) + .map_err(|e| format!("读取源图失败：{e}"))? + .to_rgba8(); + let width = image.width(); + let height = image.height(); + for node in nodes { + let x0 = node.global_pos_x_px.min(width.saturating_sub(1)); + let y0 = node.global_pos_y_px.min(height.saturating_sub(1)); + let x1 = node + .global_pos_x_px + .saturating_add(node.width_px) + .min(width) + .saturating_sub(1); + let y1 = node + .global_pos_y_px + .saturating_add(node.height_px) + .min(height) + .saturating_sub(1); + if x0 >= x1 || y0 >= y1 { + continue; + } + for x in x0..=x1 { + image.put_pixel(x, y0, image::Rgba([0, 255, 0, 255])); + image.put_pixel(x, y1, image::Rgba([0, 255, 0, 255])); + } + for y in y0..=y1 { + image.put_pixel(x0, y, image::Rgba([0, 255, 0, 255])); + image.put_pixel(x1, y, image::Rgba([0, 255, 0, 255])); + } + for y in y0..=y1 { + for x in x0..=x1 { + if x > x0 && x < x1 && y > y0 && y < y1 { + image.put_pixel(x, y, image::Rgba([180, 0, 180, 120])); + } + } + } + } + image::DynamicImage::ImageRgba8(image.clone()) + .save_with_format(target, image::ImageFormat::Png) + .map_err(|e| format!("写入标记图失败：{e}"))?; + let mut png = Vec::new(); + image::DynamicImage::ImageRgba8(image) + .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) + .map_err(|e| format!("编码标记图失败：{e}"))?; + Ok(format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(png) + )) +} + +async fn visual_binding( + source_url: String, + processed_url: String, + nodes: &[&SeparationNode], +) -> Result { + let llm_config = load_game_creator_app_config() + .map_err(|e| e.to_string())? + .llm; + let client = build_game_creator_llm_client_from_llm_config(&llm_config, "llm") + .map_err(|e| e.to_string())?; + let schema = strict_json_schema::()?; + let tool = LlmFunctionTool::new( + "bind_ui_elements", + "确认处理图中的区域对应哪些 UI 节点", + schema, + ) + .with_strict(true); + let base_prompt = gen_binding_prompt(nodes); + let mut repair = None; + for attempt in 0..2 { + let prompt = repair.as_ref().map_or_else( + || base_prompt.clone(), + |error: &String| format!("{base_prompt}\n上一次输出错误：{error}\n请修正并完整返回。"), + ); + let request = LlmRunRequest::new(vec![ + LlmMessage::system("你是 UI 图片视觉绑定器。只根据图像判断区域，不做 OCR。"), + LlmMessage::user_multimodal(vec![ + LlmMessageContentPart::InputText { text: prompt }, + LlmMessageContentPart::InputImage { + image_url: source_url.clone(), + }, + LlmMessageContentPart::InputImage { + image_url: processed_url.clone(), + }, + ]), + ]) + .with_function_tools(vec![tool.clone()]) + .with_tool_choice(LlmToolChoice::Required); + let result = request_ui_editor_llm(&client, &llm_config, request) + .await + .map_err(|e| e.to_string()) + .and_then(|response| { + response + .tool_calls + .into_iter() + .find(|call| call.name == "bind_ui_elements") + .map(|call| call.arguments) + .ok_or_else(|| "视觉绑定模型未返回工具调用".to_string()) + }) + .and_then(|arguments| parse_limited_llm_tool_arguments(&arguments)) + .and_then(|args| { + serde_json::from_value::(args) + .map_err(|e| format!("视觉绑定结果无效：{e}")) + }) + .and_then(|parsed| validate_binding_response(&parsed, nodes).map(|_| parsed)); + match result { + Ok(value) => return Ok(value), + Err(error) if attempt == 0 => repair = Some(error), + Err(error) => return Err(error), + } + } + Err("视觉绑定失败".to_string()) +} + +pub(crate) async fn separate_ui_impl( + project_path: String, + asset_id: String, + state: State, +) -> Result { + let session = current_platform_session().ok_or_else(|| "请先登录平台账号".to_string())?; + let root = Path::new(project_path.trim()); + let sidecar = separation_sidecar_dir(root, &asset_id)?; + fs::create_dir_all(&sidecar).map_err(|e| format!("创建 separation sidecar 失败：{e}"))?; + let state_path = sidecar.join("state.json"); + let mut separation = if state_path.exists() { + read_separation_state(&state_path)? + } else { + construct_separation_state(&state) + }; + for (tree_index, tree) in separation.unprocessed_trees.clone().iter().enumerate() { + let image = state + .ui_design_images + .get(&tree.src_ui_design) + .ok_or_else(|| "缺少源界面图".to_string())?; + let source_path = crate::project::resolve_local_project_path(root, &image.path)?; + let source_url = read_ui_reference_image_data_url(source_path).await?; + write_separation_state(&state_path, &separation)?; + loop { + let Some(current_tree) = separation.unprocessed_trees.get(tree_index) else { + break; + }; + let batch = next_leaf_batch(current_tree); + if batch.is_empty() { + break; + } + let prompt = + gen_extract_prompt(&batch.iter().map(|n| n.note.clone()).collect::>()); + let marker_path = sidecar.join(format!("marked-{}.png", separation.bound.len())); + let marked_url = build_marked_image(&source_url, &batch, &marker_path)?; + let processed_url = raw_image_edit( + &session, + &marked_url, + &prompt, + image.pixel_size.x as u32, + image.pixel_size.y as u32, + ) + .await?; + let processed_path = sidecar.join(format!("processed-{}.png", separation.bound.len())); + let processed_bytes = base64::engine::general_purpose::STANDARD + .decode( + processed_url + .split_once(',') + .map(|(_, d)| d) + .unwrap_or_default(), + ) + .map_err(|e| e.to_string())?; + fs::write(&processed_path, processed_bytes).map_err(|e| e.to_string())?; + let binding = visual_binding(source_url.clone(), processed_url, &batch).await?; + let mut cut_paths = std::collections::HashMap::new(); + for decision in &binding.decisions { + if let BindingDecision::Ok { + to_node, + separated_image_area, + } = decision + { + let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str())); + cut_processed_image(&processed_path, separated_image_area, &cut_path)?; + cut_paths.insert(to_node.clone(), cut_path.to_string_lossy().to_string()); + } + } + apply_batch_patch(&mut separation, tree_index, &binding.decisions, &cut_paths)?; + write_separation_state(&state_path, &separation)?; + } + } + let _ = fs::remove_file(state_path); + Ok(separation_dto(&separation)) +} + +fn cut_processed_image(source: &Path, area: &BindingArea, target: &Path) -> Result<(), String> { + let image = image::open(source).map_err(|e| format!("读取处理图失败：{e}"))?; + area.validate_in(image.width(), image.height())?; + let cropped = image.crop_imm( + area.global_pos_x_px, + area.global_pos_y_px, + area.width_px, + area.height_px, + ); + cropped + .save_with_format(target, ImageFormat::Png) + .map_err(|e| format!("写入 cut 图片失败：{e}")) +} + +fn remove_ids(node: &mut SeparationNode, ids: &std::collections::HashSet) { + node.children.retain(|child| !ids.contains(&child.id)); + for child in &mut node.children { + remove_ids(child, ids); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ui_editor::component::image::{ImageComponent, ImageType}; + use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode; + use crate::ui_editor::layout::control_layout::ControlLayout; + use crate::ui_editor::layout::node::{NodeMetadata, NodeSource, StageStatus}; + use crate::ui_editor::resource::ui_design_image::UIDesignImage; + use nalgebra::Vector2; + use std::collections::HashMap; + use typed_floats::tf32::StrictlyPositiveFinite; + + fn node(id: &str, components: Vec, children: Vec) -> Node { + Node { + id: NodeId::new(id).unwrap(), + layout: ControlLayout::default(), + metadata: NodeMetadata { + name: id.to_string(), + description: String::new(), + layout_status: StageStatus::NoProblem, + components_status: StageStatus::NoProblem, + allow_llm_edit_layout: true, + allow_llm_edit_component: true, + source: NodeSource::Llm, + }, + components, + children_display_mode: ChildrenDisplayMode::Stack, + children, + } + } + fn state(root: Node) -> State { + let image_id = UIDesignImageId::new("page").unwrap(); + State { + ui_trees: vec![UITree { + src_ui_design: image_id.clone(), + root, + }], + ui_design_images: HashMap::from([( + image_id, + UIDesignImage { + metadata: crate::ui_editor::resource::ui_design_image::UIDesignImageMetadata { + name: "page".to_string(), + description: String::new(), + role: None, + slave_to: None, + }, + path: "page.png".to_string(), + pixel_size: Vector2::new(100.0, 100.0), + pixels_per_unit: StrictlyPositiveFinite::new(1.0).unwrap(), + }, + )]), + sprite_assets: HashMap::new(), + font_assets: HashMap::new(), + } + } + #[test] + fn construction_filters_pure_nodes_and_passes_children_through() { + let image = Component::Image(ImageComponent { + target_graphic: None, + image_type: ImageType::Simple { + preserve_aspect: false, + }, + }); + let root = node( + "root", + vec![], + vec![node( + "container", + vec![], + vec![node("image", vec![image], vec![])], + )], + ); + let result = construct_separation_state(&state(root)); + assert_eq!( + result.unprocessed_trees[0].root.children[0].id.as_str(), + "image" + ); + } + #[test] + fn binding_validation_requires_exact_batch_coverage() { + let node = SeparationNode { + id: NodeId::new("image").unwrap(), + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 1, + height_px: 1, + note: SeparationNote { + description: "image".to_string(), + }, + children: vec![], + rework_count: 0, + }; + assert!(validate_binding_response(&BindingResp { decisions: vec![] }, &[&node]).is_err()); + } + #[test] + fn sidecar_name_uses_asset_id_digest() { + let dir = separation_sidecar_dir(Path::new("/tmp/project"), "ui:1").unwrap(); + assert!(dir.to_string_lossy().contains("ui_1-")); + assert!(dir.to_string_lossy().ends_with("-separation")); + } + #[test] + fn patch_collects_bound_and_removes_leaf() { + let image = Component::Image(ImageComponent { + target_graphic: None, + image_type: ImageType::Simple { + preserve_aspect: false, + }, + }); + let mut state = construct_separation_state(&state(node( + "root", + vec![], + vec![node("image", vec![image], vec![])], + ))); + let id = NodeId::new("image").unwrap(); + let decisions = vec![BindingDecision::Ok { + to_node: id.clone(), + separated_image_area: BindingArea { + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 1, + height_px: 1, + }, + }]; + let paths = HashMap::from([(id.clone(), "ui/.sidecar/cut.png".to_string())]); + apply_batch_patch(&mut state, 0, &decisions, &paths).unwrap(); + assert_eq!(state.bound[0].node_id, id); + assert!(state.unprocessed_trees[0].root.children.is_empty()); + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs index 04b72c0b3..2384476b0 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/persistence.rs @@ -196,7 +196,7 @@ pub(crate) fn generate_ui_design_code_at( }) } -fn generated_file_stem(asset_id: &str) -> String { +pub(crate) fn generated_file_stem(asset_id: &str) -> String { let mut stem = String::new(); for character in asset_id.chars() { if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') {