From 109cc81740b33cfe01db35068a7d506e097e1904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Tue, 8 Sep 2026 23:44:04 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8B=86=E5=88=86UI=E8=87=AA=E5=8A=A8=E5=88=86?= =?UTF-8?q?=E7=A6=BB=E5=91=BD=E4=BB=A4=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留原始图片分离与绑定提示词 提取模型、树、持久化和工作流子模块 新增可配置 more_turn 反馈重试 harness --- .../src/ui_editor/commands/separation.rs | 900 ------------------ .../src/ui_editor/commands/separation/mod.rs | 140 +++ .../ui_editor/commands/separation/model.rs | 123 +++ .../commands/separation/persistence.rs | 49 + .../ui_editor/commands/separation/prompt.rs | 52 + .../src/ui_editor/commands/separation/tree.rs | 158 +++ .../ui_editor/commands/separation/workflow.rs | 420 ++++++++ .../src-tauri/src/ui_editor/commands/utils.rs | 49 + 8 files changed, 991 insertions(+), 900 deletions(-) delete mode 100644 apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation.rs create mode 100644 apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs create mode 100644 apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model.rs create mode 100644 apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/persistence.rs create mode 100644 apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs create mode 100644 apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/tree.rs create mode 100644 apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation.rs deleted file mode 100644 index fb65a02c5..000000000 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation.rs +++ /dev/null @@ -1,900 +0,0 @@ -use crate::config::{build_game_creator_llm_client_from_llm_config, load_game_creator_app_config}; -use crate::platform_session::current_platform_session; -use crate::ui_editor::commands::utils::{ - parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm, - strict_json_schema, -}; -use crate::ui_editor::component::image::ImageComponent; -use crate::ui_editor::component::Component; -use crate::ui_editor::layout::node::Node; -use crate::ui_editor::state::{State, UITree}; -use crate::ui_editor::utils::{NodeId, UIDesignImageId}; -use base64::Engine as _; -use image::ImageFormat; -use platform_llm::{ - LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice, -}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use std::fs; -use std::path::{Path, PathBuf}; -use ts_rs::TS; - -pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1"; -pub const MAX_REWORK_COUNT: u32 = 3; - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)] -#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] -pub struct SeparationNote { - pub description: String, -} - -impl SeparationNote { - pub fn as_prompt(&self) -> String { - format!("- {}", self.description.trim()) - } -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] -#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] -pub struct SeparationNode { - pub id: NodeId, - pub global_pos_x_px: u32, - pub global_pos_y_px: u32, - pub width_px: u32, - pub height_px: u32, - pub note: SeparationNote, - pub children: Vec, - pub rework_count: u32, -} - -impl SeparationNode { - pub fn as_prompt(&self) -> String { - format!( - "node_id={} area=({}, {}, {}, {}) {}", - self.id.as_str(), - self.global_pos_x_px, - self.global_pos_y_px, - self.width_px, - self.height_px, - self.note.as_prompt() - ) - } -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] -#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] -pub struct SeparationTree { - pub src_ui_design: UIDesignImageId, - pub root: SeparationNode, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] -#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] -pub struct BoundNode { - pub node_id: NodeId, - pub cut_image_path: String, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] -#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] -pub struct ProblematicNode { - pub node_id: NodeId, - pub problem_description: String, - pub rework_count: u32, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] -#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] -pub struct SeparationState { - pub schema_version: String, - pub unprocessed_trees: Vec, - pub bound: Vec, - pub problematic_nodes: Vec, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] -#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] -pub struct SeparationDTO { - pub bound_nodes: Vec, - pub problematic_nodes: Vec, -} - -#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] -#[schemars(deny_unknown_fields)] -pub struct BindingArea { - pub global_pos_x_px: u32, - pub global_pos_y_px: u32, - pub width_px: u32, - pub height_px: u32, -} - -impl BindingArea { - pub fn validate_in(&self, image_width: u32, image_height: u32) -> Result<(), String> { - if self.width_px == 0 || self.height_px == 0 { - return Err("BindingArea 宽度和高度必须大于 0".to_string()); - } - let max_x = self - .global_pos_x_px - .checked_add(self.width_px) - .ok_or_else(|| "BindingArea 横向范围溢出".to_string())?; - let max_y = self - .global_pos_y_px - .checked_add(self.height_px) - .ok_or_else(|| "BindingArea 纵向范围溢出".to_string())?; - if max_x > image_width || max_y > image_height { - return Err("BindingArea 超出处理图边界".to_string()); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] -#[schemars(deny_unknown_fields)] -pub enum BindingDecision { - Ok { - separated_image_area: BindingArea, - to_node: NodeId, - }, - NeedRework { - problem_description: String, - to_node: NodeId, - }, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] -#[schemars(deny_unknown_fields)] -pub struct BindingResp { - pub decisions: Vec, -} - -const SHARED_SEPARATION_REQ: &str = r#" -MUST hard edges; preserve no glow/blur beyond the exact visible shape. -NEVER keep its parent's background with it. -UI elements marked with GREEN line frames are extraction marks only; never include the frame. -PURPLE filled areas represent removed elements; reconstruct the background under them. -"#; - -pub fn gen_extract_prompt(separation_notes: &[SeparationNote]) -> String { - let mut result = format!("This is a UI design image, not a normal photo/illustration.\nExtract distinct UI elements as independent layers with clean edges and full transparency outside each element.\nKeep every element at its original position on a transparent canvas.\n{}\nElements to extract:\n", SHARED_SEPARATION_REQ); - for note in separation_notes { - result.push_str(¬e.as_prompt()); - result.push('\n'); - } - result -} - -pub fn gen_binding_prompt(nodes: &[&SeparationNode]) -> String { - let mut result = format!("You are reviewing a UI elements separation result.\n{}\nThe source and processed images use top-left pixel coordinates.\nReturn one decision for every requested node. For a failed node, provide a short repair description.\nNodes:\n", SHARED_SEPARATION_REQ); - for node in nodes { - result.push_str(&node.as_prompt()); - result.push('\n'); - } - result -} - -fn is_unbound_image(node: &Node) -> bool { - node.components.iter().any(|component| { - matches!( - component, - Component::Image(ImageComponent { - target_graphic: None, - .. - }) - ) - }) -} - -fn node_pixel_rect( - node: &Node, - parent: &crate::ui_editor::layout::dimension::UIRect, - ppu: f32, -) -> (u32, u32, u32, u32) { - let rect = node.layout.transform.resolve(parent); - let x = (rect.min.x * ppu).max(0.0).round() as u32; - let y = (rect.min.y * ppu).max(0.0).round() as u32; - let w = (rect.size.x * ppu).max(0.0).round() as u32; - let h = (rect.size.y * ppu).max(0.0).round() as u32; - (x, y, w, h) -} - -fn node_description(node: &Node) -> String { - let name = node.metadata.name.trim(); - let description = node.metadata.description.trim(); - match (name.is_empty(), description.is_empty()) { - (true, true) => "未命名 UI 图片元素".to_string(), - (false, true) => name.to_string(), - (true, false) => description.to_string(), - (false, false) => format!("{name}：{description}"), - } -} - -fn collect_todo_nodes( - node: &Node, - parent: &crate::ui_editor::layout::dimension::UIRect, - ppu: f32, - output: &mut Vec, -) { - let mut children = Vec::new(); - let rect = node.layout.transform.resolve(parent); - for child in &node.children { - collect_todo_nodes(child, &rect, ppu, &mut children); - } - if is_unbound_image(node) { - let (x, y, w, h) = node_pixel_rect(node, parent, ppu); - output.push(SeparationNode { - id: node.id.clone(), - global_pos_x_px: x, - global_pos_y_px: y, - width_px: w, - height_px: h, - note: SeparationNote { - description: node_description(node), - }, - children, - rework_count: 0, - }); - } else { - output.extend(children); - } -} - -pub fn construct_separation_state(state: &State) -> SeparationState { - let unprocessed_trees = state - .ui_trees - .iter() - .filter_map(|tree| { - let image = state.ui_design_images.get(&tree.src_ui_design)?; - let ppu = image.pixels_per_unit.get(); - let size = image.pixel_size / ppu; - let root_rect = - crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size); - let mut children = Vec::new(); - collect_todo_nodes(&tree.root, &root_rect, ppu, &mut children); - (!children.is_empty()).then(|| SeparationTree { - src_ui_design: tree.src_ui_design.clone(), - root: SeparationNode { - id: tree.root.id.clone(), - global_pos_x_px: 0, - global_pos_y_px: 0, - width_px: image.pixel_size.x.max(0.0).round() as u32, - height_px: image.pixel_size.y.max(0.0).round() as u32, - note: SeparationNote::default(), - children, - rework_count: 0, - }, - }) - }) - .collect(); - SeparationState { - schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(), - unprocessed_trees, - bound: Vec::new(), - problematic_nodes: Vec::new(), - } -} - -pub fn next_leaf_batch(tree: &SeparationTree) -> Vec<&SeparationNode> { - fn leaves<'a>(node: &'a SeparationNode, output: &mut Vec<&'a SeparationNode>) { - if node.children.is_empty() { - output.push(node); - } else { - for child in &node.children { - leaves(child, output); - } - } - } - let mut output = Vec::new(); - leaves(&tree.root, &mut output); - output -} - -pub fn validate_binding_response( - response: &BindingResp, - batch: &[&SeparationNode], -) -> Result<(), String> { - let expected = batch - .iter() - .map(|node| node.id.clone()) - .collect::>(); - let mut seen = std::collections::HashSet::new(); - for decision in &response.decisions { - let node_id = match decision { - BindingDecision::Ok { to_node, .. } | BindingDecision::NeedRework { to_node, .. } => { - to_node - } - }; - if !expected.contains(node_id) { - return Err(format!("视觉绑定返回了未知节点：{}", node_id.as_str())); - } - if !seen.insert(node_id.clone()) { - return Err(format!("视觉绑定重复返回节点：{}", node_id.as_str())); - } - if let BindingDecision::NeedRework { - problem_description, - .. - } = decision - { - if problem_description.trim().is_empty() { - return Err("NeedRework 必须包含问题描述".to_string()); - } - } - } - if seen.len() != expected.len() { - return Err("视觉绑定未覆盖当前 batch 的全部节点".to_string()); - } - Ok(()) -} - -pub fn separation_sidecar_dir(root: &Path, asset_id: &str) -> Result { - if asset_id.trim().is_empty() || asset_id.trim() != asset_id { - return Err("UI 资源 ID 无效".to_string()); - } - let dir = root.join("ui").join(format!( - ".{}-separation", - crate::ui_editor::persistence::generated_file_stem(asset_id) - )); - if !dir.starts_with(root) { - return Err("separation sidecar 路径越界".to_string()); - } - Ok(dir) -} - -pub fn write_separation_state(path: &Path, state: &SeparationState) -> Result<(), String> { - if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION { - return Err("不支持的 separation state schema".to_string()); - } - let bytes = serde_json::to_vec_pretty(state) - .map_err(|error| format!("序列化 separation state 失败：{error}"))?; - let parent = path - .parent() - .ok_or_else(|| "separation state 路径缺少父目录".to_string())?; - fs::create_dir_all(parent).map_err(|error| format!("创建 separation sidecar 失败：{error}"))?; - let temporary = path.with_extension("json.tmp"); - fs::write(&temporary, bytes).map_err(|error| format!("写入 separation state 失败：{error}"))?; - fs::rename(&temporary, path).map_err(|error| format!("安装 separation state 失败：{error}")) -} - -pub fn read_separation_state(path: &Path) -> Result { - let bytes = fs::read(path).map_err(|error| format!("读取 separation state 失败：{error}"))?; - let state: SeparationState = serde_json::from_slice(&bytes) - .map_err(|error| format!("解析 separation state 失败：{error}"))?; - if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION { - return Err("不支持的 separation state schema".to_string()); - } - Ok(state) -} - -pub fn separation_dto(state: &SeparationState) -> SeparationDTO { - SeparationDTO { - bound_nodes: state.bound.clone(), - problematic_nodes: state.problematic_nodes.clone(), - } -} - -pub fn apply_batch_patch( - state: &mut SeparationState, - tree_index: usize, - decisions: &[BindingDecision], - cut_paths: &std::collections::HashMap, -) -> Result<(), String> { - let tree = state - .unprocessed_trees - .get_mut(tree_index) - .ok_or_else(|| "separation tree 索引无效".to_string())?; - let batch = next_leaf_batch(tree); - validate_binding_response( - &BindingResp { - decisions: decisions.to_vec(), - }, - &batch, - )?; - let rework_counts = batch - .iter() - .map(|node| (node.id.clone(), node.rework_count)) - .collect::>(); - let mut ids = std::collections::HashSet::new(); - for decision in decisions { - match decision { - BindingDecision::Ok { to_node, .. } => { - let path = cut_paths - .get(to_node) - .ok_or_else(|| format!("缺少节点 {} 的 cut 图片", to_node.as_str()))?; - state.bound.push(BoundNode { - node_id: to_node.clone(), - cut_image_path: path.clone(), - }); - ids.insert(to_node.clone()); - } - BindingDecision::NeedRework { - to_node, - problem_description, - } => { - let count = rework_counts.get(to_node).copied().unwrap_or(0) + 1; - if count >= MAX_REWORK_COUNT { - state.problematic_nodes.push(ProblematicNode { - node_id: to_node.clone(), - problem_description: problem_description.clone(), - rework_count: count, - }); - ids.insert(to_node.clone()); - } else { - increment_rework_count(&mut tree.root, to_node, count); - } - } - } - } - remove_ids(&mut tree.root, &ids); - Ok(()) -} - -fn mark_batch_problematic( - state: &mut SeparationState, - tree_index: usize, - batch: &[&SeparationNode], - error: String, -) { - let ids = batch - .iter() - .map(|node| node.id.clone()) - .collect::>(); - for node in batch { - state.problematic_nodes.push(ProblematicNode { - node_id: node.id.clone(), - problem_description: error.clone(), - rework_count: MAX_REWORK_COUNT, - }); - } - if let Some(tree) = state.unprocessed_trees.get_mut(tree_index) { - remove_ids(&mut tree.root, &ids); - } -} - -fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) { - if node.id == *id { - node.rework_count = count; - return; - } - for child in &mut node.children { - increment_rework_count(child, id, count); - } -} - -#[derive(Deserialize)] -struct RawEditResponse { - data: Vec, -} -#[derive(Deserialize)] -struct RawEditItem { - b64_json: String, -} - -async fn raw_image_edit( - session: &crate::platform_session::PlatformSessionSnapshot, - image_data_url: &str, - prompt: &str, - width: u32, - height: u32, -) -> Result { - let (mime, data) = image_data_url - .split_once(",") - .ok_or_else(|| "界面图 data URL 无效".to_string())?; - let mime = mime - .strip_prefix("data:") - .and_then(|v| v.strip_suffix(";base64")) - .unwrap_or("image/png"); - let client = crate::http_client::agc_main_site_client_builder() - .build() - .map_err(|e| format!("创建图片编辑客户端失败：{e}"))?; - let url = format!( - "{}/api/raw/v1/images/edit", - session.api_base_url.trim_end_matches('/') - ); - let body = serde_json::json!({ - "image": {"data": data, "mimeType": mime}, - "prompt": prompt, - "width": width, - "height": height, - "output_format": "png", - "background": "transparent" - }); - let response = crate::http_client::with_agc_main_site_marker( - client - .post(url) - .bearer_auth(&session.access_token) - .json(&body), - ) - .send() - .await - .map_err(|e| format!("图片分离请求失败：{e}"))?; - if !response.status().is_success() { - return Err(format!("图片分离请求失败（HTTP {}）", response.status())); - } - let payload = response - .json::() - .await - .map_err(|e| format!("解析图片分离响应失败：{e}"))?; - payload - .data - .into_iter() - .next() - .map(|item| format!("data:image/png;base64,{}", item.b64_json)) - .ok_or_else(|| "图片分离响应没有图像".to_string()) -} - -fn build_marked_image( - source_url: &str, - nodes: &[&SeparationNode], - target: &Path, -) -> Result { - let encoded = source_url - .split_once(',') - .map(|(_, d)| d) - .ok_or_else(|| "源图 data URL 无效".to_string())?; - let bytes = base64::engine::general_purpose::STANDARD - .decode(encoded) - .map_err(|e| format!("解码源图失败：{e}"))?; - let mut image = image::load_from_memory(&bytes) - .map_err(|e| format!("读取源图失败：{e}"))? - .to_rgba8(); - let width = image.width(); - let height = image.height(); - for node in nodes { - let x0 = node.global_pos_x_px.min(width.saturating_sub(1)); - let y0 = node.global_pos_y_px.min(height.saturating_sub(1)); - let x1 = node - .global_pos_x_px - .saturating_add(node.width_px) - .min(width) - .saturating_sub(1); - let y1 = node - .global_pos_y_px - .saturating_add(node.height_px) - .min(height) - .saturating_sub(1); - if x0 >= x1 || y0 >= y1 { - continue; - } - for x in x0..=x1 { - image.put_pixel(x, y0, image::Rgba([0, 255, 0, 255])); - image.put_pixel(x, y1, image::Rgba([0, 255, 0, 255])); - } - for y in y0..=y1 { - image.put_pixel(x0, y, image::Rgba([0, 255, 0, 255])); - image.put_pixel(x1, y, image::Rgba([0, 255, 0, 255])); - } - for y in y0..=y1 { - for x in x0..=x1 { - if x > x0 && x < x1 && y > y0 && y < y1 { - image.put_pixel(x, y, image::Rgba([180, 0, 180, 120])); - } - } - } - } - image::DynamicImage::ImageRgba8(image.clone()) - .save_with_format(target, image::ImageFormat::Png) - .map_err(|e| format!("写入标记图失败：{e}"))?; - let mut png = Vec::new(); - image::DynamicImage::ImageRgba8(image) - .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) - .map_err(|e| format!("编码标记图失败：{e}"))?; - Ok(format!( - "data:image/png;base64,{}", - base64::engine::general_purpose::STANDARD.encode(png) - )) -} - -async fn visual_binding( - source_url: String, - processed_url: String, - nodes: &[&SeparationNode], -) -> Result { - let llm_config = load_game_creator_app_config() - .map_err(|e| e.to_string())? - .llm; - let client = build_game_creator_llm_client_from_llm_config(&llm_config, "llm") - .map_err(|e| e.to_string())?; - let schema = strict_json_schema::()?; - let tool = LlmFunctionTool::new( - "bind_ui_elements", - "确认处理图中的区域对应哪些 UI 节点", - schema, - ) - .with_strict(true); - let base_prompt = gen_binding_prompt(nodes); - let mut repair = None; - for attempt in 0..2 { - let prompt = repair.as_ref().map_or_else( - || base_prompt.clone(), - |error: &String| format!("{base_prompt}\n上一次输出错误：{error}\n请修正并完整返回。"), - ); - let request = LlmRunRequest::new(vec![ - LlmMessage::system("你是 UI 图片视觉绑定器。只根据图像判断区域，不做 OCR。"), - LlmMessage::user_multimodal(vec![ - LlmMessageContentPart::InputText { text: prompt }, - LlmMessageContentPart::InputImage { - image_url: source_url.clone(), - }, - LlmMessageContentPart::InputImage { - image_url: processed_url.clone(), - }, - ]), - ]) - .with_function_tools(vec![tool.clone()]) - .with_tool_choice(LlmToolChoice::Required); - let result = request_ui_editor_llm(&client, &llm_config, request) - .await - .map_err(|e| e.to_string()) - .and_then(|response| { - response - .tool_calls - .into_iter() - .find(|call| call.name == "bind_ui_elements") - .map(|call| call.arguments) - .ok_or_else(|| "视觉绑定模型未返回工具调用".to_string()) - }) - .and_then(|arguments| parse_limited_llm_tool_arguments(&arguments)) - .and_then(|args| { - serde_json::from_value::(args) - .map_err(|e| format!("视觉绑定结果无效：{e}")) - }) - .and_then(|parsed| validate_binding_response(&parsed, nodes).map(|_| parsed)); - match result { - Ok(value) => return Ok(value), - Err(error) if attempt == 0 => repair = Some(error), - Err(error) => return Err(error), - } - } - Err("视觉绑定失败".to_string()) -} - -pub(crate) async fn separate_ui_impl( - project_path: String, - asset_id: String, - state: State, -) -> Result { - let session = current_platform_session().ok_or_else(|| "请先登录平台账号".to_string())?; - let root = Path::new(project_path.trim()); - let sidecar = separation_sidecar_dir(root, &asset_id)?; - fs::create_dir_all(&sidecar).map_err(|e| format!("创建 separation sidecar 失败：{e}"))?; - let state_path = sidecar.join("state.json"); - let mut separation = if state_path.exists() { - read_separation_state(&state_path)? - } else { - construct_separation_state(&state) - }; - for (tree_index, tree) in separation.unprocessed_trees.clone().iter().enumerate() { - let image = state - .ui_design_images - .get(&tree.src_ui_design) - .ok_or_else(|| "缺少源界面图".to_string())?; - let source_path = crate::project::resolve_local_project_path(root, &image.path)?; - let source_url = read_ui_reference_image_data_url(source_path).await?; - write_separation_state(&state_path, &separation)?; - loop { - let Some(current_tree) = separation.unprocessed_trees.get(tree_index) else { - break; - }; - let batch_nodes = next_leaf_batch(current_tree) - .into_iter() - .cloned() - .collect::>(); - if batch_nodes.is_empty() { - break; - } - let batch = batch_nodes.iter().collect::>(); - let prompt = - gen_extract_prompt(&batch.iter().map(|n| n.note.clone()).collect::>()); - let marker_path = sidecar.join(format!("marked-{}.png", separation.bound.len())); - let marked_url = match build_marked_image(&source_url, &batch, &marker_path) { - Ok(value) => value, - Err(error) => { - mark_batch_problematic(&mut separation, tree_index, &batch, error); - write_separation_state(&state_path, &separation)?; - continue; - } - }; - let processed_url = match raw_image_edit( - &session, - &marked_url, - &prompt, - image.pixel_size.x as u32, - image.pixel_size.y as u32, - ) - .await - { - Ok(value) => value, - Err(error) => { - mark_batch_problematic(&mut separation, tree_index, &batch, error); - write_separation_state(&state_path, &separation)?; - continue; - } - }; - let processed_path = sidecar.join(format!("processed-{}.png", separation.bound.len())); - let processed_bytes = base64::engine::general_purpose::STANDARD - .decode( - processed_url - .split_once(',') - .map(|(_, d)| d) - .unwrap_or_default(), - ) - .map_err(|e| e.to_string())?; - fs::write(&processed_path, processed_bytes).map_err(|e| e.to_string())?; - let binding = match visual_binding(source_url.clone(), processed_url, &batch).await { - Ok(value) => value, - Err(error) => { - mark_batch_problematic(&mut separation, tree_index, &batch, error); - write_separation_state(&state_path, &separation)?; - continue; - } - }; - let mut cut_paths = std::collections::HashMap::new(); - for decision in &binding.decisions { - if let BindingDecision::Ok { - to_node, - separated_image_area, - } = decision - { - let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str())); - cut_processed_image(&processed_path, separated_image_area, &cut_path)?; - cut_paths.insert(to_node.clone(), cut_path.to_string_lossy().to_string()); - } - } - apply_batch_patch(&mut separation, tree_index, &binding.decisions, &cut_paths)?; - write_separation_state(&state_path, &separation)?; - } - } - let _ = fs::remove_file(state_path); - Ok(separation_dto(&separation)) -} - -fn cut_processed_image(source: &Path, area: &BindingArea, target: &Path) -> Result<(), String> { - let image = image::open(source).map_err(|e| format!("读取处理图失败：{e}"))?; - area.validate_in(image.width(), image.height())?; - let cropped = image.crop_imm( - area.global_pos_x_px, - area.global_pos_y_px, - area.width_px, - area.height_px, - ); - cropped - .save_with_format(target, ImageFormat::Png) - .map_err(|e| format!("写入 cut 图片失败：{e}")) -} - -fn remove_ids(node: &mut SeparationNode, ids: &std::collections::HashSet) { - node.children.retain(|child| !ids.contains(&child.id)); - for child in &mut node.children { - remove_ids(child, ids); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ui_editor::component::image::{ImageComponent, ImageType}; - use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode; - use crate::ui_editor::layout::control_layout::ControlLayout; - use crate::ui_editor::layout::node::{NodeMetadata, NodeSource, StageStatus}; - use crate::ui_editor::resource::ui_design_image::UIDesignImage; - use nalgebra::Vector2; - use std::collections::HashMap; - use typed_floats::tf32::StrictlyPositiveFinite; - - fn node(id: &str, components: Vec, children: Vec) -> Node { - Node { - id: NodeId::new(id).unwrap(), - layout: ControlLayout::default(), - metadata: NodeMetadata { - name: id.to_string(), - description: String::new(), - layout_status: StageStatus::NoProblem, - components_status: StageStatus::NoProblem, - allow_llm_edit_layout: true, - allow_llm_edit_component: true, - source: NodeSource::Llm, - }, - components, - children_display_mode: ChildrenDisplayMode::Stack, - children, - } - } - fn state(root: Node) -> State { - let image_id = UIDesignImageId::new("page").unwrap(); - State { - ui_trees: vec![UITree { - src_ui_design: image_id.clone(), - root, - }], - ui_design_images: HashMap::from([( - image_id, - UIDesignImage { - metadata: crate::ui_editor::resource::ui_design_image::UIDesignImageMetadata { - name: "page".to_string(), - description: String::new(), - role: None, - slave_to: None, - }, - path: "page.png".to_string(), - pixel_size: Vector2::new(100.0, 100.0), - pixels_per_unit: StrictlyPositiveFinite::new(1.0).unwrap(), - }, - )]), - sprite_assets: HashMap::new(), - font_assets: HashMap::new(), - } - } - #[test] - fn construction_filters_pure_nodes_and_passes_children_through() { - let image = Component::Image(ImageComponent { - target_graphic: None, - image_type: ImageType::Simple { - preserve_aspect: false, - }, - }); - let root = node( - "root", - vec![], - vec![node( - "container", - vec![], - vec![node("image", vec![image], vec![])], - )], - ); - let result = construct_separation_state(&state(root)); - assert_eq!( - result.unprocessed_trees[0].root.children[0].id.as_str(), - "image" - ); - } - #[test] - fn binding_validation_requires_exact_batch_coverage() { - let node = SeparationNode { - id: NodeId::new("image").unwrap(), - global_pos_x_px: 0, - global_pos_y_px: 0, - width_px: 1, - height_px: 1, - note: SeparationNote { - description: "image".to_string(), - }, - children: vec![], - rework_count: 0, - }; - assert!(validate_binding_response(&BindingResp { decisions: vec![] }, &[&node]).is_err()); - } - #[test] - fn sidecar_name_uses_asset_id_digest() { - let dir = separation_sidecar_dir(Path::new("/tmp/project"), "ui:1").unwrap(); - assert!(dir.to_string_lossy().contains("ui_1-")); - assert!(dir.to_string_lossy().ends_with("-separation")); - } - #[test] - fn patch_collects_bound_and_removes_leaf() { - let image = Component::Image(ImageComponent { - target_graphic: None, - image_type: ImageType::Simple { - preserve_aspect: false, - }, - }); - let mut state = construct_separation_state(&state(node( - "root", - vec![], - vec![node("image", vec![image], vec![])], - ))); - let id = NodeId::new("image").unwrap(); - let decisions = vec![BindingDecision::Ok { - to_node: id.clone(), - separated_image_area: BindingArea { - global_pos_x_px: 0, - global_pos_y_px: 0, - width_px: 1, - height_px: 1, - }, - }]; - let paths = HashMap::from([(id.clone(), "ui/.sidecar/cut.png".to_string())]); - apply_batch_patch(&mut state, 0, &decisions, &paths).unwrap(); - assert_eq!(state.bound[0].node_id, id); - assert!(state.unprocessed_trees[0].root.children.is_empty()); - } -} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs new file mode 100644 index 000000000..d103da6e1 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs @@ -0,0 +1,140 @@ +mod model; +mod persistence; +mod prompt; +mod tree; +mod workflow; + +pub use model::*; +pub use persistence::*; +pub use tree::*; +pub(crate) use workflow::separate_ui_impl; +#[cfg(test)] +mod tests { + use super::*; + use crate::ui_editor::component::image::{ImageComponent, ImageType}; + use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode; + use crate::ui_editor::layout::control_layout::ControlLayout; + use crate::ui_editor::layout::node::{NodeMetadata, NodeSource, StageStatus}; + use crate::ui_editor::resource::ui_design_image::UIDesignImage; + use nalgebra::Vector2; + use std::collections::HashMap; + use typed_floats::tf32::StrictlyPositiveFinite; + + fn node(id: &str, components: Vec, children: Vec) -> Node { + Node { + id: NodeId::new(id).unwrap(), + layout: ControlLayout::default(), + metadata: NodeMetadata { + name: id.to_string(), + description: String::new(), + layout_status: StageStatus::NoProblem, + components_status: StageStatus::NoProblem, + allow_llm_edit_layout: true, + allow_llm_edit_component: true, + source: NodeSource::Llm, + }, + components, + children_display_mode: ChildrenDisplayMode::Stack, + children, + } + } + fn state(root: Node) -> State { + let image_id = UIDesignImageId::new("page").unwrap(); + State { + ui_trees: vec![UITree { + src_ui_design: image_id.clone(), + root, + }], + ui_design_images: HashMap::from([( + image_id, + UIDesignImage { + metadata: crate::ui_editor::resource::ui_design_image::UIDesignImageMetadata { + name: "page".to_string(), + description: String::new(), + role: None, + slave_to: None, + }, + path: "page.png".to_string(), + pixel_size: Vector2::new(100.0, 100.0), + pixels_per_unit: StrictlyPositiveFinite::new(1.0).unwrap(), + }, + )]), + sprite_assets: HashMap::new(), + font_assets: HashMap::new(), + } + } + #[test] + fn construction_filters_pure_nodes_and_passes_children_through() { + let image = Component::Image(ImageComponent { + target_graphic: None, + image_type: ImageType::Simple { + preserve_aspect: false, + }, + }); + let root = node( + "root", + vec![], + vec![node( + "container", + vec![], + vec![node("image", vec![image], vec![])], + )], + ); + let result = construct_separation_state(&state(root)); + assert_eq!( + result.unprocessed_trees[0].root.children[0].id.as_str(), + "image" + ); + } + #[test] + fn binding_validation_requires_exact_batch_coverage() { + let node = SeparationNode { + id: NodeId::new("image").unwrap(), + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 1, + height_px: 1, + note: SeparationNote { + description: "image".to_string(), + text_note: String::new(), + }, + children: vec![], + rework_count: 0, + }; + assert!(validate_binding_response(&BindingResp { decisions: vec![] }, &[&node]).is_err()); + } + #[test] + fn sidecar_name_uses_asset_id_digest() { + let dir = separation_sidecar_dir(Path::new("/tmp/project"), "ui:1").unwrap(); + assert!(dir.to_string_lossy().contains("ui_1-")); + assert!(dir.to_string_lossy().ends_with("-separation")); + } + #[test] + fn patch_collects_bound_and_removes_leaf() { + let image = Component::Image(ImageComponent { + target_graphic: None, + image_type: ImageType::Simple { + preserve_aspect: false, + }, + }); + let mut state = construct_separation_state(&state(node( + "root", + vec![], + vec![node("image", vec![image], vec![])], + ))); + let id = NodeId::new("image").unwrap(); + let decisions = vec![BindingDecision::Ok { + to_node: id.clone(), + separated_image_area: BindingArea { + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: 1, + height_px: 1, + }, + }]; + let paths = HashMap::from([(id.clone(), "ui/.sidecar/cut.png".to_string())]); + apply_batch_patch(&mut state, 0, &decisions, &paths).unwrap(); + assert_eq!(state.bound[0].node_id, id); + assert!(state.unprocessed_trees[0].root.children.is_empty()); + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model.rs new file mode 100644 index 000000000..87ffc1832 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model.rs @@ -0,0 +1,123 @@ +use crate::ui_editor::utils::{NodeId, UIDesignImageId}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1"; +pub const MAX_REWORK_COUNT: u32 = 3; + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationNote { + pub description: String, + pub text_note: String, +} +impl SeparationNote { + pub fn as_prompt(&self) -> String { + format!("desc: {} {}", self.description, self.text_note) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationNode { + pub id: NodeId, + pub global_pos_x_px: u32, + pub global_pos_y_px: u32, + pub width_px: u32, + pub height_px: u32, + pub note: SeparationNote, + pub children: Vec, + pub rework_count: u32, +} +impl SeparationNode { + pub fn as_prompt(&self) -> String { + format!( + "node_id={} area=({}, {}, {}, {}) {}", + self.id.as_str(), + self.global_pos_x_px, + self.global_pos_y_px, + self.width_px, + self.height_px, + self.note.as_prompt() + ) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationTree { + pub src_ui_design: UIDesignImageId, + pub root: SeparationNode, +} +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct BoundNode { + pub node_id: NodeId, + pub cut_image_path: String, +} +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct ProblematicNode { + pub node_id: NodeId, + pub problem_description: String, + pub rework_count: u32, +} +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationState { + pub schema_version: String, + pub unprocessed_trees: Vec, + pub bound: Vec, + pub problematic_nodes: Vec, +} +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct SeparationDTO { + pub bound_nodes: Vec, + pub problematic_nodes: Vec, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub struct BindingArea { + pub global_pos_x_px: u32, + pub global_pos_y_px: u32, + pub width_px: u32, + pub height_px: u32, +} +impl BindingArea { + pub fn validate_in(&self, w: u32, h: u32) -> Result<(), String> { + if self.width_px == 0 || self.height_px == 0 { + return Err("BindingArea 宽度和高度必须大于 0".into()); + } + if self + .global_pos_x_px + .checked_add(self.width_px) + .is_none_or(|v| v > w) + || self + .global_pos_y_px + .checked_add(self.height_px) + .is_none_or(|v| v > h) + { + return Err("BindingArea 超出处理图边界".into()); + } + Ok(()) + } +} +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub enum BindingDecision { + Ok { + separated_image_area: BindingArea, + to_node: NodeId, + }, + NeedRework { + problem_description: String, + to_node: NodeId, + }, +} +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +pub struct BindingResp { + pub decisions: Vec, +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/persistence.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/persistence.rs new file mode 100644 index 000000000..5d391e2e4 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/persistence.rs @@ -0,0 +1,49 @@ +use super::model::*; +use crate::ui_editor::commands::separation::*; +use std::fs; +use std::path::{Path, PathBuf}; +pub fn separation_sidecar_dir(root: &Path, asset_id: &str) -> Result { + if asset_id.trim().is_empty() || asset_id.trim() != asset_id { + return Err("UI 资源 ID 无效".to_string()); + } + let dir = root.join("ui").join(format!( + ".{}-separation", + crate::ui_editor::persistence::generated_file_stem(asset_id) + )); + if !dir.starts_with(root) { + return Err("separation sidecar 路径越界".to_string()); + } + Ok(dir) +} + +pub fn write_separation_state(path: &Path, state: &SeparationState) -> Result<(), String> { + if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION { + return Err("不支持的 separation state schema".to_string()); + } + let bytes = serde_json::to_vec_pretty(state) + .map_err(|error| format!("序列化 separation state 失败：{error}"))?; + let parent = path + .parent() + .ok_or_else(|| "separation state 路径缺少父目录".to_string())?; + fs::create_dir_all(parent).map_err(|error| format!("创建 separation sidecar 失败：{error}"))?; + let temporary = path.with_extension("json.tmp"); + fs::write(&temporary, bytes).map_err(|error| format!("写入 separation state 失败：{error}"))?; + fs::rename(&temporary, path).map_err(|error| format!("安装 separation state 失败：{error}")) +} + +pub fn read_separation_state(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|error| format!("读取 separation state 失败：{error}"))?; + let state: SeparationState = serde_json::from_slice(&bytes) + .map_err(|error| format!("解析 separation state 失败：{error}"))?; + if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION { + return Err("不支持的 separation state schema".to_string()); + } + Ok(state) +} + +pub fn separation_dto(state: &SeparationState) -> SeparationDTO { + SeparationDTO { + bound_nodes: state.bound.clone(), + problematic_nodes: state.problematic_nodes.clone(), + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs new file mode 100644 index 000000000..3343ae1dc --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs @@ -0,0 +1,52 @@ +use crate::ui_editor::commands::separation::{SeparationNode, SeparationNote}; + +const SHARED_SEPARATION_REQ: &str = r#" + + MUST hard edges; preserve no glow/blur beyond the exact visible shape. + NEVER keep its parent's background with it. + + UI elements that needs to extract has been marked with GREEN line frames (only for mark purpose, NEVER wrap a frame in your extraction). + On some UI elements, there is some PURPLE filled area, they were removed UI elements, reconstruct the background under where they were. +"#; +pub(super) fn gen_extract_prompt(separation_notes: Vec) -> String { + let extract_system_prompt = format!( + r#" + This is a UI design image, not a normal photo/illustration. Extract it strictly as UI elements/layers, not as a generic foreground/background extraction. + Treat distinct UI element as its own layer with hard, clean, pixel-accurate edges and full transparency outside the element. + + MUST keep each element at its original position on a transparent canvas. + {SHARED_SEPARATION_REQ} + here are UI elements to extract: + + "# + ); + let mut result = extract_system_prompt; + result.reserve(512); + for elem in separation_notes { + result.push_str(&elem.as_prompt()); + result.push('\n'); + } + result +} +pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String { + let binding_system_prompt = format!( + r#" + You are working under a UI elements separation workflow. + You will be given a src UI design image and a processed image, where some ui elements are separated. + Here were the separation requirements: + ``` + {SHARED_SEPARATION_REQ} + ``` + You need to recognize and review the separation: + + these node need handle: + "# + ); + let mut result = binding_system_prompt; + result.reserve(512); + for elem in nodes { + result.push_str(&elem.as_prompt()); + result.push('\n'); + } + result +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/tree.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/tree.rs new file mode 100644 index 000000000..25b7cf6f3 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/tree.rs @@ -0,0 +1,158 @@ +use super::model::*; +use crate::ui_editor::component::{image::ImageComponent, Component}; +use crate::ui_editor::layout::node::Node; +use crate::ui_editor::state::{State, UITree}; +use crate::ui_editor::utils::NodeId; +fn is_unbound_image(node: &Node) -> bool { + node.components.iter().any(|component| { + matches!( + component, + Component::Image(ImageComponent { + target_graphic: None, + .. + }) + ) + }) +} + +fn node_pixel_rect( + node: &Node, + parent: &crate::ui_editor::layout::dimension::UIRect, + ppu: f32, +) -> (u32, u32, u32, u32) { + let rect = node.layout.transform.resolve(parent); + let x = (rect.min.x * ppu).max(0.0).round() as u32; + let y = (rect.min.y * ppu).max(0.0).round() as u32; + let w = (rect.size.x * ppu).max(0.0).round() as u32; + let h = (rect.size.y * ppu).max(0.0).round() as u32; + (x, y, w, h) +} + +fn node_description(node: &Node) -> String { + let name = node.metadata.name.trim(); + let description = node.metadata.description.trim(); + match (name.is_empty(), description.is_empty()) { + (true, true) => "未命名 UI 图片元素".to_string(), + (false, true) => name.to_string(), + (true, false) => description.to_string(), + (false, false) => format!("{name}：{description}"), + } +} + +fn collect_todo_nodes( + node: &Node, + parent: &crate::ui_editor::layout::dimension::UIRect, + ppu: f32, + output: &mut Vec, +) { + let mut children = Vec::new(); + let rect = node.layout.transform.resolve(parent); + for child in &node.children { + collect_todo_nodes(child, &rect, ppu, &mut children); + } + if is_unbound_image(node) { + let (x, y, w, h) = node_pixel_rect(node, parent, ppu); + output.push(SeparationNode { + id: node.id.clone(), + global_pos_x_px: x, + global_pos_y_px: y, + width_px: w, + height_px: h, + note: SeparationNote { + description: node_description(node), + text_note: String::new(), + }, + children, + rework_count: 0, + }); + } else { + output.extend(children); + } +} + +pub fn construct_separation_state(state: &State) -> SeparationState { + let unprocessed_trees = state + .ui_trees + .iter() + .filter_map(|tree| { + let image = state.ui_design_images.get(&tree.src_ui_design)?; + let ppu = image.pixels_per_unit.get(); + let size = image.pixel_size / ppu; + let root_rect = + crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size); + let mut children = Vec::new(); + collect_todo_nodes(&tree.root, &root_rect, ppu, &mut children); + (!children.is_empty()).then(|| SeparationTree { + src_ui_design: tree.src_ui_design.clone(), + root: SeparationNode { + id: tree.root.id.clone(), + global_pos_x_px: 0, + global_pos_y_px: 0, + width_px: image.pixel_size.x.max(0.0).round() as u32, + height_px: image.pixel_size.y.max(0.0).round() as u32, + note: SeparationNote::default(), + children, + rework_count: 0, + }, + }) + }) + .collect(); + SeparationState { + schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(), + unprocessed_trees, + bound: Vec::new(), + problematic_nodes: Vec::new(), + } +} + +pub fn next_leaf_batch(tree: &SeparationTree) -> Vec<&SeparationNode> { + fn leaves<'a>(node: &'a SeparationNode, output: &mut Vec<&'a SeparationNode>) { + if node.children.is_empty() { + output.push(node); + } else { + for child in &node.children { + leaves(child, output); + } + } + } + let mut output = Vec::new(); + leaves(&tree.root, &mut output); + output +} + +pub fn validate_binding_response( + response: &BindingResp, + batch: &[&SeparationNode], +) -> Result<(), String> { + let expected = batch + .iter() + .map(|node| node.id.clone()) + .collect::>(); + let mut seen = std::collections::HashSet::new(); + for decision in &response.decisions { + let node_id = match decision { + BindingDecision::Ok { to_node, .. } | BindingDecision::NeedRework { to_node, .. } => { + to_node + } + }; + if !expected.contains(node_id) { + return Err(format!("视觉绑定返回了未知节点：{}", node_id.as_str())); + } + if !seen.insert(node_id.clone()) { + return Err(format!("视觉绑定重复返回节点：{}", node_id.as_str())); + } + if let BindingDecision::NeedRework { + problem_description, + .. + } = decision + { + if problem_description.trim().is_empty() { + return Err("NeedRework 必须包含问题描述".to_string()); + } + } + } + if seen.len() != expected.len() { + return Err("视觉绑定未覆盖当前 batch 的全部节点".to_string()); + } + Ok(()) +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs new file mode 100644 index 000000000..3ca8fc937 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs @@ -0,0 +1,420 @@ +use super::model::*; +use super::prompt::{gen_binding_prompt, gen_extract_prompt}; +use crate::config::{build_game_creator_llm_client_from_llm_config, load_game_creator_app_config}; +use crate::platform_session::current_platform_session; +use crate::ui_editor::commands::separation::*; +use crate::ui_editor::commands::utils::{ + parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm, + request_with_feedback, strict_json_schema, +}; +use crate::ui_editor::state::State; +use crate::ui_editor::utils::NodeId; +use base64::Engine as _; +use image::ImageFormat; +use platform_llm::{ + LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice, +}; +use serde::Deserialize; +use std::fs; +use std::path::Path; +pub fn apply_batch_patch( + state: &mut SeparationState, + tree_index: usize, + decisions: &[BindingDecision], + cut_paths: &std::collections::HashMap, +) -> Result<(), String> { + let tree = state + .unprocessed_trees + .get_mut(tree_index) + .ok_or_else(|| "separation tree 索引无效".to_string())?; + let batch = next_leaf_batch(tree); + validate_binding_response( + &BindingResp { + decisions: decisions.to_vec(), + }, + &batch, + )?; + let rework_counts = batch + .iter() + .map(|node| (node.id.clone(), node.rework_count)) + .collect::>(); + let mut ids = std::collections::HashSet::new(); + for decision in decisions { + match decision { + BindingDecision::Ok { to_node, .. } => { + let path = cut_paths + .get(to_node) + .ok_or_else(|| format!("缺少节点 {} 的 cut 图片", to_node.as_str()))?; + state.bound.push(BoundNode { + node_id: to_node.clone(), + cut_image_path: path.clone(), + }); + ids.insert(to_node.clone()); + } + BindingDecision::NeedRework { + to_node, + problem_description, + } => { + let count = rework_counts.get(to_node).copied().unwrap_or(0) + 1; + if count >= MAX_REWORK_COUNT { + state.problematic_nodes.push(ProblematicNode { + node_id: to_node.clone(), + problem_description: problem_description.clone(), + rework_count: count, + }); + ids.insert(to_node.clone()); + } else { + increment_rework_count(&mut tree.root, to_node, count); + } + } + } + } + remove_ids(&mut tree.root, &ids); + Ok(()) +} + +fn mark_batch_problematic( + state: &mut SeparationState, + tree_index: usize, + batch: &[&SeparationNode], + error: String, +) { + let ids = batch + .iter() + .map(|node| node.id.clone()) + .collect::>(); + for node in batch { + state.problematic_nodes.push(ProblematicNode { + node_id: node.id.clone(), + problem_description: error.clone(), + rework_count: MAX_REWORK_COUNT, + }); + } + if let Some(tree) = state.unprocessed_trees.get_mut(tree_index) { + remove_ids(&mut tree.root, &ids); + } +} + +fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) { + if node.id == *id { + node.rework_count = count; + return; + } + for child in &mut node.children { + increment_rework_count(child, id, count); + } +} + +#[derive(Deserialize)] +struct RawEditResponse { + data: Vec, +} +#[derive(Deserialize)] +struct RawEditItem { + b64_json: String, +} + +async fn raw_image_edit( + session: &crate::platform_session::PlatformSessionSnapshot, + image_data_url: &str, + prompt: &str, + width: u32, + height: u32, +) -> Result { + let (mime, data) = image_data_url + .split_once(",") + .ok_or_else(|| "界面图 data URL 无效".to_string())?; + let mime = mime + .strip_prefix("data:") + .and_then(|v| v.strip_suffix(";base64")) + .unwrap_or("image/png"); + let client = crate::http_client::agc_main_site_client_builder() + .build() + .map_err(|e| format!("创建图片编辑客户端失败：{e}"))?; + let url = format!( + "{}/api/raw/v1/images/edit", + session.api_base_url.trim_end_matches('/') + ); + let body = serde_json::json!({ + "image": {"data": data, "mimeType": mime}, + "prompt": prompt, + "width": width, + "height": height, + "output_format": "png", + "background": "transparent" + }); + let response = crate::http_client::with_agc_main_site_marker( + client + .post(url) + .bearer_auth(&session.access_token) + .json(&body), + ) + .send() + .await + .map_err(|e| format!("图片分离请求失败：{e}"))?; + if !response.status().is_success() { + return Err(format!("图片分离请求失败（HTTP {}）", response.status())); + } + let payload = response + .json::() + .await + .map_err(|e| format!("解析图片分离响应失败：{e}"))?; + payload + .data + .into_iter() + .next() + .map(|item| format!("data:image/png;base64,{}", item.b64_json)) + .ok_or_else(|| "图片分离响应没有图像".to_string()) +} + +fn build_marked_image( + source_url: &str, + nodes: &[&SeparationNode], + target: &Path, +) -> Result { + let encoded = source_url + .split_once(',') + .map(|(_, d)| d) + .ok_or_else(|| "源图 data URL 无效".to_string())?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|e| format!("解码源图失败：{e}"))?; + let mut image = image::load_from_memory(&bytes) + .map_err(|e| format!("读取源图失败：{e}"))? + .to_rgba8(); + let width = image.width(); + let height = image.height(); + for node in nodes { + let x0 = node.global_pos_x_px.min(width.saturating_sub(1)); + let y0 = node.global_pos_y_px.min(height.saturating_sub(1)); + let x1 = node + .global_pos_x_px + .saturating_add(node.width_px) + .min(width) + .saturating_sub(1); + let y1 = node + .global_pos_y_px + .saturating_add(node.height_px) + .min(height) + .saturating_sub(1); + if x0 >= x1 || y0 >= y1 { + continue; + } + for x in x0..=x1 { + image.put_pixel(x, y0, image::Rgba([0, 255, 0, 255])); + image.put_pixel(x, y1, image::Rgba([0, 255, 0, 255])); + } + for y in y0..=y1 { + image.put_pixel(x0, y, image::Rgba([0, 255, 0, 255])); + image.put_pixel(x1, y, image::Rgba([0, 255, 0, 255])); + } + for y in y0..=y1 { + for x in x0..=x1 { + if x > x0 && x < x1 && y > y0 && y < y1 { + image.put_pixel(x, y, image::Rgba([180, 0, 180, 120])); + } + } + } + } + image::DynamicImage::ImageRgba8(image.clone()) + .save_with_format(target, image::ImageFormat::Png) + .map_err(|e| format!("写入标记图失败：{e}"))?; + let mut png = Vec::new(); + image::DynamicImage::ImageRgba8(image) + .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) + .map_err(|e| format!("编码标记图失败：{e}"))?; + Ok(format!( + "data:image/png;base64,{}", + base64::engine::general_purpose::STANDARD.encode(png) + )) +} + +async fn visual_binding( + source_url: String, + processed_url: String, + nodes: &[&SeparationNode], +) -> Result { + let llm_config = load_game_creator_app_config() + .map_err(|e| e.to_string())? + .llm; + let client = build_game_creator_llm_client_from_llm_config(&llm_config, "llm") + .map_err(|e| e.to_string())?; + let schema = strict_json_schema::()?; + let tool = LlmFunctionTool::new( + "bind_ui_elements", + "确认处理图中的区域对应哪些 UI 节点", + schema, + ) + .with_strict(true); + let base_prompt = gen_binding_prompt(nodes.to_vec()); + request_with_feedback( + 2, + |feedback| { + let prompt = feedback.map_or_else( + || base_prompt.clone(), + |error| format!("{base_prompt}\n上一次输出错误：{error}\n请修正并完整返回。"), + ); + let source_url = source_url.clone(); + let processed_url = processed_url.clone(); + let tool = tool.clone(); + let client = client.clone(); + let llm_config = llm_config.clone(); + async move { + let request = LlmRunRequest::new(vec![ + LlmMessage::system("你是 UI 图片视觉绑定器。只根据图像判断区域，不做 OCR。"), + LlmMessage::user_multimodal(vec![ + LlmMessageContentPart::InputText { text: prompt }, + LlmMessageContentPart::InputImage { + image_url: source_url, + }, + LlmMessageContentPart::InputImage { + image_url: processed_url, + }, + ]), + ]) + .with_function_tools(vec![tool.clone()]) + .with_tool_choice(LlmToolChoice::Required); + request_ui_editor_llm(&client, &llm_config, request) + .await + .map_err(|e| e.to_string()) + .and_then(|response| { + response + .tool_calls + .into_iter() + .find(|call| call.name == "bind_ui_elements") + .map(|call| call.arguments) + .ok_or_else(|| "视觉绑定模型未返回工具调用".to_string()) + }) + .and_then(|arguments| parse_limited_llm_tool_arguments(&arguments)) + .and_then(|args| { + serde_json::from_value::(args) + .map_err(|e| format!("视觉绑定结果无效：{e}")) + }) + .and_then(|parsed| Ok(parsed)) + } + }, + |value: &BindingResp| validate_binding_response(value, nodes), + ) + .await +} + +pub(crate) async fn separate_ui_impl( + project_path: String, + asset_id: String, + state: State, +) -> Result { + let session = current_platform_session().ok_or_else(|| "请先登录平台账号".to_string())?; + let root = Path::new(project_path.trim()); + let sidecar = separation_sidecar_dir(root, &asset_id)?; + fs::create_dir_all(&sidecar).map_err(|e| format!("创建 separation sidecar 失败：{e}"))?; + let state_path = sidecar.join("state.json"); + let mut separation = if state_path.exists() { + read_separation_state(&state_path)? + } else { + construct_separation_state(&state) + }; + for (tree_index, tree) in separation.unprocessed_trees.clone().iter().enumerate() { + let image = state + .ui_design_images + .get(&tree.src_ui_design) + .ok_or_else(|| "缺少源界面图".to_string())?; + let source_path = crate::project::resolve_local_project_path(root, &image.path)?; + let source_url = read_ui_reference_image_data_url(source_path).await?; + write_separation_state(&state_path, &separation)?; + loop { + let Some(current_tree) = separation.unprocessed_trees.get(tree_index) else { + break; + }; + let batch_nodes = next_leaf_batch(current_tree) + .into_iter() + .cloned() + .collect::>(); + if batch_nodes.is_empty() { + break; + } + let batch = batch_nodes.iter().collect::>(); + let prompt = + gen_extract_prompt(batch.iter().map(|n| n.note.clone()).collect::>()); + let marker_path = sidecar.join(format!("marked-{}.png", separation.bound.len())); + let marked_url = match build_marked_image(&source_url, &batch, &marker_path) { + Ok(value) => value, + Err(error) => { + mark_batch_problematic(&mut separation, tree_index, &batch, error); + write_separation_state(&state_path, &separation)?; + continue; + } + }; + let processed_url = match raw_image_edit( + &session, + &marked_url, + &prompt, + image.pixel_size.x as u32, + image.pixel_size.y as u32, + ) + .await + { + Ok(value) => value, + Err(error) => { + mark_batch_problematic(&mut separation, tree_index, &batch, error); + write_separation_state(&state_path, &separation)?; + continue; + } + }; + let processed_path = sidecar.join(format!("processed-{}.png", separation.bound.len())); + let processed_bytes = base64::engine::general_purpose::STANDARD + .decode( + processed_url + .split_once(',') + .map(|(_, d)| d) + .unwrap_or_default(), + ) + .map_err(|e| e.to_string())?; + fs::write(&processed_path, processed_bytes).map_err(|e| e.to_string())?; + let binding = match visual_binding(source_url.clone(), processed_url, &batch).await { + Ok(value) => value, + Err(error) => { + mark_batch_problematic(&mut separation, tree_index, &batch, error); + write_separation_state(&state_path, &separation)?; + continue; + } + }; + let mut cut_paths = std::collections::HashMap::new(); + for decision in &binding.decisions { + if let BindingDecision::Ok { + to_node, + separated_image_area, + } = decision + { + let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str())); + cut_processed_image(&processed_path, separated_image_area, &cut_path)?; + cut_paths.insert(to_node.clone(), cut_path.to_string_lossy().to_string()); + } + } + apply_batch_patch(&mut separation, tree_index, &binding.decisions, &cut_paths)?; + write_separation_state(&state_path, &separation)?; + } + } + let _ = fs::remove_file(state_path); + Ok(separation_dto(&separation)) +} + +fn cut_processed_image(source: &Path, area: &BindingArea, target: &Path) -> Result<(), String> { + let image = image::open(source).map_err(|e| format!("读取处理图失败：{e}"))?; + area.validate_in(image.width(), image.height())?; + let cropped = image.crop_imm( + area.global_pos_x_px, + area.global_pos_y_px, + area.width_px, + area.height_px, + ); + cropped + .save_with_format(target, ImageFormat::Png) + .map_err(|e| format!("写入 cut 图片失败：{e}")) +} + +fn remove_ids(node: &mut SeparationNode, ids: &std::collections::HashSet) { + node.children.retain(|child| !ids.contains(&child.id)); + for child in &mut node.children { + remove_ids(child, ids); + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs index 4faa40579..8aec25302 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs @@ -4,6 +4,7 @@ use base64::Engine as _; use platform_llm::{LlmClient, LlmError, LlmRunRequest, LlmRunResponse}; use schemars::JsonSchema; use std::fs::File; +use std::future::Future; use std::io::Read; use std::path::{Path, PathBuf}; @@ -25,6 +26,33 @@ pub(crate) async fn request_ui_editor_llm( request_game_creator_llm_text(client, llm, request).await } +/// 结构化 LLM 请求的小型 repair harness：第一次请求或校验失败后， +/// 将错误反馈给模型并只额外重试一次。网络/模型调用本身的错误也会 +/// 进入第二次请求的反馈文本；调用方负责在第二次失败后决定业务状态。 +pub(crate) async fn request_with_feedback( + more_turn: usize, + request: Request, + validate: Validate, +) -> Result +where + Request: Fn(Option) -> Fut, + Fut: Future>, + Validate: Fn(&T) -> Result<(), String>, +{ + let mut feedback = None; + for attempt in 0..=more_turn { + let result = request(feedback.clone()) + .await + .and_then(|value| validate(&value).map(|_| value)); + match result { + Ok(value) => return Ok(value), + Err(error) if attempt < more_turn => feedback = Some(error), + Err(error) => return Err(error), + } + } + unreachable!("repair harness always returns within requested turns") +} + pub(crate) fn parse_limited_llm_tool_arguments( arguments: &str, ) -> Result { @@ -144,6 +172,27 @@ mod tests { ); } + #[tokio::test] + async fn feedback_harness_zero_more_turn_calls_once_without_feedback() { + let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let seen = calls.clone(); + let result = request_with_feedback( + 0, + move |feedback| { + let seen = seen.clone(); + async move { + seen.lock().unwrap().push(feedback); + Ok::<_, String>(serde_json::json!({"ok": true})) + } + }, + |_| Ok(()), + ) + .await + .expect("single turn should succeed"); + assert_eq!(result, serde_json::json!({"ok": true})); + assert_eq!(calls.lock().unwrap().as_slice(), &[None]); + } + #[test] fn reference_image_rejects_file_over_five_mib_before_reading() { let directory = tempfile::tempdir().expect("reference image fixture");