From 0dddec7b502e224aef338da4a243cf491e64ac8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Fri, 14 Aug 2026 20:25:34 +0800 Subject: [PATCH] =?UTF-8?q?=E6=89=A9=E5=B1=95=E5=92=8C=E4=BC=98=E5=8C=96?= =?UTF-8?q?=20UI=20=E8=AF=86=E5=88=AB=E9=80=BB=E8=BE=91=EF=BC=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 实现多图片输入的 UI 树分离与独立验证。 - 引入 `RecognitionTree` 数据结构，支持每张输入图片一棵独立树。 - 重构节点转换逻辑，统一 pixels_per_unit 坐标系统。 - 添加逻辑以验证树与上下文图片的一一对应关系。 - 优化节点转换函数和 UI 树生成流程，提高结构一致性。 - 补充测试用例，验证树结构生成和转换正确性。 --- .../src-tauri/src/main.rs | 6 + .../src-tauri/src/ui_editor/commands/merge.rs | 384 ++++++++++++++++++ .../src-tauri/src/ui_editor/commands/mod.rs | 4 + .../src/ui_editor/commands/recognition.rs | 264 +++++++----- .../src/features/ui-editor/merge.ts | 10 + .../src/features/ui-editor/types/MergeDTO.ts | 4 + 6 files changed, 570 insertions(+), 102 deletions(-) create mode 100644 apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/merge.rs create mode 100644 apps/ai-game-creator-shell/src/features/ui-editor/merge.ts create mode 100644 apps/ai-game-creator-shell/src/features/ui-editor/types/MergeDTO.ts diff --git a/apps/ai-game-creator-shell/src-tauri/src/main.rs b/apps/ai-game-creator-shell/src-tauri/src/main.rs index e952ab35e..ab551e5fa 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/main.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/main.rs @@ -126,6 +126,11 @@ async fn recognize_ui( ui_editor::commands::recognize_ui_impl(project_path, state).await } +#[tauri::command] +async fn merge_ui(state: ui_editor::state::State) -> Result { + ui_editor::commands::merge_ui_impl(state).await +} + #[derive(Debug, Eq, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] struct InitLocalProjectResult { @@ -2261,6 +2266,7 @@ fn main() { import_ui_editor_remote_assets, suggest_ui_design_semantic, recognize_ui, + merge_ui, generate_platform_art_asset, open_canvas_project, get_game_creation_agent_capabilities, diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/merge.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/merge.rs new file mode 100644 index 000000000..94dbd6742 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/merge.rs @@ -0,0 +1,384 @@ +use crate::config::build_game_creator_llm_client_from_config; +use crate::ui_editor::commands::utils::strict_json_schema; +use crate::ui_editor::state::{State, UITree}; +use platform_llm::{LlmFunctionTool, LlmMessage, LlmRunRequest, LlmToolChoice}; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +const MERGE_TOOL_NAME: &str = "merge_ui_trees"; + +const SYSTEM_PROMPT: &str = r#" +角色： +你是游戏 UI 多树结构合并器。 + +任务： +用户会提供同一个 UI 系统中多张参考图各自的 UI 树。请返回一棵新的合并计划树。 + +规则： +* Simple 引用一个原始节点 id，并用返回的 children 定义它在新树中的子节点。 +* Merged 表示多个节点描述同一个共同组件(它们可能是同一组件的不同状态)，merged_from 中按语义排列需要放入容器的节点。 +* 可以在任意层级使用 Merged，不限于各输入树的根节点。 +* 一些共用的框架/层次/...在树中只保留一个, 优先保留优先级高的树中的节点 +* 面向用户的节点名称和结构判断使用中文语义理解。 +"#; + +mod llm_contract { + use super::strict_json_schema; + use crate::ui_editor::layout::node::{Node as LayoutNode, NodeMetadata}; + use crate::ui_editor::state::State; + use crate::ui_editor::utils::NodeId; + use schemars::JsonSchema; + use serde::{Deserialize, Serialize}; + + #[derive(Clone, Debug, Serialize)] + pub(super) struct OriginalNode { + id: NodeId, + metadata: NodeMetadata, + children: Vec, + } + + #[derive(Clone, Debug, Serialize)] + pub(super) struct OriginalTree { + priority: u32, + root: OriginalNode, + } + + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] + #[serde(deny_unknown_fields)] + #[schemars(deny_unknown_fields)] + pub(super) struct MergedNode { + pub(super) name: String, + pub(super) description: String, + pub(super) merged_from: Vec, + } + + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] + #[serde(deny_unknown_fields)] + #[schemars(deny_unknown_fields)] + pub(super) struct SimpleNode { + pub(super) id: NodeId, + pub(super) children: Vec, + } + + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] + pub(super) enum Node { + Simple(SimpleNode), + Merged(MergedNode), + } + + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] + #[serde(deny_unknown_fields)] + #[schemars(deny_unknown_fields)] + pub(super) struct MergeResponse { + pub(super) root: Node, + } + + pub(super) fn schema() -> Result { + strict_json_schema::() + } + + fn project_node(node: &LayoutNode) -> OriginalNode { + OriginalNode { + id: node.id.clone(), + metadata: node.metadata.clone(), + children: node.children.iter().map(project_node).collect(), + } + } + + pub(super) fn input_trees( + state: &State, + priorities: &[u32], + ) -> Result, String> { + if state.ui_trees.len() != priorities.len() { + return Err("UI 树优先级数量不匹配".to_string()); + } + Ok(state + .ui_trees + .iter() + .zip(priorities) + .map(|(tree, priority)| OriginalTree { + priority: *priority, + root: project_node(&tree.root), + }) + .collect()) + } +} + +mod priority { + use crate::ui_editor::resource::ui_design_image::UIDesignImage; + use crate::ui_editor::state::State; + use crate::ui_editor::utils::UIDesignImageId; + use std::collections::{HashMap, HashSet}; + + fn ancestor_chain( + image_id: &UIDesignImageId, + images: &HashMap, + ) -> Result, String> { + let mut chain = Vec::new(); + let mut seen = HashSet::new(); + let mut current = image_id.clone(); + loop { + if !seen.insert(current.clone()) { + return Err(format!( + "界面图 slave_to 关系存在循环：{}", + current.as_str() + )); + } + chain.push(current.clone()); + let image = images + .get(¤t) + .ok_or_else(|| format!("界面图 slave_to 引用了缺失界面图：{}", current.as_str()))?; + match &image.metadata.slave_to { + Some(parent) => current = parent.clone(), + None => return Ok(chain), + } + } + } + + pub(super) fn for_state(state: &State) -> Result, String> { + let source_ids = state + .ui_trees + .iter() + .map(|tree| tree.src_ui_design.clone()) + .collect::>(); + let chains = source_ids + .iter() + .map(|source_id| ancestor_chain(source_id, &state.ui_design_images)) + .collect::, _>>()?; + Ok(source_ids + .iter() + .map(|candidate| { + chains + .iter() + .filter(|chain| chain.contains(candidate)) + .count() as u32 + }) + .collect()) + } +} + +mod materialize { + use super::llm_contract::Node; + use crate::ui_editor::layout::node::{ + Node as LayoutNode, NodeMetadata, NodeSource, NodeStatus, + }; + use crate::ui_editor::layout::transform::Transform; + use crate::ui_editor::state::{State, UITree}; + use crate::ui_editor::utils::{NodeId, UIDesignImageId}; + use std::collections::{HashMap, HashSet}; + + #[derive(Clone)] + struct OriginalNodeRecord { + node: LayoutNode, + priority: u32, + src_ui_design: UIDesignImageId, + } + + struct BuiltNode { + node: LayoutNode, + priority: u32, + src_ui_design: UIDesignImageId, + } + + fn collect_original_nodes( + node: &LayoutNode, + priority: u32, + src_ui_design: &UIDesignImageId, + records: &mut HashMap, + ) -> Result<(), String> { + if records + .insert( + node.id.clone(), + OriginalNodeRecord { + node: node.clone(), + priority, + src_ui_design: src_ui_design.clone(), + }, + ) + .is_some() + { + return Err(format!("输入 UI 树包含重复节点 ID：{}", node.id.as_str())); + } + for child in &node.children { + collect_original_nodes(child, priority, src_ui_design, records)?; + } + Ok(()) + } + + fn oldest_member_index(members: &[BuiltNode]) -> Result { + if members.is_empty() { + return Err("MergedNode.merged_from 不能为空".to_string()); + } + let mut best_index = 0; + let mut best_priority = 0; + for (candidate_index, candidate) in members.iter().enumerate() { + if candidate.priority > best_priority { + best_index = candidate_index; + best_priority = candidate.priority; + } + } + Ok(best_index) + } + + fn random_node_id(occupied: &mut HashSet) -> Result { + loop { + let id = NodeId::new(uuid::Uuid::new_v4().simple().to_string()) + .map_err(|error| format!("生成合并容器节点 ID 失败：{error}"))?; + if occupied.insert(id.clone()) { + return Ok(id); + } + } + } + + fn build_node( + plan: Node, + records: &HashMap, + used_original_ids: &mut HashSet, + occupied_ids: &mut HashSet, + ) -> Result { + match plan { + Node::Simple(simple) => { + if !used_original_ids.insert(simple.id.clone()) { + return Err(format!("合并计划重复引用节点 ID：{}", simple.id.as_str())); + } + let original = records + .get(&simple.id) + .ok_or_else(|| format!("合并计划引用了未知节点 ID：{}", simple.id.as_str()))?; + let mut node = original.node.clone(); + node.children = simple + .children + .into_iter() + .map(|child| { + build_node(child, records, used_original_ids, occupied_ids) + .map(|built| built.node) + }) + .collect::, _>>()?; + Ok(BuiltNode { + node, + priority: original.priority, + src_ui_design: original.src_ui_design.clone(), + }) + } + Node::Merged(merged) => { + let container_name = merged.name; + let container_description = merged.description; + let mut members = merged + .merged_from + .into_iter() + .map(|member| build_node(member, records, used_original_ids, occupied_ids)) + .collect::, _>>()?; + let oldest_index = oldest_member_index(&members)?; + let original_transform = members[oldest_index].node.transform; + let priority = members[oldest_index].priority; + let src_ui_design = members[oldest_index].src_ui_design.clone(); + for member in &mut members { + member.node.transform = Transform::stretch(); + } + Ok(BuiltNode { + node: LayoutNode { + id: random_node_id(occupied_ids)?, + transform: original_transform, + metadata: NodeMetadata { + name: container_name, + description: container_description, + status: NodeStatus::Passed, + source: NodeSource::Llm, + }, + components: Vec::new(), + children: members.into_iter().map(|member| member.node).collect(), + }, + priority, + src_ui_design, + }) + } + } + } + + pub(super) fn plan(plan: Node, state: &State, priorities: &[u32]) -> Result { + if state.ui_trees.len() != priorities.len() { + return Err("UI 树优先级数量不匹配".to_string()); + } + let mut records = HashMap::new(); + for (tree, priority) in state.ui_trees.iter().zip(priorities) { + collect_original_nodes(&tree.root, *priority, &tree.src_ui_design, &mut records)?; + } + let mut occupied_ids = records.keys().cloned().collect::>(); + let mut used_original_ids = HashSet::new(); + let built = build_node(plan, &records, &mut used_original_ids, &mut occupied_ids)?; + Ok(UITree { + src_ui_design: built.src_ui_design, + root: built.node, + }) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] +#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] +pub struct MergeDTO { + pub ui_tree: UITree, +} + +pub(crate) async fn merge_ui_impl(state: State) -> Result { + if state.ui_trees.is_empty() { + eprintln!("ui_merge.error stage=validate reason=no_trees"); + return Err("请先完成 UI 结构识别".to_string()); + } + let priorities = priority::for_state(&state).map_err(|error| { + eprintln!("ui_merge.error stage=build_priority error={error}"); + error + })?; + let trees = llm_contract::input_trees(&state, &priorities).map_err(|error| { + eprintln!("ui_merge.error stage=build_input error={error}"); + error + })?; + let records_json = serde_json::to_string(&trees).map_err(|error| { + eprintln!("ui_merge.error stage=serialize_input error={error}"); + format!("序列化 UI 合并输入失败：{error}") + })?; + let client = build_game_creator_llm_client_from_config().map_err(|error| { + eprintln!("ui_merge.error stage=build_client error={error}"); + error + })?; + let schema = llm_contract::schema().map_err(|error| { + eprintln!("ui_merge.error stage=build_schema error={error}"); + error + })?; + let tool = LlmFunctionTool::new( + MERGE_TOOL_NAME, + "把多张参考图各自的 UI 树合并为一棵新的 UI 计划树", + schema, + ) + .with_strict(true); + let response = client + .run( + LlmRunRequest::new(vec![ + LlmMessage::system(SYSTEM_PROMPT), + LlmMessage::user(format!("待合并 UI 树：\n{records_json}")), + ]) + .with_function_tools(vec![tool]) + .with_tool_choice(LlmToolChoice::Required), + ) + .await + .map_err(|error| { + eprintln!("ui_merge.error stage=llm_request error={error}"); + format!("UI 树合并失败：{error}") + })?; + let call = response + .tool_calls + .iter() + .find(|call| call.name == MERGE_TOOL_NAME) + .ok_or_else(|| { + eprintln!("ui_merge.error stage=parse_tool_call reason=missing_tool_call"); + format!("LLM 未返回 {MERGE_TOOL_NAME} 工具调用") + })?; + let parsed = + serde_json::from_str::(&call.arguments).map_err(|error| { + eprintln!("ui_merge.error stage=parse_arguments error={error}"); + format!("UI 合并工具参数无效：{error}") + })?; + let ui_tree = materialize::plan(parsed.root, &state, &priorities).map_err(|error| { + eprintln!("ui_merge.error stage=materialize error={error}"); + error + })?; + Ok(MergeDTO { ui_tree }) +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs index d9132d2f8..0b67edb53 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs @@ -1,7 +1,11 @@ +pub mod merge; pub mod recognition; pub mod ui_design_suggestion; pub mod utils; +pub mod binding; +pub(crate) use merge::merge_ui_impl; +pub use merge::MergeDTO; pub(crate) use recognition::recognize_ui_impl; pub use recognition::RecognitionDTO; pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl; diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs index 98bcba54f..ac433b09e 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs @@ -13,7 +13,7 @@ use platform_llm::{ }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::Path; use ts_rs::TS; @@ -30,17 +30,17 @@ const SYSTEM_PROMPT: &str = r#" 同时分析同一 UI 系统的全部参考图，建立UI树 用户会给你一些UI截图(它们从属于同一个UI系统)和对应的元数据, 请用给定的工具描述UI结构 -你是游戏 UI 结构识别器。 识别规则： * 只识别 UI，不识别场景人物、地形、建筑、光影和背景装饰。 * 不创建没有独立功能的微小装饰Node。 * 无法确定类型、层级、关系时，在 UnSure 中写明原因。 -* source=user、reviewStatus=confirmed 或 semanticSource=user 的组件不得被 AI 覆盖。 -* 每个Node必须使用 src_ui_design_image_id 指定坐标来源(所属UI design图)，并在该图片的原始像素坐标系中输出 -global_pos_x_px、global_pos_y_px、width_px、height_px；坐标相对于来源图片左上角 -* 多张图都是在描述同一个UI系统, 必须合并共用公共的框架/层次, 禁止简单每个图一个树, 坐标等有矛盾的地方父级优先 +* 返回的 trees 必须与输入图片一一对应，每张输入图片只能有一棵树，不能合并多张图片的树。 每棵树的 src_ui_design_image_id 必须等于对应输入图片标注的 id。 +* 每棵树必须使用自己的输入图片原始像素坐标系(0,0 is left top)输出 + global_pos_x_px、global_pos_y_px、width_px、height_px； * 面向用户的字段如名称描述等请用中文 +* 由于每个截图未必是完整的, 可能是局部的, 每棵树描述清楚每个截图上UI的层次结构即可 +* 不同树的共用框架/层次/...请使用使用相同的名称描述. 不同状态/变体名称使用相同的前缀, 用后缀区别 "#; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] @@ -98,7 +98,6 @@ struct RecognitionNode { global_pos_y_px: u32, width_px: u32, height_px: u32, - src_ui_design_image_id: UIDesignImageId, local_anchor: Anchor, name: String, description: String, @@ -108,10 +107,17 @@ struct RecognitionNode { #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] #[schemars(deny_unknown_fields)] -struct RecognitionResponse { +struct RecognitionTree { + src_ui_design_image_id: UIDesignImageId, children: Vec, } +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[schemars(deny_unknown_fields)] +struct RecognitionResponse { + trees: Vec, +} + #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] #[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] pub struct RecognitionDTO { @@ -195,23 +201,17 @@ fn image_layout_size(image: &UIDesignImage) -> Result, String> { fn convert_node( source: &RecognitionNode, - page_image: &UIDesignImage, - source_image: &UIDesignImage, - images: &HashMap, + image_id: &UIDesignImageId, + image: &UIDesignImage, parent_rect: UIRect, ) -> Result { - // LLM 输出的是 source_image 原始像素。Rust 先按来源图尺寸归一化， - // 再缩放到 Page 图像素尺寸，最后使用 Page 的 pixels_per_unit 转成布局单位。 - let source_size_px = source_image.pixel_size; - let page_size_px = page_image.pixel_size; - let source_size_valid = source_size_px + // LLM 输出的是当前树所属图片的原始像素。Rust 只按该图片自己的 + // pixels_per_unit 转成布局单位，绝不把坐标归一化或映射到 Page 图片。 + let image_size_px = image.pixel_size; + let image_size_valid = image_size_px .iter() .all(|value| value.is_finite() && *value > 0.0); - if !source_size_valid - || !page_size_px - .iter() - .all(|value| value.is_finite() && *value > 0.0) - { + if !image_size_valid { return Err("界面图尺寸无效".to_string()); } let source_min_px = Vector2::new(source.global_pos_x_px as f32, source.global_pos_y_px as f32); @@ -220,19 +220,19 @@ fn convert_node( let mut blocked = source.width_px == 0 || source.height_px == 0; if source_min_px.x < 0.0 || source_min_px.y < 0.0 - || source_max_px.x > source_size_px.x - || source_max_px.y > source_size_px.y + || source_max_px.x > image_size_px.x + || source_max_px.y > image_size_px.y { blocked = true; } - let scale = page_size_px.component_div(&source_size_px); - let page_min_px = source_min_px.component_mul(&scale); - let page_size_node_px = source_size_node_px.component_mul(&scale); - let ppu = page_image.pixels_per_unit.get(); - let target_min = Point2::new(page_min_px.x / ppu, page_min_px.y / ppu); + let ppu = image.pixels_per_unit.get(); + if !ppu.is_finite() || ppu <= 0.0 { + return Err("界面图 pixels_per_unit 无效".to_string()); + } + let target_min = Point2::new(source_min_px.x / ppu, source_min_px.y / ppu); let target_rect = UIRect::new( target_min, - Vector2::new(page_size_node_px.x / ppu, page_size_node_px.y / ppu), + Vector2::new(source_size_node_px.x / ppu, source_size_node_px.y / ppu), ); let (anchor_min, anchor_max) = anchor_ranges(&source.local_anchor) .unwrap_or((Vector2::new(0.5, 0.5), Vector2::new(0.5, 0.5))); @@ -262,16 +262,7 @@ fn convert_node( let children = source .children .iter() - .map(|child| { - let child_source_image = - images.get(&child.src_ui_design_image_id).ok_or_else(|| { - format!( - "节点引用了缺失界面图 {}", - child.src_ui_design_image_id.as_str() - ) - })?; - convert_node(child, page_image, child_source_image, images, target_rect) - }) + .map(|child| convert_node(child, image_id, image, target_rect)) .collect::, _>>()?; Ok(LayoutNode { id: random_node_id()?, @@ -300,18 +291,23 @@ fn validate_confidence(nodes: &[RecognitionNode]) -> Result<(), String> { Ok(()) } -fn validate_source_image_ids( - nodes: &[RecognitionNode], +fn validate_tree_image_ids( + trees: &[RecognitionTree], allowed_ids: &[UIDesignImageId], ) -> Result<(), String> { - for node in nodes { - if !allowed_ids - .iter() - .any(|allowed| allowed == &node.src_ui_design_image_id) - { - return Err("LLM 返回了不属于当前 Page 上下文的节点来源界面图 ID".to_string()); + let allowed = allowed_ids.iter().cloned().collect::>(); + let mut seen = HashSet::new(); + for tree in trees { + if !allowed.contains(&tree.src_ui_design_image_id) { + return Err("LLM 返回了不属于当前 Page 上下文的界面图 ID".to_string()); } - validate_source_image_ids(&node.children, allowed_ids)?; + if !seen.insert(tree.src_ui_design_image_id.clone()) { + return Err("LLM 为同一界面图返回了重复 UI 树".to_string()); + } + validate_confidence(&tree.children)?; + } + if seen.len() != allowed.len() { + return Err("LLM 未为当前 Page 上下文的每张界面图返回 UI 树".to_string()); } Ok(()) } @@ -320,6 +316,43 @@ fn validate_source_image_ids( mod tests { use super::*; + fn test_image() -> UIDesignImage { + UIDesignImage { + metadata: crate::ui_editor::resource::ui_design_image::UIDesignImageMetadata { + name: "测试图".to_string(), + description: String::new(), + role: None, + slave_to: None, + }, + path: "test.png".to_string(), + pixel_size: Vector2::new(1000.0, 500.0), + pixels_per_unit: typed_floats::tf32::StrictlyPositiveFinite::new(2.0) + .expect("valid pixels per unit"), + } + } + + fn test_node() -> RecognitionNode { + RecognitionNode { + global_pos_x_px: 100, + global_pos_y_px: 50, + width_px: 200, + height_px: 100, + local_anchor: Anchor::Preset(PresetAnchor { + horizontal: HorizonalAnchor::Left, + vertical: VerticalAnchor::Top, + }), + name: "节点".to_string(), + description: String::new(), + children: Vec::new(), + confidence: Confidence::Confident, + } + } + + fn assert_rect_close(actual: UIRect, expected: UIRect) { + assert!((actual.min - expected.min).norm() <= 1.0e-4); + assert!((actual.size - expected.size).norm() <= 1.0e-4); + } + fn assert_no_one_of(value: &serde_json::Value) { match value { serde_json::Value::Object(object) => { @@ -372,7 +405,6 @@ mod tests { global_pos_y_px: 0, width_px: 1, height_px: 1, - src_ui_design_image_id: UIDesignImageId::new("page").expect("image id"), local_anchor: Anchor::Preset(PresetAnchor { horizontal: HorizonalAnchor::Left, vertical: VerticalAnchor::Top, @@ -384,6 +416,43 @@ mod tests { }; assert!(validate_confidence(&[node]).is_err()); } + + #[test] + fn conversion_stays_in_the_tree_image_coordinate_system() { + let image_id = UIDesignImageId::new("slave").expect("valid image id"); + let image = test_image(); + let root_rect = UIRect::new(Point2::origin(), image_layout_size(&image).unwrap()); + let converted = + convert_node(&test_node(), &image_id, &image, root_rect).expect("node conversion"); + + assert_rect_close( + converted.transform.resolve(&root_rect), + UIRect::new(Point2::new(50.0, 25.0), Vector2::new(100.0, 50.0)), + ); + } + + #[test] + fn tree_validation_requires_exactly_one_tree_per_context_image() { + let page = UIDesignImageId::new("page").expect("valid image id"); + let slave = UIDesignImageId::new("slave").expect("valid image id"); + let tree = |id: UIDesignImageId| RecognitionTree { + src_ui_design_image_id: id, + children: Vec::new(), + }; + + assert!(validate_tree_image_ids( + &[tree(page.clone()), tree(slave.clone())], + &[page.clone(), slave.clone()], + ) + .is_ok()); + assert!( + validate_tree_image_ids(&[tree(page.clone())], &[page.clone(), slave.clone()]).is_err() + ); + assert!( + validate_tree_image_ids(&[tree(page.clone()), tree(page.clone())], &[page, slave],) + .is_err() + ); + } } fn page_image_ids(state: &State) -> Vec { @@ -439,14 +508,10 @@ pub(crate) async fn recognize_ui_impl( let root = Path::new(project_path.trim()); let mut ui_trees = Vec::with_capacity(page_ids.len()); for page_id in page_ids { - let page = state - .ui_design_images - .get(&page_id) - .ok_or_else(|| "缺少 Page 界面图资源".to_string())?; let mut context_ids = vec![page_id.clone()]; context_ids.extend(slave_image_ids(&state, &page_id)); - // The model may cite only this Page or its direct slave images as evidence. - let allowed_source_ids = context_ids.clone(); + // 当前只支持 Page 的直接 slave；嵌套 slave 关系的识别批次待后续设计。 + // TODO: 支持 slave_to 链的递归分组和独立识别。 let mut parts = Vec::with_capacity(context_ids.len() * 2); for (index, context_id) in context_ids.iter().enumerate() { let image = state @@ -484,7 +549,7 @@ pub(crate) async fn recognize_ui_impl( } let tool = LlmFunctionTool::new( "recognize_ui_structure", - "识别当前 Page 及其 slave 参考图中的 UI 节点，并只返回这一棵树的节点列表", + "识别当前 Page 及其直接 slave 参考图，并为每张输入图片各返回一棵 UI 树", schema.clone(), ) .with_strict(true); @@ -533,57 +598,52 @@ pub(crate) async fn recognize_ui_impl( ); format!("Page {} 的识别工具参数无效：{error}", page_id.as_str()) })?; - validate_confidence(&parsed.children).map_err(|error| { + validate_tree_image_ids(&parsed.trees, &context_ids).map_err(|error| { eprintln!( - "ui_recognition.error stage=validate_confidence page={} error={error}", + "ui_recognition.error stage=validate_trees page={} error={error}", page_id.as_str() ); error })?; - validate_source_image_ids(&parsed.children, &allowed_source_ids).map_err(|error| { - eprintln!( - "ui_recognition.error stage=validate_source_ids page={} error={error}", - page_id.as_str() - ); - error - })?; - let size = image_layout_size(page)?; - let root_rect = UIRect::new(Point2::origin(), size); - let children = parsed - .children - .iter() - .map(|node| { - let source_image = state - .ui_design_images - .get(&node.src_ui_design_image_id) - .ok_or_else(|| { - format!( - "Page {} 的节点引用了缺失界面图 {}", - page_id.as_str(), - node.src_ui_design_image_id.as_str() - ) - })?; - convert_node(node, page, source_image, &state.ui_design_images, root_rect) - }) - .collect::, _>>()?; - let root = LayoutNode { - id: random_node_id()?, - transform: Transform::stretch(), - metadata: NodeMetadata { - name: "页面根节点".to_string(), - description: String::new(), - status: NodeStatus::Passed, - source: NodeSource::System, - }, - // TODO: root components can host page-level background content once component preview is supported. - components: Vec::new(), - children, - }; - // Tree identity and root identity are assigned by Rust, never chosen by the model. - ui_trees.push(UITree { - src_ui_design: page_id, - root, - }); + let mut parsed_trees = parsed + .trees + .into_iter() + .map(|tree| (tree.src_ui_design_image_id.clone(), tree)) + .collect::>(); + for image_id in context_ids { + let image = state + .ui_design_images + .get(&image_id) + .ok_or_else(|| "缺少识别上下文界面图资源".to_string())?; + let tree = parsed_trees + .remove(&image_id) + .ok_or_else(|| format!("缺少界面图 {} 的识别树", image_id.as_str()))?; + let size = image_layout_size(image)?; + let root_rect = UIRect::new(Point2::origin(), size); + let children = tree + .children + .iter() + .map(|node| convert_node(node, &image_id, image, root_rect)) + .collect::, _>>()?; + let root = LayoutNode { + id: random_node_id()?, + transform: Transform::stretch(), + metadata: NodeMetadata { + name: "页面根节点".to_string(), + description: String::new(), + status: NodeStatus::Passed, + source: NodeSource::System, + }, + // TODO: root components can host page-level background content once component preview is supported. + components: Vec::new(), + children, + }; + // Tree identity and root identity are assigned by Rust, never chosen by the model. + ui_trees.push(UITree { + src_ui_design: image_id, + root, + }); + } } Ok(RecognitionDTO { ui_trees }) } diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/merge.ts b/apps/ai-game-creator-shell/src/features/ui-editor/merge.ts new file mode 100644 index 000000000..b90781bb4 --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/merge.ts @@ -0,0 +1,10 @@ +import type { MergeDTO } from './types/MergeDTO'; +import type { State } from './types/State'; + +/** 合并阶段返回一棵新树，成功后替换当前全部逐图识别树。 */ +export function applyMergeResult(state: State, result: MergeDTO): State { + return { + ...structuredClone(state), + ui_trees: [structuredClone(result.ui_tree)], + }; +} diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/MergeDTO.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/MergeDTO.ts new file mode 100644 index 000000000..366a191e3 --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/MergeDTO.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { UITree } from "./UITree"; + +export type MergeDTO = { ui_tree: UITree, };