扩展和优化 UI 识别逻辑:

- 实现多图片输入的 UI 树分离与独立验证。
- 引入 `RecognitionTree` 数据结构,支持每张输入图片一棵独立树。
- 重构节点转换逻辑,统一 pixels_per_unit 坐标系统。
- 添加逻辑以验证树与上下文图片的一一对应关系。
- 优化节点转换函数和 UI 树生成流程,提高结构一致性。
- 补充测试用例,验证树结构生成和转换正确性。
This commit is contained in:
2026-08-14 20:25:34 +08:00
parent 9610eb6dbc
commit 0dddec7b50
6 changed files with 570 additions and 102 deletions
@@ -126,6 +126,11 @@ async fn recognize_ui(
ui_editor::commands::recognize_ui_impl(project_path, state).await
}
#[tauri::command]
async fn merge_ui(state: ui_editor::state::State) -> Result<ui_editor::commands::MergeDTO, String> {
ui_editor::commands::merge_ui_impl(state).await
}
#[derive(Debug, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
struct InitLocalProjectResult {
@@ -2261,6 +2266,7 @@ fn main() {
import_ui_editor_remote_assets,
suggest_ui_design_semantic,
recognize_ui,
merge_ui,
generate_platform_art_asset,
open_canvas_project,
get_game_creation_agent_capabilities,
@@ -0,0 +1,384 @@
use crate::config::build_game_creator_llm_client_from_config;
use crate::ui_editor::commands::utils::strict_json_schema;
use crate::ui_editor::state::{State, UITree};
use platform_llm::{LlmFunctionTool, LlmMessage, LlmRunRequest, LlmToolChoice};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
const MERGE_TOOL_NAME: &str = "merge_ui_trees";
const SYSTEM_PROMPT: &str = r#"
角色:
你是游戏 UI 多树结构合并器。
任务:
用户会提供同一个 UI 系统中多张参考图各自的 UI 树。请返回一棵新的合并计划树。
规则:
* Simple 引用一个原始节点 id,并用返回的 children 定义它在新树中的子节点。
* Merged 表示多个节点描述同一个共同组件(它们可能是同一组件的不同状态),merged_from 中按语义排列需要放入容器的节点。
* 可以在任意层级使用 Merged,不限于各输入树的根节点。
* 一些共用的框架/层次/...在树中只保留一个, 优先保留优先级高的树中的节点
* 面向用户的节点名称和结构判断使用中文语义理解。
"#;
mod llm_contract {
use super::strict_json_schema;
use crate::ui_editor::layout::node::{Node as LayoutNode, NodeMetadata};
use crate::ui_editor::state::State;
use crate::ui_editor::utils::NodeId;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize)]
pub(super) struct OriginalNode {
id: NodeId,
metadata: NodeMetadata,
children: Vec<OriginalNode>,
}
#[derive(Clone, Debug, Serialize)]
pub(super) struct OriginalTree {
priority: u32,
root: OriginalNode,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
#[serde(deny_unknown_fields)]
#[schemars(deny_unknown_fields)]
pub(super) struct MergedNode {
pub(super) name: String,
pub(super) description: String,
pub(super) merged_from: Vec<Node>,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
#[serde(deny_unknown_fields)]
#[schemars(deny_unknown_fields)]
pub(super) struct SimpleNode {
pub(super) id: NodeId,
pub(super) children: Vec<Node>,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
pub(super) enum Node {
Simple(SimpleNode),
Merged(MergedNode),
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
#[serde(deny_unknown_fields)]
#[schemars(deny_unknown_fields)]
pub(super) struct MergeResponse {
pub(super) root: Node,
}
pub(super) fn schema() -> Result<serde_json::Value, String> {
strict_json_schema::<MergeResponse>()
}
fn project_node(node: &LayoutNode) -> OriginalNode {
OriginalNode {
id: node.id.clone(),
metadata: node.metadata.clone(),
children: node.children.iter().map(project_node).collect(),
}
}
pub(super) fn input_trees(
state: &State,
priorities: &[u32],
) -> Result<Vec<OriginalTree>, String> {
if state.ui_trees.len() != priorities.len() {
return Err("UI 树优先级数量不匹配".to_string());
}
Ok(state
.ui_trees
.iter()
.zip(priorities)
.map(|(tree, priority)| OriginalTree {
priority: *priority,
root: project_node(&tree.root),
})
.collect())
}
}
mod priority {
use crate::ui_editor::resource::ui_design_image::UIDesignImage;
use crate::ui_editor::state::State;
use crate::ui_editor::utils::UIDesignImageId;
use std::collections::{HashMap, HashSet};
fn ancestor_chain(
image_id: &UIDesignImageId,
images: &HashMap<UIDesignImageId, UIDesignImage>,
) -> Result<Vec<UIDesignImageId>, String> {
let mut chain = Vec::new();
let mut seen = HashSet::new();
let mut current = image_id.clone();
loop {
if !seen.insert(current.clone()) {
return Err(format!(
"界面图 slave_to 关系存在循环:{}",
current.as_str()
));
}
chain.push(current.clone());
let image = images
.get(&current)
.ok_or_else(|| format!("界面图 slave_to 引用了缺失界面图:{}", current.as_str()))?;
match &image.metadata.slave_to {
Some(parent) => current = parent.clone(),
None => return Ok(chain),
}
}
}
pub(super) fn for_state(state: &State) -> Result<Vec<u32>, String> {
let source_ids = state
.ui_trees
.iter()
.map(|tree| tree.src_ui_design.clone())
.collect::<Vec<_>>();
let chains = source_ids
.iter()
.map(|source_id| ancestor_chain(source_id, &state.ui_design_images))
.collect::<Result<Vec<_>, _>>()?;
Ok(source_ids
.iter()
.map(|candidate| {
chains
.iter()
.filter(|chain| chain.contains(candidate))
.count() as u32
})
.collect())
}
}
mod materialize {
use super::llm_contract::Node;
use crate::ui_editor::layout::node::{
Node as LayoutNode, NodeMetadata, NodeSource, NodeStatus,
};
use crate::ui_editor::layout::transform::Transform;
use crate::ui_editor::state::{State, UITree};
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
use std::collections::{HashMap, HashSet};
#[derive(Clone)]
struct OriginalNodeRecord {
node: LayoutNode,
priority: u32,
src_ui_design: UIDesignImageId,
}
struct BuiltNode {
node: LayoutNode,
priority: u32,
src_ui_design: UIDesignImageId,
}
fn collect_original_nodes(
node: &LayoutNode,
priority: u32,
src_ui_design: &UIDesignImageId,
records: &mut HashMap<NodeId, OriginalNodeRecord>,
) -> Result<(), String> {
if records
.insert(
node.id.clone(),
OriginalNodeRecord {
node: node.clone(),
priority,
src_ui_design: src_ui_design.clone(),
},
)
.is_some()
{
return Err(format!("输入 UI 树包含重复节点 ID{}", node.id.as_str()));
}
for child in &node.children {
collect_original_nodes(child, priority, src_ui_design, records)?;
}
Ok(())
}
fn oldest_member_index(members: &[BuiltNode]) -> Result<usize, String> {
if members.is_empty() {
return Err("MergedNode.merged_from 不能为空".to_string());
}
let mut best_index = 0;
let mut best_priority = 0;
for (candidate_index, candidate) in members.iter().enumerate() {
if candidate.priority > best_priority {
best_index = candidate_index;
best_priority = candidate.priority;
}
}
Ok(best_index)
}
fn random_node_id(occupied: &mut HashSet<NodeId>) -> Result<NodeId, String> {
loop {
let id = NodeId::new(uuid::Uuid::new_v4().simple().to_string())
.map_err(|error| format!("生成合并容器节点 ID 失败:{error}"))?;
if occupied.insert(id.clone()) {
return Ok(id);
}
}
}
fn build_node(
plan: Node,
records: &HashMap<NodeId, OriginalNodeRecord>,
used_original_ids: &mut HashSet<NodeId>,
occupied_ids: &mut HashSet<NodeId>,
) -> Result<BuiltNode, String> {
match plan {
Node::Simple(simple) => {
if !used_original_ids.insert(simple.id.clone()) {
return Err(format!("合并计划重复引用节点 ID{}", simple.id.as_str()));
}
let original = records
.get(&simple.id)
.ok_or_else(|| format!("合并计划引用了未知节点 ID{}", simple.id.as_str()))?;
let mut node = original.node.clone();
node.children = simple
.children
.into_iter()
.map(|child| {
build_node(child, records, used_original_ids, occupied_ids)
.map(|built| built.node)
})
.collect::<Result<Vec<_>, _>>()?;
Ok(BuiltNode {
node,
priority: original.priority,
src_ui_design: original.src_ui_design.clone(),
})
}
Node::Merged(merged) => {
let container_name = merged.name;
let container_description = merged.description;
let mut members = merged
.merged_from
.into_iter()
.map(|member| build_node(member, records, used_original_ids, occupied_ids))
.collect::<Result<Vec<_>, _>>()?;
let oldest_index = oldest_member_index(&members)?;
let original_transform = members[oldest_index].node.transform;
let priority = members[oldest_index].priority;
let src_ui_design = members[oldest_index].src_ui_design.clone();
for member in &mut members {
member.node.transform = Transform::stretch();
}
Ok(BuiltNode {
node: LayoutNode {
id: random_node_id(occupied_ids)?,
transform: original_transform,
metadata: NodeMetadata {
name: container_name,
description: container_description,
status: NodeStatus::Passed,
source: NodeSource::Llm,
},
components: Vec::new(),
children: members.into_iter().map(|member| member.node).collect(),
},
priority,
src_ui_design,
})
}
}
}
pub(super) fn plan(plan: Node, state: &State, priorities: &[u32]) -> Result<UITree, String> {
if state.ui_trees.len() != priorities.len() {
return Err("UI 树优先级数量不匹配".to_string());
}
let mut records = HashMap::new();
for (tree, priority) in state.ui_trees.iter().zip(priorities) {
collect_original_nodes(&tree.root, *priority, &tree.src_ui_design, &mut records)?;
}
let mut occupied_ids = records.keys().cloned().collect::<HashSet<_>>();
let mut used_original_ids = HashSet::new();
let built = build_node(plan, &records, &mut used_original_ids, &mut occupied_ids)?;
Ok(UITree {
src_ui_design: built.src_ui_design,
root: built.node,
})
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct MergeDTO {
pub ui_tree: UITree,
}
pub(crate) async fn merge_ui_impl(state: State) -> Result<MergeDTO, String> {
if state.ui_trees.is_empty() {
eprintln!("ui_merge.error stage=validate reason=no_trees");
return Err("请先完成 UI 结构识别".to_string());
}
let priorities = priority::for_state(&state).map_err(|error| {
eprintln!("ui_merge.error stage=build_priority error={error}");
error
})?;
let trees = llm_contract::input_trees(&state, &priorities).map_err(|error| {
eprintln!("ui_merge.error stage=build_input error={error}");
error
})?;
let records_json = serde_json::to_string(&trees).map_err(|error| {
eprintln!("ui_merge.error stage=serialize_input error={error}");
format!("序列化 UI 合并输入失败:{error}")
})?;
let client = build_game_creator_llm_client_from_config().map_err(|error| {
eprintln!("ui_merge.error stage=build_client error={error}");
error
})?;
let schema = llm_contract::schema().map_err(|error| {
eprintln!("ui_merge.error stage=build_schema error={error}");
error
})?;
let tool = LlmFunctionTool::new(
MERGE_TOOL_NAME,
"把多张参考图各自的 UI 树合并为一棵新的 UI 计划树",
schema,
)
.with_strict(true);
let response = client
.run(
LlmRunRequest::new(vec![
LlmMessage::system(SYSTEM_PROMPT),
LlmMessage::user(format!("待合并 UI 树:\n{records_json}")),
])
.with_function_tools(vec![tool])
.with_tool_choice(LlmToolChoice::Required),
)
.await
.map_err(|error| {
eprintln!("ui_merge.error stage=llm_request error={error}");
format!("UI 树合并失败:{error}")
})?;
let call = response
.tool_calls
.iter()
.find(|call| call.name == MERGE_TOOL_NAME)
.ok_or_else(|| {
eprintln!("ui_merge.error stage=parse_tool_call reason=missing_tool_call");
format!("LLM 未返回 {MERGE_TOOL_NAME} 工具调用")
})?;
let parsed =
serde_json::from_str::<llm_contract::MergeResponse>(&call.arguments).map_err(|error| {
eprintln!("ui_merge.error stage=parse_arguments error={error}");
format!("UI 合并工具参数无效:{error}")
})?;
let ui_tree = materialize::plan(parsed.root, &state, &priorities).map_err(|error| {
eprintln!("ui_merge.error stage=materialize error={error}");
error
})?;
Ok(MergeDTO { ui_tree })
}
@@ -1,7 +1,11 @@
pub mod merge;
pub mod recognition;
pub mod ui_design_suggestion;
pub mod utils;
pub mod binding;
pub(crate) use merge::merge_ui_impl;
pub use merge::MergeDTO;
pub(crate) use recognition::recognize_ui_impl;
pub use recognition::RecognitionDTO;
pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl;
@@ -13,7 +13,7 @@ use platform_llm::{
};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::path::Path;
use ts_rs::TS;
@@ -30,17 +30,17 @@ const SYSTEM_PROMPT: &str = r#"
同时分析同一 UI 系统的全部参考图,建立UI树
用户会给你一些UI截图(它们从属于同一个UI系统)和对应的元数据, 请用给定的工具描述UI结构
你是游戏 UI 结构识别器。
识别规则:
* 只识别 UI,不识别场景人物、地形、建筑、光影和背景装饰。
* 不创建没有独立功能的微小装饰Node。
* 无法确定类型、层级、关系时,在 UnSure 中写明原因。
* source=user、reviewStatus=confirmed 或 semanticSource=user 的组件不得被 AI 覆盖
* 每个Node必须使用 src_ui_design_image_id 指定坐标来源(所属UI design图),并在该图片原始像素坐标系输出
global_pos_x_px、global_pos_y_px、width_px、height_px坐标相对于来源图片左上角
* 多张图都是在描述同一个UI系统, 必须合并共用公共的框架/层次, 禁止简单每个图一个树, 坐标等有矛盾的地方父级优先
* 返回的 trees 必须与输入图片一一对应,每张输入图片只能有一棵树,不能合并多张图片的树。 每棵树的 src_ui_design_image_id 必须等于对应输入图片标注的 id
* 每棵树必须使用自己的输入图片原始像素坐标系(0,0 is left top)输出
global_pos_x_px、global_pos_y_px、width_px、height_px
* 面向用户的字段如名称描述等请用中文
* 由于每个截图未必是完整的, 可能是局部的, 每棵树描述清楚每个截图上UI的层次结构即可
* 不同树的共用框架/层次/...请使用使用相同的名称描述. 不同状态/变体名称使用相同的前缀, 用后缀区别
"#;
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
@@ -98,7 +98,6 @@ struct RecognitionNode {
global_pos_y_px: u32,
width_px: u32,
height_px: u32,
src_ui_design_image_id: UIDesignImageId,
local_anchor: Anchor,
name: String,
description: String,
@@ -108,10 +107,17 @@ struct RecognitionNode {
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
#[schemars(deny_unknown_fields)]
struct RecognitionResponse {
struct RecognitionTree {
src_ui_design_image_id: UIDesignImageId,
children: Vec<RecognitionNode>,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
#[schemars(deny_unknown_fields)]
struct RecognitionResponse {
trees: Vec<RecognitionTree>,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct RecognitionDTO {
@@ -195,23 +201,17 @@ fn image_layout_size(image: &UIDesignImage) -> Result<Vector2<f32>, String> {
fn convert_node(
source: &RecognitionNode,
page_image: &UIDesignImage,
source_image: &UIDesignImage,
images: &HashMap<UIDesignImageId, UIDesignImage>,
image_id: &UIDesignImageId,
image: &UIDesignImage,
parent_rect: UIRect,
) -> Result<LayoutNode, String> {
// LLM 输出的是 source_image 原始像素。Rust 先按来源图尺寸归一化,
// 再缩放到 Page 图像素尺寸,最后使用 Page 的 pixels_per_unit 转成布局单位。
let source_size_px = source_image.pixel_size;
let page_size_px = page_image.pixel_size;
let source_size_valid = source_size_px
// LLM 输出的是当前树所属图片的原始像素。Rust 只按该图片自己的
// pixels_per_unit 转成布局单位,绝不把坐标归一化或映射到 Page 图片
let image_size_px = image.pixel_size;
let image_size_valid = image_size_px
.iter()
.all(|value| value.is_finite() && *value > 0.0);
if !source_size_valid
|| !page_size_px
.iter()
.all(|value| value.is_finite() && *value > 0.0)
{
if !image_size_valid {
return Err("界面图尺寸无效".to_string());
}
let source_min_px = Vector2::new(source.global_pos_x_px as f32, source.global_pos_y_px as f32);
@@ -220,19 +220,19 @@ fn convert_node(
let mut blocked = source.width_px == 0 || source.height_px == 0;
if source_min_px.x < 0.0
|| source_min_px.y < 0.0
|| source_max_px.x > source_size_px.x
|| source_max_px.y > source_size_px.y
|| source_max_px.x > image_size_px.x
|| source_max_px.y > image_size_px.y
{
blocked = true;
}
let scale = page_size_px.component_div(&source_size_px);
let page_min_px = source_min_px.component_mul(&scale);
let page_size_node_px = source_size_node_px.component_mul(&scale);
let ppu = page_image.pixels_per_unit.get();
let target_min = Point2::new(page_min_px.x / ppu, page_min_px.y / ppu);
let ppu = image.pixels_per_unit.get();
if !ppu.is_finite() || ppu <= 0.0 {
return Err("界面图 pixels_per_unit 无效".to_string());
}
let target_min = Point2::new(source_min_px.x / ppu, source_min_px.y / ppu);
let target_rect = UIRect::new(
target_min,
Vector2::new(page_size_node_px.x / ppu, page_size_node_px.y / ppu),
Vector2::new(source_size_node_px.x / ppu, source_size_node_px.y / ppu),
);
let (anchor_min, anchor_max) = anchor_ranges(&source.local_anchor)
.unwrap_or((Vector2::new(0.5, 0.5), Vector2::new(0.5, 0.5)));
@@ -262,16 +262,7 @@ fn convert_node(
let children = source
.children
.iter()
.map(|child| {
let child_source_image =
images.get(&child.src_ui_design_image_id).ok_or_else(|| {
format!(
"节点引用了缺失界面图 {}",
child.src_ui_design_image_id.as_str()
)
})?;
convert_node(child, page_image, child_source_image, images, target_rect)
})
.map(|child| convert_node(child, image_id, image, target_rect))
.collect::<Result<Vec<_>, _>>()?;
Ok(LayoutNode {
id: random_node_id()?,
@@ -300,18 +291,23 @@ fn validate_confidence(nodes: &[RecognitionNode]) -> Result<(), String> {
Ok(())
}
fn validate_source_image_ids(
nodes: &[RecognitionNode],
fn validate_tree_image_ids(
trees: &[RecognitionTree],
allowed_ids: &[UIDesignImageId],
) -> Result<(), String> {
for node in nodes {
if !allowed_ids
.iter()
.any(|allowed| allowed == &node.src_ui_design_image_id)
{
return Err("LLM 返回了不属于当前 Page 上下文的节点来源界面图 ID".to_string());
let allowed = allowed_ids.iter().cloned().collect::<HashSet<_>>();
let mut seen = HashSet::new();
for tree in trees {
if !allowed.contains(&tree.src_ui_design_image_id) {
return Err("LLM 返回了不属于当前 Page 上下文的界面图 ID".to_string());
}
validate_source_image_ids(&node.children, allowed_ids)?;
if !seen.insert(tree.src_ui_design_image_id.clone()) {
return Err("LLM 为同一界面图返回了重复 UI 树".to_string());
}
validate_confidence(&tree.children)?;
}
if seen.len() != allowed.len() {
return Err("LLM 未为当前 Page 上下文的每张界面图返回 UI 树".to_string());
}
Ok(())
}
@@ -320,6 +316,43 @@ fn validate_source_image_ids(
mod tests {
use super::*;
fn test_image() -> UIDesignImage {
UIDesignImage {
metadata: crate::ui_editor::resource::ui_design_image::UIDesignImageMetadata {
name: "测试图".to_string(),
description: String::new(),
role: None,
slave_to: None,
},
path: "test.png".to_string(),
pixel_size: Vector2::new(1000.0, 500.0),
pixels_per_unit: typed_floats::tf32::StrictlyPositiveFinite::new(2.0)
.expect("valid pixels per unit"),
}
}
fn test_node() -> RecognitionNode {
RecognitionNode {
global_pos_x_px: 100,
global_pos_y_px: 50,
width_px: 200,
height_px: 100,
local_anchor: Anchor::Preset(PresetAnchor {
horizontal: HorizonalAnchor::Left,
vertical: VerticalAnchor::Top,
}),
name: "节点".to_string(),
description: String::new(),
children: Vec::new(),
confidence: Confidence::Confident,
}
}
fn assert_rect_close(actual: UIRect, expected: UIRect) {
assert!((actual.min - expected.min).norm() <= 1.0e-4);
assert!((actual.size - expected.size).norm() <= 1.0e-4);
}
fn assert_no_one_of(value: &serde_json::Value) {
match value {
serde_json::Value::Object(object) => {
@@ -372,7 +405,6 @@ mod tests {
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
src_ui_design_image_id: UIDesignImageId::new("page").expect("image id"),
local_anchor: Anchor::Preset(PresetAnchor {
horizontal: HorizonalAnchor::Left,
vertical: VerticalAnchor::Top,
@@ -384,6 +416,43 @@ mod tests {
};
assert!(validate_confidence(&[node]).is_err());
}
#[test]
fn conversion_stays_in_the_tree_image_coordinate_system() {
let image_id = UIDesignImageId::new("slave").expect("valid image id");
let image = test_image();
let root_rect = UIRect::new(Point2::origin(), image_layout_size(&image).unwrap());
let converted =
convert_node(&test_node(), &image_id, &image, root_rect).expect("node conversion");
assert_rect_close(
converted.transform.resolve(&root_rect),
UIRect::new(Point2::new(50.0, 25.0), Vector2::new(100.0, 50.0)),
);
}
#[test]
fn tree_validation_requires_exactly_one_tree_per_context_image() {
let page = UIDesignImageId::new("page").expect("valid image id");
let slave = UIDesignImageId::new("slave").expect("valid image id");
let tree = |id: UIDesignImageId| RecognitionTree {
src_ui_design_image_id: id,
children: Vec::new(),
};
assert!(validate_tree_image_ids(
&[tree(page.clone()), tree(slave.clone())],
&[page.clone(), slave.clone()],
)
.is_ok());
assert!(
validate_tree_image_ids(&[tree(page.clone())], &[page.clone(), slave.clone()]).is_err()
);
assert!(
validate_tree_image_ids(&[tree(page.clone()), tree(page.clone())], &[page, slave],)
.is_err()
);
}
}
fn page_image_ids(state: &State) -> Vec<UIDesignImageId> {
@@ -439,14 +508,10 @@ pub(crate) async fn recognize_ui_impl(
let root = Path::new(project_path.trim());
let mut ui_trees = Vec::with_capacity(page_ids.len());
for page_id in page_ids {
let page = state
.ui_design_images
.get(&page_id)
.ok_or_else(|| "缺少 Page 界面图资源".to_string())?;
let mut context_ids = vec![page_id.clone()];
context_ids.extend(slave_image_ids(&state, &page_id));
// The model may cite only this Page or its direct slave images as evidence.
let allowed_source_ids = context_ids.clone();
// 当前只支持 Page 的直接 slave;嵌套 slave 关系的识别批次待后续设计。
// TODO: 支持 slave_to 链的递归分组和独立识别。
let mut parts = Vec::with_capacity(context_ids.len() * 2);
for (index, context_id) in context_ids.iter().enumerate() {
let image = state
@@ -484,7 +549,7 @@ pub(crate) async fn recognize_ui_impl(
}
let tool = LlmFunctionTool::new(
"recognize_ui_structure",
"识别当前 Page 及其 slave 参考图中的 UI 节点,并只返回一棵树的节点列表",
"识别当前 Page 及其直接 slave 参考图,并为每张输入图片各返回一棵 UI ",
schema.clone(),
)
.with_strict(true);
@@ -533,57 +598,52 @@ pub(crate) async fn recognize_ui_impl(
);
format!("Page {} 的识别工具参数无效:{error}", page_id.as_str())
})?;
validate_confidence(&parsed.children).map_err(|error| {
validate_tree_image_ids(&parsed.trees, &context_ids).map_err(|error| {
eprintln!(
"ui_recognition.error stage=validate_confidence page={} error={error}",
"ui_recognition.error stage=validate_trees page={} error={error}",
page_id.as_str()
);
error
})?;
validate_source_image_ids(&parsed.children, &allowed_source_ids).map_err(|error| {
eprintln!(
"ui_recognition.error stage=validate_source_ids page={} error={error}",
page_id.as_str()
);
error
})?;
let size = image_layout_size(page)?;
let root_rect = UIRect::new(Point2::origin(), size);
let children = parsed
.children
.iter()
.map(|node| {
let source_image = state
.ui_design_images
.get(&node.src_ui_design_image_id)
.ok_or_else(|| {
format!(
"Page {} 的节点引用了缺失界面图 {}",
page_id.as_str(),
node.src_ui_design_image_id.as_str()
)
})?;
convert_node(node, page, source_image, &state.ui_design_images, root_rect)
})
.collect::<Result<Vec<_>, _>>()?;
let root = LayoutNode {
id: random_node_id()?,
transform: Transform::stretch(),
metadata: NodeMetadata {
name: "页面根节点".to_string(),
description: String::new(),
status: NodeStatus::Passed,
source: NodeSource::System,
},
// TODO: root components can host page-level background content once component preview is supported.
components: Vec::new(),
children,
};
// Tree identity and root identity are assigned by Rust, never chosen by the model.
ui_trees.push(UITree {
src_ui_design: page_id,
root,
});
let mut parsed_trees = parsed
.trees
.into_iter()
.map(|tree| (tree.src_ui_design_image_id.clone(), tree))
.collect::<HashMap<_, _>>();
for image_id in context_ids {
let image = state
.ui_design_images
.get(&image_id)
.ok_or_else(|| "缺少识别上下文界面图资源".to_string())?;
let tree = parsed_trees
.remove(&image_id)
.ok_or_else(|| format!("缺少界面图 {} 的识别树", image_id.as_str()))?;
let size = image_layout_size(image)?;
let root_rect = UIRect::new(Point2::origin(), size);
let children = tree
.children
.iter()
.map(|node| convert_node(node, &image_id, image, root_rect))
.collect::<Result<Vec<_>, _>>()?;
let root = LayoutNode {
id: random_node_id()?,
transform: Transform::stretch(),
metadata: NodeMetadata {
name: "页面根节点".to_string(),
description: String::new(),
status: NodeStatus::Passed,
source: NodeSource::System,
},
// TODO: root components can host page-level background content once component preview is supported.
components: Vec::new(),
children,
};
// Tree identity and root identity are assigned by Rust, never chosen by the model.
ui_trees.push(UITree {
src_ui_design: image_id,
root,
});
}
}
Ok(RecognitionDTO { ui_trees })
}
@@ -0,0 +1,10 @@
import type { MergeDTO } from './types/MergeDTO';
import type { State } from './types/State';
/** 合并阶段返回一棵新树,成功后替换当前全部逐图识别树。 */
export function applyMergeResult(state: State, result: MergeDTO): State {
return {
...structuredClone(state),
ui_trees: [structuredClone(result.ui_tree)],
};
}
@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { UITree } from "./UITree";
export type MergeDTO = { ui_tree: UITree, };