扩展和优化 UI 识别逻辑:
- 实现多图片输入的 UI 树分离与独立验证。 - 引入 `RecognitionTree` 数据结构,支持每张输入图片一棵独立树。 - 重构节点转换逻辑,统一 pixels_per_unit 坐标系统。 - 添加逻辑以验证树与上下文图片的一一对应关系。 - 优化节点转换函数和 UI 树生成流程,提高结构一致性。 - 补充测试用例,验证树结构生成和转换正确性。
This commit is contained in:
@@ -126,6 +126,11 @@ async fn recognize_ui(
|
||||
ui_editor::commands::recognize_ui_impl(project_path, state).await
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
async fn merge_ui(state: ui_editor::state::State) -> Result<ui_editor::commands::MergeDTO, String> {
|
||||
ui_editor::commands::merge_ui_impl(state).await
|
||||
}
|
||||
|
||||
#[derive(Debug, Eq, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct InitLocalProjectResult {
|
||||
@@ -2261,6 +2266,7 @@ fn main() {
|
||||
import_ui_editor_remote_assets,
|
||||
suggest_ui_design_semantic,
|
||||
recognize_ui,
|
||||
merge_ui,
|
||||
generate_platform_art_asset,
|
||||
open_canvas_project,
|
||||
get_game_creation_agent_capabilities,
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
use crate::config::build_game_creator_llm_client_from_config;
|
||||
use crate::ui_editor::commands::utils::strict_json_schema;
|
||||
use crate::ui_editor::state::{State, UITree};
|
||||
use platform_llm::{LlmFunctionTool, LlmMessage, LlmRunRequest, LlmToolChoice};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use ts_rs::TS;
|
||||
|
||||
const MERGE_TOOL_NAME: &str = "merge_ui_trees";
|
||||
|
||||
const SYSTEM_PROMPT: &str = r#"
|
||||
角色:
|
||||
你是游戏 UI 多树结构合并器。
|
||||
|
||||
任务:
|
||||
用户会提供同一个 UI 系统中多张参考图各自的 UI 树。请返回一棵新的合并计划树。
|
||||
|
||||
规则:
|
||||
* Simple 引用一个原始节点 id,并用返回的 children 定义它在新树中的子节点。
|
||||
* Merged 表示多个节点描述同一个共同组件(它们可能是同一组件的不同状态),merged_from 中按语义排列需要放入容器的节点。
|
||||
* 可以在任意层级使用 Merged,不限于各输入树的根节点。
|
||||
* 一些共用的框架/层次/...在树中只保留一个, 优先保留优先级高的树中的节点
|
||||
* 面向用户的节点名称和结构判断使用中文语义理解。
|
||||
"#;
|
||||
|
||||
mod llm_contract {
|
||||
use super::strict_json_schema;
|
||||
use crate::ui_editor::layout::node::{Node as LayoutNode, NodeMetadata};
|
||||
use crate::ui_editor::state::State;
|
||||
use crate::ui_editor::utils::NodeId;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(super) struct OriginalNode {
|
||||
id: NodeId,
|
||||
metadata: NodeMetadata,
|
||||
children: Vec<OriginalNode>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(super) struct OriginalTree {
|
||||
priority: u32,
|
||||
root: OriginalNode,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
pub(super) struct MergedNode {
|
||||
pub(super) name: String,
|
||||
pub(super) description: String,
|
||||
pub(super) merged_from: Vec<Node>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
pub(super) struct SimpleNode {
|
||||
pub(super) id: NodeId,
|
||||
pub(super) children: Vec<Node>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
pub(super) enum Node {
|
||||
Simple(SimpleNode),
|
||||
Merged(MergedNode),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
pub(super) struct MergeResponse {
|
||||
pub(super) root: Node,
|
||||
}
|
||||
|
||||
pub(super) fn schema() -> Result<serde_json::Value, String> {
|
||||
strict_json_schema::<MergeResponse>()
|
||||
}
|
||||
|
||||
fn project_node(node: &LayoutNode) -> OriginalNode {
|
||||
OriginalNode {
|
||||
id: node.id.clone(),
|
||||
metadata: node.metadata.clone(),
|
||||
children: node.children.iter().map(project_node).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn input_trees(
|
||||
state: &State,
|
||||
priorities: &[u32],
|
||||
) -> Result<Vec<OriginalTree>, String> {
|
||||
if state.ui_trees.len() != priorities.len() {
|
||||
return Err("UI 树优先级数量不匹配".to_string());
|
||||
}
|
||||
Ok(state
|
||||
.ui_trees
|
||||
.iter()
|
||||
.zip(priorities)
|
||||
.map(|(tree, priority)| OriginalTree {
|
||||
priority: *priority,
|
||||
root: project_node(&tree.root),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
mod priority {
|
||||
use crate::ui_editor::resource::ui_design_image::UIDesignImage;
|
||||
use crate::ui_editor::state::State;
|
||||
use crate::ui_editor::utils::UIDesignImageId;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
fn ancestor_chain(
|
||||
image_id: &UIDesignImageId,
|
||||
images: &HashMap<UIDesignImageId, UIDesignImage>,
|
||||
) -> Result<Vec<UIDesignImageId>, String> {
|
||||
let mut chain = Vec::new();
|
||||
let mut seen = HashSet::new();
|
||||
let mut current = image_id.clone();
|
||||
loop {
|
||||
if !seen.insert(current.clone()) {
|
||||
return Err(format!(
|
||||
"界面图 slave_to 关系存在循环:{}",
|
||||
current.as_str()
|
||||
));
|
||||
}
|
||||
chain.push(current.clone());
|
||||
let image = images
|
||||
.get(¤t)
|
||||
.ok_or_else(|| format!("界面图 slave_to 引用了缺失界面图:{}", current.as_str()))?;
|
||||
match &image.metadata.slave_to {
|
||||
Some(parent) => current = parent.clone(),
|
||||
None => return Ok(chain),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn for_state(state: &State) -> Result<Vec<u32>, String> {
|
||||
let source_ids = state
|
||||
.ui_trees
|
||||
.iter()
|
||||
.map(|tree| tree.src_ui_design.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let chains = source_ids
|
||||
.iter()
|
||||
.map(|source_id| ancestor_chain(source_id, &state.ui_design_images))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(source_ids
|
||||
.iter()
|
||||
.map(|candidate| {
|
||||
chains
|
||||
.iter()
|
||||
.filter(|chain| chain.contains(candidate))
|
||||
.count() as u32
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
mod materialize {
|
||||
use super::llm_contract::Node;
|
||||
use crate::ui_editor::layout::node::{
|
||||
Node as LayoutNode, NodeMetadata, NodeSource, NodeStatus,
|
||||
};
|
||||
use crate::ui_editor::layout::transform::Transform;
|
||||
use crate::ui_editor::state::{State, UITree};
|
||||
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct OriginalNodeRecord {
|
||||
node: LayoutNode,
|
||||
priority: u32,
|
||||
src_ui_design: UIDesignImageId,
|
||||
}
|
||||
|
||||
struct BuiltNode {
|
||||
node: LayoutNode,
|
||||
priority: u32,
|
||||
src_ui_design: UIDesignImageId,
|
||||
}
|
||||
|
||||
fn collect_original_nodes(
|
||||
node: &LayoutNode,
|
||||
priority: u32,
|
||||
src_ui_design: &UIDesignImageId,
|
||||
records: &mut HashMap<NodeId, OriginalNodeRecord>,
|
||||
) -> Result<(), String> {
|
||||
if records
|
||||
.insert(
|
||||
node.id.clone(),
|
||||
OriginalNodeRecord {
|
||||
node: node.clone(),
|
||||
priority,
|
||||
src_ui_design: src_ui_design.clone(),
|
||||
},
|
||||
)
|
||||
.is_some()
|
||||
{
|
||||
return Err(format!("输入 UI 树包含重复节点 ID:{}", node.id.as_str()));
|
||||
}
|
||||
for child in &node.children {
|
||||
collect_original_nodes(child, priority, src_ui_design, records)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn oldest_member_index(members: &[BuiltNode]) -> Result<usize, String> {
|
||||
if members.is_empty() {
|
||||
return Err("MergedNode.merged_from 不能为空".to_string());
|
||||
}
|
||||
let mut best_index = 0;
|
||||
let mut best_priority = 0;
|
||||
for (candidate_index, candidate) in members.iter().enumerate() {
|
||||
if candidate.priority > best_priority {
|
||||
best_index = candidate_index;
|
||||
best_priority = candidate.priority;
|
||||
}
|
||||
}
|
||||
Ok(best_index)
|
||||
}
|
||||
|
||||
fn random_node_id(occupied: &mut HashSet<NodeId>) -> Result<NodeId, String> {
|
||||
loop {
|
||||
let id = NodeId::new(uuid::Uuid::new_v4().simple().to_string())
|
||||
.map_err(|error| format!("生成合并容器节点 ID 失败:{error}"))?;
|
||||
if occupied.insert(id.clone()) {
|
||||
return Ok(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_node(
|
||||
plan: Node,
|
||||
records: &HashMap<NodeId, OriginalNodeRecord>,
|
||||
used_original_ids: &mut HashSet<NodeId>,
|
||||
occupied_ids: &mut HashSet<NodeId>,
|
||||
) -> Result<BuiltNode, String> {
|
||||
match plan {
|
||||
Node::Simple(simple) => {
|
||||
if !used_original_ids.insert(simple.id.clone()) {
|
||||
return Err(format!("合并计划重复引用节点 ID:{}", simple.id.as_str()));
|
||||
}
|
||||
let original = records
|
||||
.get(&simple.id)
|
||||
.ok_or_else(|| format!("合并计划引用了未知节点 ID:{}", simple.id.as_str()))?;
|
||||
let mut node = original.node.clone();
|
||||
node.children = simple
|
||||
.children
|
||||
.into_iter()
|
||||
.map(|child| {
|
||||
build_node(child, records, used_original_ids, occupied_ids)
|
||||
.map(|built| built.node)
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(BuiltNode {
|
||||
node,
|
||||
priority: original.priority,
|
||||
src_ui_design: original.src_ui_design.clone(),
|
||||
})
|
||||
}
|
||||
Node::Merged(merged) => {
|
||||
let container_name = merged.name;
|
||||
let container_description = merged.description;
|
||||
let mut members = merged
|
||||
.merged_from
|
||||
.into_iter()
|
||||
.map(|member| build_node(member, records, used_original_ids, occupied_ids))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let oldest_index = oldest_member_index(&members)?;
|
||||
let original_transform = members[oldest_index].node.transform;
|
||||
let priority = members[oldest_index].priority;
|
||||
let src_ui_design = members[oldest_index].src_ui_design.clone();
|
||||
for member in &mut members {
|
||||
member.node.transform = Transform::stretch();
|
||||
}
|
||||
Ok(BuiltNode {
|
||||
node: LayoutNode {
|
||||
id: random_node_id(occupied_ids)?,
|
||||
transform: original_transform,
|
||||
metadata: NodeMetadata {
|
||||
name: container_name,
|
||||
description: container_description,
|
||||
status: NodeStatus::Passed,
|
||||
source: NodeSource::Llm,
|
||||
},
|
||||
components: Vec::new(),
|
||||
children: members.into_iter().map(|member| member.node).collect(),
|
||||
},
|
||||
priority,
|
||||
src_ui_design,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn plan(plan: Node, state: &State, priorities: &[u32]) -> Result<UITree, String> {
|
||||
if state.ui_trees.len() != priorities.len() {
|
||||
return Err("UI 树优先级数量不匹配".to_string());
|
||||
}
|
||||
let mut records = HashMap::new();
|
||||
for (tree, priority) in state.ui_trees.iter().zip(priorities) {
|
||||
collect_original_nodes(&tree.root, *priority, &tree.src_ui_design, &mut records)?;
|
||||
}
|
||||
let mut occupied_ids = records.keys().cloned().collect::<HashSet<_>>();
|
||||
let mut used_original_ids = HashSet::new();
|
||||
let built = build_node(plan, &records, &mut used_original_ids, &mut occupied_ids)?;
|
||||
Ok(UITree {
|
||||
src_ui_design: built.src_ui_design,
|
||||
root: built.node,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct MergeDTO {
|
||||
pub ui_tree: UITree,
|
||||
}
|
||||
|
||||
pub(crate) async fn merge_ui_impl(state: State) -> Result<MergeDTO, String> {
|
||||
if state.ui_trees.is_empty() {
|
||||
eprintln!("ui_merge.error stage=validate reason=no_trees");
|
||||
return Err("请先完成 UI 结构识别".to_string());
|
||||
}
|
||||
let priorities = priority::for_state(&state).map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=build_priority error={error}");
|
||||
error
|
||||
})?;
|
||||
let trees = llm_contract::input_trees(&state, &priorities).map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=build_input error={error}");
|
||||
error
|
||||
})?;
|
||||
let records_json = serde_json::to_string(&trees).map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=serialize_input error={error}");
|
||||
format!("序列化 UI 合并输入失败:{error}")
|
||||
})?;
|
||||
let client = build_game_creator_llm_client_from_config().map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=build_client error={error}");
|
||||
error
|
||||
})?;
|
||||
let schema = llm_contract::schema().map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=build_schema error={error}");
|
||||
error
|
||||
})?;
|
||||
let tool = LlmFunctionTool::new(
|
||||
MERGE_TOOL_NAME,
|
||||
"把多张参考图各自的 UI 树合并为一棵新的 UI 计划树",
|
||||
schema,
|
||||
)
|
||||
.with_strict(true);
|
||||
let response = client
|
||||
.run(
|
||||
LlmRunRequest::new(vec![
|
||||
LlmMessage::system(SYSTEM_PROMPT),
|
||||
LlmMessage::user(format!("待合并 UI 树:\n{records_json}")),
|
||||
])
|
||||
.with_function_tools(vec![tool])
|
||||
.with_tool_choice(LlmToolChoice::Required),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=llm_request error={error}");
|
||||
format!("UI 树合并失败:{error}")
|
||||
})?;
|
||||
let call = response
|
||||
.tool_calls
|
||||
.iter()
|
||||
.find(|call| call.name == MERGE_TOOL_NAME)
|
||||
.ok_or_else(|| {
|
||||
eprintln!("ui_merge.error stage=parse_tool_call reason=missing_tool_call");
|
||||
format!("LLM 未返回 {MERGE_TOOL_NAME} 工具调用")
|
||||
})?;
|
||||
let parsed =
|
||||
serde_json::from_str::<llm_contract::MergeResponse>(&call.arguments).map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=parse_arguments error={error}");
|
||||
format!("UI 合并工具参数无效:{error}")
|
||||
})?;
|
||||
let ui_tree = materialize::plan(parsed.root, &state, &priorities).map_err(|error| {
|
||||
eprintln!("ui_merge.error stage=materialize error={error}");
|
||||
error
|
||||
})?;
|
||||
Ok(MergeDTO { ui_tree })
|
||||
}
|
||||
@@ -1,7 +1,11 @@
|
||||
pub mod merge;
|
||||
pub mod recognition;
|
||||
pub mod ui_design_suggestion;
|
||||
pub mod utils;
|
||||
pub mod binding;
|
||||
|
||||
pub(crate) use merge::merge_ui_impl;
|
||||
pub use merge::MergeDTO;
|
||||
pub(crate) use recognition::recognize_ui_impl;
|
||||
pub use recognition::RecognitionDTO;
|
||||
pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl;
|
||||
|
||||
@@ -13,7 +13,7 @@ use platform_llm::{
|
||||
};
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use ts_rs::TS;
|
||||
|
||||
@@ -30,17 +30,17 @@ const SYSTEM_PROMPT: &str = r#"
|
||||
同时分析同一 UI 系统的全部参考图,建立UI树
|
||||
|
||||
用户会给你一些UI截图(它们从属于同一个UI系统)和对应的元数据, 请用给定的工具描述UI结构
|
||||
你是游戏 UI 结构识别器。
|
||||
|
||||
识别规则:
|
||||
* 只识别 UI,不识别场景人物、地形、建筑、光影和背景装饰。
|
||||
* 不创建没有独立功能的微小装饰Node。
|
||||
* 无法确定类型、层级、关系时,在 UnSure 中写明原因。
|
||||
* source=user、reviewStatus=confirmed 或 semanticSource=user 的组件不得被 AI 覆盖。
|
||||
* 每个Node必须使用 src_ui_design_image_id 指定坐标来源(所属UI design图),并在该图片的原始像素坐标系中输出
|
||||
global_pos_x_px、global_pos_y_px、width_px、height_px;坐标相对于来源图片左上角
|
||||
* 多张图都是在描述同一个UI系统, 必须合并共用公共的框架/层次, 禁止简单每个图一个树, 坐标等有矛盾的地方父级优先
|
||||
* 返回的 trees 必须与输入图片一一对应,每张输入图片只能有一棵树,不能合并多张图片的树。 每棵树的 src_ui_design_image_id 必须等于对应输入图片标注的 id。
|
||||
* 每棵树必须使用自己的输入图片原始像素坐标系(0,0 is left top)输出
|
||||
global_pos_x_px、global_pos_y_px、width_px、height_px;
|
||||
* 面向用户的字段如名称描述等请用中文
|
||||
* 由于每个截图未必是完整的, 可能是局部的, 每棵树描述清楚每个截图上UI的层次结构即可
|
||||
* 不同树的共用框架/层次/...请使用使用相同的名称描述. 不同状态/变体名称使用相同的前缀, 用后缀区别
|
||||
"#;
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
@@ -98,7 +98,6 @@ struct RecognitionNode {
|
||||
global_pos_y_px: u32,
|
||||
width_px: u32,
|
||||
height_px: u32,
|
||||
src_ui_design_image_id: UIDesignImageId,
|
||||
local_anchor: Anchor,
|
||||
name: String,
|
||||
description: String,
|
||||
@@ -108,10 +107,17 @@ struct RecognitionNode {
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
struct RecognitionResponse {
|
||||
struct RecognitionTree {
|
||||
src_ui_design_image_id: UIDesignImageId,
|
||||
children: Vec<RecognitionNode>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
struct RecognitionResponse {
|
||||
trees: Vec<RecognitionTree>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct RecognitionDTO {
|
||||
@@ -195,23 +201,17 @@ fn image_layout_size(image: &UIDesignImage) -> Result<Vector2<f32>, String> {
|
||||
|
||||
fn convert_node(
|
||||
source: &RecognitionNode,
|
||||
page_image: &UIDesignImage,
|
||||
source_image: &UIDesignImage,
|
||||
images: &HashMap<UIDesignImageId, UIDesignImage>,
|
||||
image_id: &UIDesignImageId,
|
||||
image: &UIDesignImage,
|
||||
parent_rect: UIRect,
|
||||
) -> Result<LayoutNode, String> {
|
||||
// LLM 输出的是 source_image 原始像素。Rust 先按来源图尺寸归一化,
|
||||
// 再缩放到 Page 图像素尺寸,最后使用 Page 的 pixels_per_unit 转成布局单位。
|
||||
let source_size_px = source_image.pixel_size;
|
||||
let page_size_px = page_image.pixel_size;
|
||||
let source_size_valid = source_size_px
|
||||
// LLM 输出的是当前树所属图片的原始像素。Rust 只按该图片自己的
|
||||
// pixels_per_unit 转成布局单位,绝不把坐标归一化或映射到 Page 图片。
|
||||
let image_size_px = image.pixel_size;
|
||||
let image_size_valid = image_size_px
|
||||
.iter()
|
||||
.all(|value| value.is_finite() && *value > 0.0);
|
||||
if !source_size_valid
|
||||
|| !page_size_px
|
||||
.iter()
|
||||
.all(|value| value.is_finite() && *value > 0.0)
|
||||
{
|
||||
if !image_size_valid {
|
||||
return Err("界面图尺寸无效".to_string());
|
||||
}
|
||||
let source_min_px = Vector2::new(source.global_pos_x_px as f32, source.global_pos_y_px as f32);
|
||||
@@ -220,19 +220,19 @@ fn convert_node(
|
||||
let mut blocked = source.width_px == 0 || source.height_px == 0;
|
||||
if source_min_px.x < 0.0
|
||||
|| source_min_px.y < 0.0
|
||||
|| source_max_px.x > source_size_px.x
|
||||
|| source_max_px.y > source_size_px.y
|
||||
|| source_max_px.x > image_size_px.x
|
||||
|| source_max_px.y > image_size_px.y
|
||||
{
|
||||
blocked = true;
|
||||
}
|
||||
let scale = page_size_px.component_div(&source_size_px);
|
||||
let page_min_px = source_min_px.component_mul(&scale);
|
||||
let page_size_node_px = source_size_node_px.component_mul(&scale);
|
||||
let ppu = page_image.pixels_per_unit.get();
|
||||
let target_min = Point2::new(page_min_px.x / ppu, page_min_px.y / ppu);
|
||||
let ppu = image.pixels_per_unit.get();
|
||||
if !ppu.is_finite() || ppu <= 0.0 {
|
||||
return Err("界面图 pixels_per_unit 无效".to_string());
|
||||
}
|
||||
let target_min = Point2::new(source_min_px.x / ppu, source_min_px.y / ppu);
|
||||
let target_rect = UIRect::new(
|
||||
target_min,
|
||||
Vector2::new(page_size_node_px.x / ppu, page_size_node_px.y / ppu),
|
||||
Vector2::new(source_size_node_px.x / ppu, source_size_node_px.y / ppu),
|
||||
);
|
||||
let (anchor_min, anchor_max) = anchor_ranges(&source.local_anchor)
|
||||
.unwrap_or((Vector2::new(0.5, 0.5), Vector2::new(0.5, 0.5)));
|
||||
@@ -262,16 +262,7 @@ fn convert_node(
|
||||
let children = source
|
||||
.children
|
||||
.iter()
|
||||
.map(|child| {
|
||||
let child_source_image =
|
||||
images.get(&child.src_ui_design_image_id).ok_or_else(|| {
|
||||
format!(
|
||||
"节点引用了缺失界面图 {}",
|
||||
child.src_ui_design_image_id.as_str()
|
||||
)
|
||||
})?;
|
||||
convert_node(child, page_image, child_source_image, images, target_rect)
|
||||
})
|
||||
.map(|child| convert_node(child, image_id, image, target_rect))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(LayoutNode {
|
||||
id: random_node_id()?,
|
||||
@@ -300,18 +291,23 @@ fn validate_confidence(nodes: &[RecognitionNode]) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_source_image_ids(
|
||||
nodes: &[RecognitionNode],
|
||||
fn validate_tree_image_ids(
|
||||
trees: &[RecognitionTree],
|
||||
allowed_ids: &[UIDesignImageId],
|
||||
) -> Result<(), String> {
|
||||
for node in nodes {
|
||||
if !allowed_ids
|
||||
.iter()
|
||||
.any(|allowed| allowed == &node.src_ui_design_image_id)
|
||||
{
|
||||
return Err("LLM 返回了不属于当前 Page 上下文的节点来源界面图 ID".to_string());
|
||||
let allowed = allowed_ids.iter().cloned().collect::<HashSet<_>>();
|
||||
let mut seen = HashSet::new();
|
||||
for tree in trees {
|
||||
if !allowed.contains(&tree.src_ui_design_image_id) {
|
||||
return Err("LLM 返回了不属于当前 Page 上下文的界面图 ID".to_string());
|
||||
}
|
||||
validate_source_image_ids(&node.children, allowed_ids)?;
|
||||
if !seen.insert(tree.src_ui_design_image_id.clone()) {
|
||||
return Err("LLM 为同一界面图返回了重复 UI 树".to_string());
|
||||
}
|
||||
validate_confidence(&tree.children)?;
|
||||
}
|
||||
if seen.len() != allowed.len() {
|
||||
return Err("LLM 未为当前 Page 上下文的每张界面图返回 UI 树".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -320,6 +316,43 @@ fn validate_source_image_ids(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn test_image() -> UIDesignImage {
|
||||
UIDesignImage {
|
||||
metadata: crate::ui_editor::resource::ui_design_image::UIDesignImageMetadata {
|
||||
name: "测试图".to_string(),
|
||||
description: String::new(),
|
||||
role: None,
|
||||
slave_to: None,
|
||||
},
|
||||
path: "test.png".to_string(),
|
||||
pixel_size: Vector2::new(1000.0, 500.0),
|
||||
pixels_per_unit: typed_floats::tf32::StrictlyPositiveFinite::new(2.0)
|
||||
.expect("valid pixels per unit"),
|
||||
}
|
||||
}
|
||||
|
||||
fn test_node() -> RecognitionNode {
|
||||
RecognitionNode {
|
||||
global_pos_x_px: 100,
|
||||
global_pos_y_px: 50,
|
||||
width_px: 200,
|
||||
height_px: 100,
|
||||
local_anchor: Anchor::Preset(PresetAnchor {
|
||||
horizontal: HorizonalAnchor::Left,
|
||||
vertical: VerticalAnchor::Top,
|
||||
}),
|
||||
name: "节点".to_string(),
|
||||
description: String::new(),
|
||||
children: Vec::new(),
|
||||
confidence: Confidence::Confident,
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_rect_close(actual: UIRect, expected: UIRect) {
|
||||
assert!((actual.min - expected.min).norm() <= 1.0e-4);
|
||||
assert!((actual.size - expected.size).norm() <= 1.0e-4);
|
||||
}
|
||||
|
||||
fn assert_no_one_of(value: &serde_json::Value) {
|
||||
match value {
|
||||
serde_json::Value::Object(object) => {
|
||||
@@ -372,7 +405,6 @@ mod tests {
|
||||
global_pos_y_px: 0,
|
||||
width_px: 1,
|
||||
height_px: 1,
|
||||
src_ui_design_image_id: UIDesignImageId::new("page").expect("image id"),
|
||||
local_anchor: Anchor::Preset(PresetAnchor {
|
||||
horizontal: HorizonalAnchor::Left,
|
||||
vertical: VerticalAnchor::Top,
|
||||
@@ -384,6 +416,43 @@ mod tests {
|
||||
};
|
||||
assert!(validate_confidence(&[node]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn conversion_stays_in_the_tree_image_coordinate_system() {
|
||||
let image_id = UIDesignImageId::new("slave").expect("valid image id");
|
||||
let image = test_image();
|
||||
let root_rect = UIRect::new(Point2::origin(), image_layout_size(&image).unwrap());
|
||||
let converted =
|
||||
convert_node(&test_node(), &image_id, &image, root_rect).expect("node conversion");
|
||||
|
||||
assert_rect_close(
|
||||
converted.transform.resolve(&root_rect),
|
||||
UIRect::new(Point2::new(50.0, 25.0), Vector2::new(100.0, 50.0)),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_validation_requires_exactly_one_tree_per_context_image() {
|
||||
let page = UIDesignImageId::new("page").expect("valid image id");
|
||||
let slave = UIDesignImageId::new("slave").expect("valid image id");
|
||||
let tree = |id: UIDesignImageId| RecognitionTree {
|
||||
src_ui_design_image_id: id,
|
||||
children: Vec::new(),
|
||||
};
|
||||
|
||||
assert!(validate_tree_image_ids(
|
||||
&[tree(page.clone()), tree(slave.clone())],
|
||||
&[page.clone(), slave.clone()],
|
||||
)
|
||||
.is_ok());
|
||||
assert!(
|
||||
validate_tree_image_ids(&[tree(page.clone())], &[page.clone(), slave.clone()]).is_err()
|
||||
);
|
||||
assert!(
|
||||
validate_tree_image_ids(&[tree(page.clone()), tree(page.clone())], &[page, slave],)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn page_image_ids(state: &State) -> Vec<UIDesignImageId> {
|
||||
@@ -439,14 +508,10 @@ pub(crate) async fn recognize_ui_impl(
|
||||
let root = Path::new(project_path.trim());
|
||||
let mut ui_trees = Vec::with_capacity(page_ids.len());
|
||||
for page_id in page_ids {
|
||||
let page = state
|
||||
.ui_design_images
|
||||
.get(&page_id)
|
||||
.ok_or_else(|| "缺少 Page 界面图资源".to_string())?;
|
||||
let mut context_ids = vec![page_id.clone()];
|
||||
context_ids.extend(slave_image_ids(&state, &page_id));
|
||||
// The model may cite only this Page or its direct slave images as evidence.
|
||||
let allowed_source_ids = context_ids.clone();
|
||||
// 当前只支持 Page 的直接 slave;嵌套 slave 关系的识别批次待后续设计。
|
||||
// TODO: 支持 slave_to 链的递归分组和独立识别。
|
||||
let mut parts = Vec::with_capacity(context_ids.len() * 2);
|
||||
for (index, context_id) in context_ids.iter().enumerate() {
|
||||
let image = state
|
||||
@@ -484,7 +549,7 @@ pub(crate) async fn recognize_ui_impl(
|
||||
}
|
||||
let tool = LlmFunctionTool::new(
|
||||
"recognize_ui_structure",
|
||||
"识别当前 Page 及其 slave 参考图中的 UI 节点,并只返回这一棵树的节点列表",
|
||||
"识别当前 Page 及其直接 slave 参考图,并为每张输入图片各返回一棵 UI 树",
|
||||
schema.clone(),
|
||||
)
|
||||
.with_strict(true);
|
||||
@@ -533,57 +598,52 @@ pub(crate) async fn recognize_ui_impl(
|
||||
);
|
||||
format!("Page {} 的识别工具参数无效:{error}", page_id.as_str())
|
||||
})?;
|
||||
validate_confidence(&parsed.children).map_err(|error| {
|
||||
validate_tree_image_ids(&parsed.trees, &context_ids).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=validate_confidence page={} error={error}",
|
||||
"ui_recognition.error stage=validate_trees page={} error={error}",
|
||||
page_id.as_str()
|
||||
);
|
||||
error
|
||||
})?;
|
||||
validate_source_image_ids(&parsed.children, &allowed_source_ids).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=validate_source_ids page={} error={error}",
|
||||
page_id.as_str()
|
||||
);
|
||||
error
|
||||
})?;
|
||||
let size = image_layout_size(page)?;
|
||||
let root_rect = UIRect::new(Point2::origin(), size);
|
||||
let children = parsed
|
||||
.children
|
||||
.iter()
|
||||
.map(|node| {
|
||||
let source_image = state
|
||||
.ui_design_images
|
||||
.get(&node.src_ui_design_image_id)
|
||||
.ok_or_else(|| {
|
||||
format!(
|
||||
"Page {} 的节点引用了缺失界面图 {}",
|
||||
page_id.as_str(),
|
||||
node.src_ui_design_image_id.as_str()
|
||||
)
|
||||
})?;
|
||||
convert_node(node, page, source_image, &state.ui_design_images, root_rect)
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let root = LayoutNode {
|
||||
id: random_node_id()?,
|
||||
transform: Transform::stretch(),
|
||||
metadata: NodeMetadata {
|
||||
name: "页面根节点".to_string(),
|
||||
description: String::new(),
|
||||
status: NodeStatus::Passed,
|
||||
source: NodeSource::System,
|
||||
},
|
||||
// TODO: root components can host page-level background content once component preview is supported.
|
||||
components: Vec::new(),
|
||||
children,
|
||||
};
|
||||
// Tree identity and root identity are assigned by Rust, never chosen by the model.
|
||||
ui_trees.push(UITree {
|
||||
src_ui_design: page_id,
|
||||
root,
|
||||
});
|
||||
let mut parsed_trees = parsed
|
||||
.trees
|
||||
.into_iter()
|
||||
.map(|tree| (tree.src_ui_design_image_id.clone(), tree))
|
||||
.collect::<HashMap<_, _>>();
|
||||
for image_id in context_ids {
|
||||
let image = state
|
||||
.ui_design_images
|
||||
.get(&image_id)
|
||||
.ok_or_else(|| "缺少识别上下文界面图资源".to_string())?;
|
||||
let tree = parsed_trees
|
||||
.remove(&image_id)
|
||||
.ok_or_else(|| format!("缺少界面图 {} 的识别树", image_id.as_str()))?;
|
||||
let size = image_layout_size(image)?;
|
||||
let root_rect = UIRect::new(Point2::origin(), size);
|
||||
let children = tree
|
||||
.children
|
||||
.iter()
|
||||
.map(|node| convert_node(node, &image_id, image, root_rect))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let root = LayoutNode {
|
||||
id: random_node_id()?,
|
||||
transform: Transform::stretch(),
|
||||
metadata: NodeMetadata {
|
||||
name: "页面根节点".to_string(),
|
||||
description: String::new(),
|
||||
status: NodeStatus::Passed,
|
||||
source: NodeSource::System,
|
||||
},
|
||||
// TODO: root components can host page-level background content once component preview is supported.
|
||||
components: Vec::new(),
|
||||
children,
|
||||
};
|
||||
// Tree identity and root identity are assigned by Rust, never chosen by the model.
|
||||
ui_trees.push(UITree {
|
||||
src_ui_design: image_id,
|
||||
root,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(RecognitionDTO { ui_trees })
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import type { MergeDTO } from './types/MergeDTO';
|
||||
import type { State } from './types/State';
|
||||
|
||||
/** 合并阶段返回一棵新树,成功后替换当前全部逐图识别树。 */
|
||||
export function applyMergeResult(state: State, result: MergeDTO): State {
|
||||
return {
|
||||
...structuredClone(state),
|
||||
ui_trees: [structuredClone(result.ui_tree)],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { UITree } from "./UITree";
|
||||
|
||||
export type MergeDTO = { ui_tree: UITree, };
|
||||
Reference in New Issue
Block a user