识别树根节点改为模型根节点

识别协议改用 trees[].root 并保留根节点非几何字段
同步根节点解析校验、提示词与 UI 树转换
This commit is contained in:
2026-09-12 18:05:07 +08:00
parent 47d09d51ef
commit 5c40abce47
@@ -12,7 +12,7 @@ use crate::ui_editor::layout::node::{Node as LayoutNode, NodeMetadata, NodeSourc
use crate::ui_editor::layout::transform::Transform;
use crate::ui_editor::resource::ui_design_image::UIDesignImage;
use crate::ui_editor::state::{State, UITree};
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
use crate::ui_editor::utils::{random_node_id, NodeId, UIDesignImageId};
use nalgebra::{Point2, Vector2};
use platform_llm::{
LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice,
@@ -37,6 +37,7 @@ const SYSTEM_PROMPT: &str = r#"
* 只识别 UI元素. 要区分动态内容, 不要白费力气识别应该由程序生成/绘制的内容.(此类内容应该用一个整体节点+自然语言描述) 除此之外必须完整包含所有元素,结构.
* 无法确定类型、层级、关系时,在 UnSure 中写明原因。
* 返回的 trees 必须与输入图片一一对应,每张输入图片只能有一棵树,不能合并多张图片的树。 每棵树的 src_ui_design_image_id 必须等于对应输入图片标注的 id。
* 每棵树root的 global_pos_x_px、global_pos_y_px、width_px、height_px、local_anchor 仅为占位并会被忽略,给合法值即可.
* 每棵树必须使用自己的输入图片原始像素坐标系(0,0 as left top)输出
global_pos_x_px、global_pos_y_px、width_px、height_px
* 为了响应式布局, 我们提供了类似godot的Anchor参数, 可以使用语义化的预设或者可custom的直接操作min max, 请准确地根据父子布局的关系使用
@@ -122,7 +123,7 @@ struct RecognitionNode {
#[schemars(deny_unknown_fields)]
struct RecognitionTree {
src_ui_design_image_id: UIDesignImageId,
children: Vec<RecognitionNode>,
root: RecognitionNode,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
@@ -150,10 +151,14 @@ fn validate_recognition_response_shape(value: &serde_json::Value) -> Result<(),
return Err(format!("识别结果最多包含 {MAX_REFERENCES} 棵界面树"));
}
for tree in trees {
let children = tree
let root = tree
.get("root")
.and_then(serde_json::Value::as_object)
.ok_or_else(|| "识别树缺少 root 节点".to_string())?;
let children = root
.get("children")
.and_then(serde_json::Value::as_array)
.ok_or_else(|| "识别树缺少 children 数组".to_string())?;
.ok_or_else(|| "识别树根节点缺少 children 数组".to_string())?;
let mut stack = children
.iter()
.map(|node| (node, 1usize))
@@ -365,7 +370,7 @@ fn validate_tree_image_ids(
if !seen.insert(tree.src_ui_design_image_id.clone()) {
return Err("LLM 为同一界面图返回了重复 UI 树".to_string());
}
validate_confidence(&tree.children)?;
validate_confidence(std::slice::from_ref(&tree.root))?;
}
if seen.len() != allowed.len() {
return Err("LLM 未为当前识别上下文的每张界面图返回 UI 树".to_string());
@@ -445,8 +450,8 @@ mod tests {
.collect::<Vec<_>>();
let response = serde_json::json!({
"trees": [
{"children": leaves.clone()},
{"children": leaves}
{"root": {"children": leaves.clone()}},
{"root": {"children": leaves}}
]
});
validate_recognition_response_shape(&response)
@@ -459,7 +464,7 @@ mod tests {
.map(|_| serde_json::json!({"children": []}))
.collect::<Vec<_>>();
assert!(validate_recognition_response_shape(&serde_json::json!({
"trees": [{"children": oversized}]
"trees": [{"root": {"children": oversized}}]
}))
.is_err());
@@ -468,7 +473,7 @@ mod tests {
nested = serde_json::json!({"children": [nested]});
}
assert!(validate_recognition_response_shape(&serde_json::json!({
"trees": [{"children": [nested]}]
"trees": [{"root": {"children": [nested]}}]
}))
.is_err());
}
@@ -587,7 +592,7 @@ mod tests {
let slave = UIDesignImageId::new("slave").expect("valid image id");
let tree = |id: UIDesignImageId| RecognitionTree {
src_ui_design_image_id: id,
children: Vec::new(),
root: test_node(),
};
assert!(validate_tree_image_ids(
@@ -820,24 +825,32 @@ pub(crate) async fn recognize_ui_impl_with_provider(
.ok_or_else(|| format!("缺少界面图 {} 的识别树", image_id.as_str()))?;
let size = image_layout_size(image)?;
let root_rect = UIRect::new(Point2::origin(), size);
let children = tree
// The recognition root is a real UI node. Its pixel geometry and
// anchor are intentionally ignored; the page root always fills
// the design image while the other recognition fields take effect.
let recognition_root = tree.root;
let children = recognition_root
.children
.iter()
.map(|node| convert_node(node, &image_id, image, root_rect))
.collect::<Result<Vec<_>, _>>()?;
let root_layout_status = match recognition_root.confidence {
Confidence::Confident => StageStatus::NoProblem,
Confidence::UnSure(reason) => StageStatus::NeedReview(reason),
};
let root = LayoutNode {
id: random_node_id(),
layout: ControlLayout::with_transform(Transform::stretch()),
metadata: NodeMetadata {
name: "页面根节点".to_string(),
description: String::new(),
layout_status: StageStatus::NoProblem,
name: recognition_root.name,
description: recognition_root.description,
layout_status: root_layout_status,
component_status: StageStatus::NoProblem,
allow_llm_edit_layout: true,
allow_llm_edit_component: true,
source: NodeSource::System,
source: NodeSource::Llm,
},
component: None,
component: recognition_root.component.into_option(),
children_display_mode: ChildrenDisplayMode::Stack,
children,
};