识别树根节点改为模型根节点
识别协议改用 trees[].root 并保留根节点非几何字段 同步根节点解析校验、提示词与 UI 树转换
This commit is contained in:
@@ -12,7 +12,7 @@ use crate::ui_editor::layout::node::{Node as LayoutNode, NodeMetadata, NodeSourc
|
||||
use crate::ui_editor::layout::transform::Transform;
|
||||
use crate::ui_editor::resource::ui_design_image::UIDesignImage;
|
||||
use crate::ui_editor::state::{State, UITree};
|
||||
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
|
||||
use crate::ui_editor::utils::{random_node_id, NodeId, UIDesignImageId};
|
||||
use nalgebra::{Point2, Vector2};
|
||||
use platform_llm::{
|
||||
LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice,
|
||||
@@ -37,6 +37,7 @@ const SYSTEM_PROMPT: &str = r#"
|
||||
* 只识别 UI元素. 要区分动态内容, 不要白费力气识别应该由程序生成/绘制的内容.(此类内容应该用一个整体节点+自然语言描述) 除此之外必须完整包含所有元素,结构.
|
||||
* 无法确定类型、层级、关系时,在 UnSure 中写明原因。
|
||||
* 返回的 trees 必须与输入图片一一对应,每张输入图片只能有一棵树,不能合并多张图片的树。 每棵树的 src_ui_design_image_id 必须等于对应输入图片标注的 id。
|
||||
* 每棵树root的 global_pos_x_px、global_pos_y_px、width_px、height_px、local_anchor 仅为占位并会被忽略,给合法值即可.
|
||||
* 每棵树必须使用自己的输入图片原始像素坐标系(0,0 as left top)输出
|
||||
global_pos_x_px、global_pos_y_px、width_px、height_px;
|
||||
* 为了响应式布局, 我们提供了类似godot的Anchor参数, 可以使用语义化的预设或者可custom的直接操作min max, 请准确地根据父子布局的关系使用
|
||||
@@ -122,7 +123,7 @@ struct RecognitionNode {
|
||||
#[schemars(deny_unknown_fields)]
|
||||
struct RecognitionTree {
|
||||
src_ui_design_image_id: UIDesignImageId,
|
||||
children: Vec<RecognitionNode>,
|
||||
root: RecognitionNode,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
@@ -150,10 +151,14 @@ fn validate_recognition_response_shape(value: &serde_json::Value) -> Result<(),
|
||||
return Err(format!("识别结果最多包含 {MAX_REFERENCES} 棵界面树"));
|
||||
}
|
||||
for tree in trees {
|
||||
let children = tree
|
||||
let root = tree
|
||||
.get("root")
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.ok_or_else(|| "识别树缺少 root 节点".to_string())?;
|
||||
let children = root
|
||||
.get("children")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.ok_or_else(|| "识别树缺少 children 数组".to_string())?;
|
||||
.ok_or_else(|| "识别树根节点缺少 children 数组".to_string())?;
|
||||
let mut stack = children
|
||||
.iter()
|
||||
.map(|node| (node, 1usize))
|
||||
@@ -365,7 +370,7 @@ fn validate_tree_image_ids(
|
||||
if !seen.insert(tree.src_ui_design_image_id.clone()) {
|
||||
return Err("LLM 为同一界面图返回了重复 UI 树".to_string());
|
||||
}
|
||||
validate_confidence(&tree.children)?;
|
||||
validate_confidence(std::slice::from_ref(&tree.root))?;
|
||||
}
|
||||
if seen.len() != allowed.len() {
|
||||
return Err("LLM 未为当前识别上下文的每张界面图返回 UI 树".to_string());
|
||||
@@ -445,8 +450,8 @@ mod tests {
|
||||
.collect::<Vec<_>>();
|
||||
let response = serde_json::json!({
|
||||
"trees": [
|
||||
{"children": leaves.clone()},
|
||||
{"children": leaves}
|
||||
{"root": {"children": leaves.clone()}},
|
||||
{"root": {"children": leaves}}
|
||||
]
|
||||
});
|
||||
validate_recognition_response_shape(&response)
|
||||
@@ -459,7 +464,7 @@ mod tests {
|
||||
.map(|_| serde_json::json!({"children": []}))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(validate_recognition_response_shape(&serde_json::json!({
|
||||
"trees": [{"children": oversized}]
|
||||
"trees": [{"root": {"children": oversized}}]
|
||||
}))
|
||||
.is_err());
|
||||
|
||||
@@ -468,7 +473,7 @@ mod tests {
|
||||
nested = serde_json::json!({"children": [nested]});
|
||||
}
|
||||
assert!(validate_recognition_response_shape(&serde_json::json!({
|
||||
"trees": [{"children": [nested]}]
|
||||
"trees": [{"root": {"children": [nested]}}]
|
||||
}))
|
||||
.is_err());
|
||||
}
|
||||
@@ -587,7 +592,7 @@ mod tests {
|
||||
let slave = UIDesignImageId::new("slave").expect("valid image id");
|
||||
let tree = |id: UIDesignImageId| RecognitionTree {
|
||||
src_ui_design_image_id: id,
|
||||
children: Vec::new(),
|
||||
root: test_node(),
|
||||
};
|
||||
|
||||
assert!(validate_tree_image_ids(
|
||||
@@ -820,24 +825,32 @@ pub(crate) async fn recognize_ui_impl_with_provider(
|
||||
.ok_or_else(|| format!("缺少界面图 {} 的识别树", image_id.as_str()))?;
|
||||
let size = image_layout_size(image)?;
|
||||
let root_rect = UIRect::new(Point2::origin(), size);
|
||||
let children = tree
|
||||
// The recognition root is a real UI node. Its pixel geometry and
|
||||
// anchor are intentionally ignored; the page root always fills
|
||||
// the design image while the other recognition fields take effect.
|
||||
let recognition_root = tree.root;
|
||||
let children = recognition_root
|
||||
.children
|
||||
.iter()
|
||||
.map(|node| convert_node(node, &image_id, image, root_rect))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let root_layout_status = match recognition_root.confidence {
|
||||
Confidence::Confident => StageStatus::NoProblem,
|
||||
Confidence::UnSure(reason) => StageStatus::NeedReview(reason),
|
||||
};
|
||||
let root = LayoutNode {
|
||||
id: random_node_id(),
|
||||
layout: ControlLayout::with_transform(Transform::stretch()),
|
||||
metadata: NodeMetadata {
|
||||
name: "页面根节点".to_string(),
|
||||
description: String::new(),
|
||||
layout_status: StageStatus::NoProblem,
|
||||
name: recognition_root.name,
|
||||
description: recognition_root.description,
|
||||
layout_status: root_layout_status,
|
||||
component_status: StageStatus::NoProblem,
|
||||
allow_llm_edit_layout: true,
|
||||
allow_llm_edit_component: true,
|
||||
source: NodeSource::System,
|
||||
source: NodeSource::Llm,
|
||||
},
|
||||
component: None,
|
||||
component: recognition_root.component.into_option(),
|
||||
children_display_mode: ChildrenDisplayMode::Stack,
|
||||
children,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user