diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs index d34a5d47b..98fd8e40e 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs @@ -12,7 +12,7 @@ use crate::ui_editor::layout::node::{Node as LayoutNode, NodeMetadata, NodeSourc use crate::ui_editor::layout::transform::Transform; use crate::ui_editor::resource::ui_design_image::UIDesignImage; use crate::ui_editor::state::{State, UITree}; -use crate::ui_editor::utils::{NodeId, UIDesignImageId}; +use crate::ui_editor::utils::{random_node_id, NodeId, UIDesignImageId}; use nalgebra::{Point2, Vector2}; use platform_llm::{ LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice, @@ -37,6 +37,7 @@ const SYSTEM_PROMPT: &str = r#" * 只识别 UI元素. 要区分动态内容, 不要白费力气识别应该由程序生成/绘制的内容.(此类内容应该用一个整体节点+自然语言描述) 除此之外必须完整包含所有元素,结构. * 无法确定类型、层级、关系时,在 UnSure 中写明原因。 * 返回的 trees 必须与输入图片一一对应,每张输入图片只能有一棵树,不能合并多张图片的树。 每棵树的 src_ui_design_image_id 必须等于对应输入图片标注的 id。 +* 每棵树root的 global_pos_x_px、global_pos_y_px、width_px、height_px、local_anchor 仅为占位并会被忽略,给合法值即可. * 每棵树必须使用自己的输入图片原始像素坐标系(0,0 as left top)输出 global_pos_x_px、global_pos_y_px、width_px、height_px; * 为了响应式布局, 我们提供了类似godot的Anchor参数, 可以使用语义化的预设或者可custom的直接操作min max, 请准确地根据父子布局的关系使用 @@ -122,7 +123,7 @@ struct RecognitionNode { #[schemars(deny_unknown_fields)] struct RecognitionTree { src_ui_design_image_id: UIDesignImageId, - children: Vec, + root: RecognitionNode, } #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] @@ -150,10 +151,14 @@ fn validate_recognition_response_shape(value: &serde_json::Value) -> Result<(), return Err(format!("识别结果最多包含 {MAX_REFERENCES} 棵界面树")); } for tree in trees { - let children = tree + let root = tree + .get("root") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| "识别树缺少 root 节点".to_string())?; + let children = root .get("children") .and_then(serde_json::Value::as_array) - .ok_or_else(|| "识别树缺少 children 数组".to_string())?; + .ok_or_else(|| "识别树根节点缺少 children 数组".to_string())?; let mut stack = children .iter() .map(|node| (node, 1usize)) @@ -365,7 +370,7 @@ fn validate_tree_image_ids( if !seen.insert(tree.src_ui_design_image_id.clone()) { return Err("LLM 为同一界面图返回了重复 UI 树".to_string()); } - validate_confidence(&tree.children)?; + validate_confidence(std::slice::from_ref(&tree.root))?; } if seen.len() != allowed.len() { return Err("LLM 未为当前识别上下文的每张界面图返回 UI 树".to_string()); @@ -445,8 +450,8 @@ mod tests { .collect::>(); let response = serde_json::json!({ "trees": [ - {"children": leaves.clone()}, - {"children": leaves} + {"root": {"children": leaves.clone()}}, + {"root": {"children": leaves}} ] }); validate_recognition_response_shape(&response) @@ -459,7 +464,7 @@ mod tests { .map(|_| serde_json::json!({"children": []})) .collect::>(); assert!(validate_recognition_response_shape(&serde_json::json!({ - "trees": [{"children": oversized}] + "trees": [{"root": {"children": oversized}}] })) .is_err()); @@ -468,7 +473,7 @@ mod tests { nested = serde_json::json!({"children": [nested]}); } assert!(validate_recognition_response_shape(&serde_json::json!({ - "trees": [{"children": [nested]}] + "trees": [{"root": {"children": [nested]}}] })) .is_err()); } @@ -587,7 +592,7 @@ mod tests { let slave = UIDesignImageId::new("slave").expect("valid image id"); let tree = |id: UIDesignImageId| RecognitionTree { src_ui_design_image_id: id, - children: Vec::new(), + root: test_node(), }; assert!(validate_tree_image_ids( @@ -820,24 +825,32 @@ pub(crate) async fn recognize_ui_impl_with_provider( .ok_or_else(|| format!("缺少界面图 {} 的识别树", image_id.as_str()))?; let size = image_layout_size(image)?; let root_rect = UIRect::new(Point2::origin(), size); - let children = tree + // The recognition root is a real UI node. Its pixel geometry and + // anchor are intentionally ignored; the page root always fills + // the design image while the other recognition fields take effect. + let recognition_root = tree.root; + let children = recognition_root .children .iter() .map(|node| convert_node(node, &image_id, image, root_rect)) .collect::, _>>()?; + let root_layout_status = match recognition_root.confidence { + Confidence::Confident => StageStatus::NoProblem, + Confidence::UnSure(reason) => StageStatus::NeedReview(reason), + }; let root = LayoutNode { id: random_node_id(), layout: ControlLayout::with_transform(Transform::stretch()), metadata: NodeMetadata { - name: "页面根节点".to_string(), - description: String::new(), - layout_status: StageStatus::NoProblem, + name: recognition_root.name, + description: recognition_root.description, + layout_status: root_layout_status, component_status: StageStatus::NoProblem, allow_llm_edit_layout: true, allow_llm_edit_component: true, - source: NodeSource::System, + source: NodeSource::Llm, }, - component: None, + component: recognition_root.component.into_option(), children_display_mode: ChildrenDisplayMode::Stack, children, };