From 580f0198c83582b4d023874b8112c81bb3c0c262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Fri, 14 Aug 2026 12:15:01 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84=20UI=20=E8=AE=BE=E8=AE=A1?= =?UTF-8?q?=E8=AF=AD=E4=B9=89=E5=BB=BA=E8=AE=AE=E5=92=8C=E7=BB=93=E6=9E=84?= =?UTF-8?q?=EF=BC=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 替换 `UIDesignSuggestion` 为树型结构 `UIDesignSuggestionTreeNode`，统一描述界面图语义。 - 更新 UI 树 `UITree` 数据结构，新增 `root` 字段取代 `children`。 - 调整 `NodeSource` 枚举，添加新值 `System`。 - 修改语义建议逻辑：移除 `oneOf` 节点限制，强制所有字段不可省略。 - 增强验证和测试覆盖率，确保数据模型和树结构一致性。 --- .../src-tauri/src/main.rs | 2 +- .../src-tauri/src/ui_editor/commands/mod.rs | 2 +- .../src/ui_editor/commands/recognition.rs | 66 ++++- .../commands/ui_design_suggestion.rs | 225 ++++++++++-------- .../src-tauri/src/ui_editor/commands/utils.rs | 3 + .../src-tauri/src/ui_editor/layout/node.rs | 1 + .../src-tauri/src/ui_editor/state/mod.rs | 2 +- .../features/ui-editor/types/NodeSource.ts | 2 +- .../ui-editor/types/UIDesignSuggestion.ts | 5 - .../src/features/ui-editor/types/UITree.ts | 2 +- 10 files changed, 199 insertions(+), 111 deletions(-) delete mode 100644 apps/ai-game-creator-shell/src/features/ui-editor/types/UIDesignSuggestion.ts diff --git a/apps/ai-game-creator-shell/src-tauri/src/main.rs b/apps/ai-game-creator-shell/src-tauri/src/main.rs index 28ae96d72..e952ab35e 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/main.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/main.rs @@ -114,7 +114,7 @@ use windows::*; async fn suggest_ui_design_semantic( project_path: String, state: ui_editor::state::State, -) -> Result, String> { +) -> Result, String> { ui_editor::commands::suggest_ui_design_semantic_impl(project_path, state).await } diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs index 94c273f9a..d9132d2f8 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/mod.rs @@ -5,4 +5,4 @@ pub mod utils; pub(crate) use recognition::recognize_ui_impl; pub use recognition::RecognitionDTO; pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl; -pub use ui_design_suggestion::UIDesignSuggestion; +pub use ui_design_suggestion::UIDesignSuggestionTreeNode; diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs index 9d1cd43f8..98bcba54f 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/recognition.rs @@ -19,12 +19,28 @@ use ts_rs::TS; const MAX_REFERENCES: usize = 4; +// TODO +// * 头像、图标或物品图只识别为一个图片或头像组件。 +// * 文本和动态数值即使没有独立素材，也必须保留为组件。 const SYSTEM_PROMPT: &str = r#" -你是游戏 UI 结构识别器。当前请求只处理一个 Page 参考图，以及直接归属于该 Page 的 slave 参考图。 -第一张标记为 PAGE 的图片是唯一输出树的画布和归属页面；其它标记为 SLAVE 的图片只提供视觉上下文。 -本次调用只能调用 recognize_ui_structure 工具返回一个节点列表，不得返回树数组、树 ID 或其它页面的节点。 -每个节点必须使用 src_ui_design_image_id 指定坐标来源，并在该图片的原始像素坐标系中输出 +角色： +你是游戏 UI 多图结构识别器。 + +任务： +同时分析同一 UI 系统的全部参考图，建立UI树 + +用户会给你一些UI截图(它们从属于同一个UI系统)和对应的元数据, 请用给定的工具描述UI结构 +你是游戏 UI 结构识别器。 + +识别规则： +* 只识别 UI，不识别场景人物、地形、建筑、光影和背景装饰。 +* 不创建没有独立功能的微小装饰Node。 +* 无法确定类型、层级、关系时，在 UnSure 中写明原因。 +* source=user、reviewStatus=confirmed 或 semanticSource=user 的组件不得被 AI 覆盖。 +* 每个Node必须使用 src_ui_design_image_id 指定坐标来源(所属UI design图)，并在该图片的原始像素坐标系中输出 global_pos_x_px、global_pos_y_px、width_px、height_px；坐标相对于来源图片左上角 +* 多张图都是在描述同一个UI系统, 必须合并共用公共的框架/层次, 禁止简单每个图一个树, 坐标等有矛盾的地方父级优先 +* 面向用户的字段如名称描述等请用中文 "#; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] @@ -82,8 +98,6 @@ struct RecognitionNode { global_pos_y_px: u32, width_px: u32, height_px: u32, - /// 视觉证据来源；只允许当前 PAGE 或其直接 slave，不能用来改变树归属或坐标系。 - #[schemars(with = "String")] src_ui_design_image_id: UIDesignImageId, local_anchor: Anchor, name: String, @@ -306,6 +320,29 @@ fn validate_source_image_ids( mod tests { use super::*; + fn assert_no_one_of(value: &serde_json::Value) { + match value { + serde_json::Value::Object(object) => { + assert!(!object.contains_key("oneOf")); + for child in object.values() { + assert_no_one_of(child); + } + } + serde_json::Value::Array(values) => { + for child in values { + assert_no_one_of(child); + } + } + _ => {} + } + } + + #[test] + fn strict_schema_rewrites_union_variants_for_provider() { + let schema = recognition_json_schema().expect("recognition schema"); + assert_no_one_of(&schema); + } + #[test] fn preset_anchor_ranges_use_top_left_image_coordinates() { let (min, max) = anchor_ranges(&Anchor::Preset(PresetAnchor { @@ -529,10 +566,23 @@ pub(crate) async fn recognize_ui_impl( convert_node(node, page, source_image, &state.ui_design_images, root_rect) }) .collect::, _>>()?; - // Tree identity is assigned by Rust from the Page image, never chosen by the model. + let root = LayoutNode { + id: random_node_id()?, + transform: Transform::stretch(), + metadata: NodeMetadata { + name: "页面根节点".to_string(), + description: String::new(), + status: NodeStatus::Passed, + source: NodeSource::System, + }, + // TODO: root components can host page-level background content once component preview is supported. + components: Vec::new(), + children, + }; + // Tree identity and root identity are assigned by Rust, never chosen by the model. ui_trees.push(UITree { src_ui_design: page_id, - children, + root, }); } Ok(RecognitionDTO { ui_trees }) diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/ui_design_suggestion.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/ui_design_suggestion.rs index 9f7515c8e..a72be8e28 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/ui_design_suggestion.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/ui_design_suggestion.rs @@ -17,8 +17,8 @@ const SYSTEM_PROMPT: &str = r#" 请识别这些 UI 参考图的界面语义，并调用 suggest_ui_design_semantics 工具返回结果。 不要在工具调用外输出 JSON、Markdown、代码围栏、注释、额外字段或解释文字。 -字段含义和 null 规则： -- ui_design_image_id：必填，必须逐字匹配当前输入参考图的 id。它标识“这条建议属于哪张图”，不是新 ID，不能为 null。 +字段含义： +- id：必填，必须逐字匹配当前输入参考图的 id。它标识“这条建议属于哪张图”，不是新 ID，不能为 null。 - name：要写入该图片 metadata 的简短、可读名称； - description：要写入该图片 metadata 的简短语义描述，例如“带底部导航的主游戏页面”。 - role：要写入该图片 metadata 的界面角色。 @@ -28,90 +28,36 @@ const SYSTEM_PROMPT: &str = r#" State 表示同一界面的状态变体， Scrolled 表示滚动或分页后的内容， Detail 表示局部详情或补充证据。 -- slave_to：要写入该图片 metadata 的归属页面 id。只有当当前图片明显是另一张输入图的局部/子界面时才填写那个宿主图的 id； - Page 不得设置宿主。 +- children：该节点直接包含的子界面图。根节点必须是 Page，子节点不能是 Page；没有子节点时返回空数组。 -所有字段都必须出现；nullable 字段使用 JSON null 表示“不要修改该字段”，不要省略字段。 -每张参考图最多返回一条建议，且不得重复 ui_design_image_id。 -以下UI设计图中的metadata部分信息已确定, 请补全未确定/为空的信息 +所有字段都必须出现，禁止省略字段、使用 null 或返回空字符串。每张参考图最多出现一次；id 必须逐字匹配输入图片 id。 +以下 UI 设计图中的 metadata 部分信息已确定，请根据已有信息补全语义. "#; #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS, JsonSchema)] #[schemars(deny_unknown_fields)] #[serde(deny_unknown_fields)] #[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] -pub struct UIDesignSuggestion { - pub ui_design_image_id: UIDesignImageId, - #[schemars(required)] - pub name: Option, - #[schemars(required)] - pub description: Option, - #[schemars(required)] - pub role: Option, - #[schemars(required)] - pub slave_to: Option, +pub struct UIDesignSuggestionTreeNode { + pub id: UIDesignImageId, + pub name: String, + pub description: String, + pub role: UIDesignImageRole, + pub children: Vec, } #[derive(Clone, Debug, Deserialize, JsonSchema)] #[schemars(deny_unknown_fields)] #[serde(deny_unknown_fields)] -struct UIDesignSuggestionResponse { - suggestions: Vec, +struct UIDesignSuggestion { + ui_designs: Vec, } fn ui_design_suggestion_json_schema() -> Result { - strict_json_schema::() + strict_json_schema::() } const MAX_REFERENCES: usize = 4; -#[cfg(test)] -mod tests { - use super::*; - - fn assert_no_refs(value: &serde_json::Value) { - match value { - serde_json::Value::Object(object) => { - assert!(!object.contains_key("$ref")); - assert!(!object.contains_key("$defs")); - for child in object.values() { - assert_no_refs(child); - } - } - serde_json::Value::Array(values) => { - for child in values { - assert_no_refs(child); - } - } - _ => {} - } - } - - #[test] - fn strict_schema_is_inline_and_requires_all_properties() { - let schema = ui_design_suggestion_json_schema().expect("schema"); - assert_no_refs(&schema); - assert_eq!(schema["required"], serde_json::json!(["suggestions"])); - let required = schema["properties"]["suggestions"]["items"]["required"] - .as_array() - .expect("item required array") - .iter() - .filter_map(serde_json::Value::as_str) - .collect::>(); - assert_eq!( - required, - [ - "ui_design_image_id", - "name", - "description", - "role", - "slave_to" - ] - .into_iter() - .collect() - ); - } -} - fn image_data_url(path: &Path, bytes: &[u8]) -> Result { let mime = match path.extension().and_then(|value| value.to_str()) { Some("png") => "image/png", @@ -126,34 +72,47 @@ fn image_data_url(path: &Path, bytes: &[u8]) -> Result { } fn validate_suggestions( - suggestions: &[UIDesignSuggestion], + suggestions: &[UIDesignSuggestionTreeNode], image_ids: &HashSet, ) -> Result<(), String> { let mut seen = HashSet::new(); - for suggestion in suggestions { - if !image_ids.contains(&suggestion.ui_design_image_id) { - return Err("LLM 返回了未知界面图 ID".to_string()); - } - if !seen.insert(suggestion.ui_design_image_id.clone()) { - return Err("LLM 返回了重复界面图建议".to_string()); - } - if suggestion.role == Some(UIDesignImageRole::Page) && suggestion.slave_to.is_some() { - return Err("主页面不能设置归属页面".to_string()); - } - if let Some(host) = &suggestion.slave_to { - if host == &suggestion.ui_design_image_id || !image_ids.contains(host) { - return Err("界面图归属页面无效".to_string()); + + fn visit( + nodes: &[UIDesignSuggestionTreeNode], + is_root: bool, + image_ids: &HashSet, + seen: &mut HashSet, + ) -> Result<(), String> { + for node in nodes { + if !image_ids.contains(&node.id) { + return Err("LLM 返回了未知界面图 ID".to_string()); } + if !seen.insert(node.id.clone()) { + return Err("LLM 返回了重复界面图建议".to_string()); + } + if node.name.trim().is_empty() || node.description.trim().is_empty() { + return Err("界面图语义字段不能是空字符串".to_string()); + } + if is_root { + if node.role != UIDesignImageRole::Page { + return Err("界面图树根节点必须是 Page".to_string()); + } + } else if node.role == UIDesignImageRole::Page { + return Err("Page 不能作为其它界面图的子节点".to_string()); + } + visit(&node.children, false, image_ids, seen)?; } + Ok(()) } - Ok(()) + + visit(suggestions, true, image_ids, &mut seen) } #[tauri::command] pub(crate) async fn suggest_ui_design_semantic_impl( project_path: String, state: State, -) -> Result, String> { +) -> Result, String> { if state.ui_design_images.is_empty() { eprintln!("ui_design_suggestion.error stage=validate reason=no_images"); return Err("请先导入界面图".to_string()); @@ -167,7 +126,6 @@ pub(crate) async fn suggest_ui_design_semantic_impl( } let root = Path::new(project_path.trim()); let mut parts = Vec::new(); - let mut ids = HashSet::new(); for (id, image) in &state.ui_design_images { ids.insert(id.clone()); @@ -187,12 +145,7 @@ pub(crate) async fn suggest_ui_design_semantic_impl( format!("读取界面图失败：{error}") })?; parts.push(LlmMessageContentPart::InputText { - text: format!( - "REFERENCE id={} metadata={} pixel_size={:?}", - id.as_str(), - image.metadata, - image.pixel_size - ), + text: format!("REFERENCE id:{} metadata:{}", id.as_str(), image.metadata,), }); parts.push(LlmMessageContentPart::InputImage { image_url: image_data_url(&absolute, &bytes)?, @@ -238,12 +191,12 @@ pub(crate) async fn suggest_ui_design_semantic_impl( eprintln!("ui_design_suggestion.error stage=parse_tool_call reason=missing_tool_call"); "LLM 未返回 suggest_ui_design_semantics 工具调用".to_string() })?; - let suggestions = serde_json::from_str::(&call.arguments) + let suggestions = serde_json::from_str::(&call.arguments) .map_err(|error| { eprintln!("ui_design_suggestion.error stage=parse_arguments error={error}"); format!("UI 语义建议工具参数无效：{error}") })? - .suggestions; + .ui_designs; validate_suggestions(&suggestions, &ids).map_err(|error| { eprintln!("ui_design_suggestion.error stage=validate_result error={error}"); error @@ -251,3 +204,89 @@ pub(crate) async fn suggest_ui_design_semantic_impl( // TODO: future single-page policy may reject multiple Page suggestions. Ok(suggestions) } + +#[cfg(test)] +mod tests { + use super::*; + + fn id(value: &str) -> UIDesignImageId { + UIDesignImageId::new(value).expect("valid image id") + } + + fn node( + value: &str, + role: UIDesignImageRole, + children: Vec, + ) -> UIDesignSuggestionTreeNode { + UIDesignSuggestionTreeNode { + id: id(value), + name: value.to_string(), + description: format!("{value} description"), + role, + children, + } + } + + fn image_ids(values: &[&str]) -> HashSet { + values.iter().map(|value| id(value)).collect() + } + + #[test] + fn validates_page_tree() { + let suggestions = vec![node( + "page", + UIDesignImageRole::Page, + vec![node("section", UIDesignImageRole::Section, Vec::new())], + )]; + assert!(validate_suggestions(&suggestions, &image_ids(&["page", "section"])).is_ok()); + } + + #[test] + fn rejects_duplicate_and_unknown_ids() { + let duplicate = vec![ + node( + "page", + UIDesignImageRole::Page, + vec![node("section", UIDesignImageRole::Section, Vec::new())], + ), + node("page", UIDesignImageRole::Page, Vec::new()), + ]; + assert!(validate_suggestions(&duplicate, &image_ids(&["page", "section"])).is_err()); + + let unknown = vec![node("missing", UIDesignImageRole::Page, Vec::new())]; + assert!(validate_suggestions(&unknown, &image_ids(&["page"])).is_err()); + } + + #[test] + fn rejects_invalid_root_roles_and_nested_pages() { + let non_page_root = vec![node("section", UIDesignImageRole::Section, Vec::new())]; + assert!(validate_suggestions(&non_page_root, &image_ids(&["section"])).is_err()); + + let nested_page = vec![node( + "page", + UIDesignImageRole::Page, + vec![node("nested", UIDesignImageRole::Page, Vec::new())], + )]; + assert!(validate_suggestions(&nested_page, &image_ids(&["page", "nested"])).is_err()); + } + + #[test] + fn strict_schema_requires_tree_fields() { + let schema = ui_design_suggestion_json_schema().expect("schema"); + assert_eq!(schema["required"], serde_json::json!(["ui_designs"])); + let node_schema = &schema["properties"]["ui_designs"]["items"]; + let required = node_schema["required"] + .as_array() + .expect("node required") + .iter() + .filter_map(serde_json::Value::as_str) + .collect::>(); + assert_eq!( + required, + ["id", "name", "description", "role", "children"] + .into_iter() + .collect() + ); + assert_eq!(node_schema["additionalProperties"], false); + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs index 832621028..02a8fcb4b 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/utils.rs @@ -13,6 +13,9 @@ pub(crate) fn strict_json_schema() -> Result { + if let Some(one_of) = object.remove("oneOf") { + object.insert("anyOf".to_string(), one_of); + } if let Some(properties) = object.get("properties").and_then(|value| value.as_object()) { object.insert( "required".to_string(), diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs index afd79baa6..66fadf84e 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/layout/node.rs @@ -17,6 +17,7 @@ pub struct Node { #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] #[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] pub enum NodeSource { + System, Human, Llm, } diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/state/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/state/mod.rs index 15a055a31..7c6c46a7a 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/state/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/state/mod.rs @@ -11,7 +11,7 @@ use ts_rs::TS; #[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))] pub struct UITree { pub src_ui_design: UIDesignImageId, - pub children: Vec, + pub root: Node, } #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)] diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/NodeSource.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/NodeSource.ts index edf3cae3f..acfd1a84f 100644 --- a/apps/ai-game-creator-shell/src/features/ui-editor/types/NodeSource.ts +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/NodeSource.ts @@ -1,3 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type NodeSource = "Human" | "Llm"; +export type NodeSource = "System" | "Human" | "Llm"; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/UIDesignSuggestion.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/UIDesignSuggestion.ts deleted file mode 100644 index b33171126..000000000 --- a/apps/ai-game-creator-shell/src/features/ui-editor/types/UIDesignSuggestion.ts +++ /dev/null @@ -1,5 +0,0 @@ -// This file is generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { UIDesignImageId } from "./UIDesignImageId"; -import type { UIDesignImageRole } from "./UIDesignImageRole"; - -export type UIDesignSuggestion = { ui_design_image_id: UIDesignImageId, name: string | null, description: string | null, role: UIDesignImageRole | null, slave_to: UIDesignImageId | null, }; diff --git a/apps/ai-game-creator-shell/src/features/ui-editor/types/UITree.ts b/apps/ai-game-creator-shell/src/features/ui-editor/types/UITree.ts index 77adcb4d6..fa0cdaf61 100644 --- a/apps/ai-game-creator-shell/src/features/ui-editor/types/UITree.ts +++ b/apps/ai-game-creator-shell/src/features/ui-editor/types/UITree.ts @@ -2,4 +2,4 @@ import type { Node } from "./Node"; import type { UIDesignImageId } from "./UIDesignImageId"; -export type UITree = { src_ui_design: UIDesignImageId, children: Array, }; +export type UITree = { src_ui_design: UIDesignImageId, root: Node, };