重构 UI 设计语义建议和结构:

- 替换 `UIDesignSuggestion` 为树型结构 `UIDesignSuggestionTreeNode`,统一描述界面图语义。
- 更新 UI 树 `UITree` 数据结构,新增 `root` 字段取代 `children`。
- 调整 `NodeSource` 枚举,添加新值 `System`。
- 修改语义建议逻辑:移除 `oneOf` 节点限制,强制所有字段不可省略。
- 增强验证和测试覆盖率,确保数据模型和树结构一致性。
This commit is contained in:
2026-08-14 12:15:01 +08:00
parent 1723aa4322
commit 580f0198c8
10 changed files with 199 additions and 111 deletions
@@ -114,7 +114,7 @@ use windows::*;
async fn suggest_ui_design_semantic(
project_path: String,
state: ui_editor::state::State,
) -> Result<Vec<ui_editor::commands::UIDesignSuggestion>, String> {
) -> Result<Vec<ui_editor::commands::UIDesignSuggestionTreeNode>, String> {
ui_editor::commands::suggest_ui_design_semantic_impl(project_path, state).await
}
@@ -5,4 +5,4 @@ pub mod utils;
pub(crate) use recognition::recognize_ui_impl;
pub use recognition::RecognitionDTO;
pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl;
pub use ui_design_suggestion::UIDesignSuggestion;
pub use ui_design_suggestion::UIDesignSuggestionTreeNode;
@@ -19,12 +19,28 @@ use ts_rs::TS;
const MAX_REFERENCES: usize = 4;
// TODO
// * 头像、图标或物品图只识别为一个图片或头像组件。
// * 文本和动态数值即使没有独立素材,也必须保留为组件。
const SYSTEM_PROMPT: &str = r#"
你是游戏 UI 结构识别器。当前请求只处理一个 Page 参考图,以及直接归属于该 Page 的 slave 参考图。
第一张标记为 PAGE 的图片是唯一输出树的画布和归属页面;其它标记为 SLAVE 的图片只提供视觉上下文
本次调用只能调用 recognize_ui_structure 工具返回一个节点列表,不得返回树数组、树 ID 或其它页面的节点。
每个节点必须使用 src_ui_design_image_id 指定坐标来源,并在该图片的原始像素坐标系中输出
角色:
你是游戏 UI 多图结构识别器
任务:
同时分析同一 UI 系统的全部参考图,建立UI树
用户会给你一些UI截图(它们从属于同一个UI系统)和对应的元数据, 请用给定的工具描述UI结构
你是游戏 UI 结构识别器。
识别规则:
* 只识别 UI,不识别场景人物、地形、建筑、光影和背景装饰。
* 不创建没有独立功能的微小装饰Node。
* 无法确定类型、层级、关系时,在 UnSure 中写明原因。
* source=user、reviewStatus=confirmed 或 semanticSource=user 的组件不得被 AI 覆盖。
* 每个Node必须使用 src_ui_design_image_id 指定坐标来源(所属UI design图),并在该图片的原始像素坐标系中输出
global_pos_x_px、global_pos_y_px、width_px、height_px;坐标相对于来源图片左上角
* 多张图都是在描述同一个UI系统, 必须合并共用公共的框架/层次, 禁止简单每个图一个树, 坐标等有矛盾的地方父级优先
* 面向用户的字段如名称描述等请用中文
"#;
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
@@ -82,8 +98,6 @@ struct RecognitionNode {
global_pos_y_px: u32,
width_px: u32,
height_px: u32,
/// 视觉证据来源;只允许当前 PAGE 或其直接 slave,不能用来改变树归属或坐标系。
#[schemars(with = "String")]
src_ui_design_image_id: UIDesignImageId,
local_anchor: Anchor,
name: String,
@@ -306,6 +320,29 @@ fn validate_source_image_ids(
mod tests {
use super::*;
fn assert_no_one_of(value: &serde_json::Value) {
match value {
serde_json::Value::Object(object) => {
assert!(!object.contains_key("oneOf"));
for child in object.values() {
assert_no_one_of(child);
}
}
serde_json::Value::Array(values) => {
for child in values {
assert_no_one_of(child);
}
}
_ => {}
}
}
#[test]
fn strict_schema_rewrites_union_variants_for_provider() {
let schema = recognition_json_schema().expect("recognition schema");
assert_no_one_of(&schema);
}
#[test]
fn preset_anchor_ranges_use_top_left_image_coordinates() {
let (min, max) = anchor_ranges(&Anchor::Preset(PresetAnchor {
@@ -529,10 +566,23 @@ pub(crate) async fn recognize_ui_impl(
convert_node(node, page, source_image, &state.ui_design_images, root_rect)
})
.collect::<Result<Vec<_>, _>>()?;
// Tree identity is assigned by Rust from the Page image, never chosen by the model.
let root = LayoutNode {
id: random_node_id()?,
transform: Transform::stretch(),
metadata: NodeMetadata {
name: "页面根节点".to_string(),
description: String::new(),
status: NodeStatus::Passed,
source: NodeSource::System,
},
// TODO: root components can host page-level background content once component preview is supported.
components: Vec::new(),
children,
};
// Tree identity and root identity are assigned by Rust, never chosen by the model.
ui_trees.push(UITree {
src_ui_design: page_id,
children,
root,
});
}
Ok(RecognitionDTO { ui_trees })
@@ -17,8 +17,8 @@ const SYSTEM_PROMPT: &str = r#"
请识别这些 UI 参考图的界面语义,并调用 suggest_ui_design_semantics 工具返回结果。
不要在工具调用外输出 JSON、Markdown、代码围栏、注释、额外字段或解释文字。
字段含义和 null 规则
- ui_design_image_id:必填,必须逐字匹配当前输入参考图的 id。它标识“这条建议属于哪张图”,不是新 ID,不能为 null。
字段含义:
- id:必填,必须逐字匹配当前输入参考图的 id。它标识“这条建议属于哪张图”,不是新 ID,不能为 null。
- name:要写入该图片 metadata 的简短、可读名称;
- description:要写入该图片 metadata 的简短语义描述,例如“带底部导航的主游戏页面”。
- role:要写入该图片 metadata 的界面角色。
@@ -28,90 +28,36 @@ const SYSTEM_PROMPT: &str = r#"
State 表示同一界面的状态变体,
Scrolled 表示滚动或分页后的内容,
Detail 表示局部详情或补充证据。
- slave_to:要写入该图片 metadata 的归属页面 id。只有当当前图片明显是另一张输入图的局部/子界面时才填写那个宿主图的 id;
Page 不得设置宿主。
- children:该节点直接包含的子界面图。根节点必须是 Page,子节点不能是 Page;没有子节点时返回空数组。
所有字段都必须出现nullable 字段使用 JSON null 表示“不要修改该字段”,不要省略字段
每张参考图最多返回一条建议,且不得重复 ui_design_image_id。
以下UI设计图中的metadata部分信息已确定, 请补全未确定/为空的信息
所有字段都必须出现,禁止省略字段使用 null 或返回空字符串。每张参考图最多出现一次;id 必须逐字匹配输入图片 id
以下 UI 设计图中的 metadata 部分信息已确定,请根据已有信息补全语义.
"#;
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS, JsonSchema)]
#[schemars(deny_unknown_fields)]
#[serde(deny_unknown_fields)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct UIDesignSuggestion {
pub ui_design_image_id: UIDesignImageId,
#[schemars(required)]
pub name: Option<String>,
#[schemars(required)]
pub description: Option<String>,
#[schemars(required)]
pub role: Option<UIDesignImageRole>,
#[schemars(required)]
pub slave_to: Option<UIDesignImageId>,
pub struct UIDesignSuggestionTreeNode {
pub id: UIDesignImageId,
pub name: String,
pub description: String,
pub role: UIDesignImageRole,
pub children: Vec<UIDesignSuggestionTreeNode>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema)]
#[schemars(deny_unknown_fields)]
#[serde(deny_unknown_fields)]
struct UIDesignSuggestionResponse {
suggestions: Vec<UIDesignSuggestion>,
struct UIDesignSuggestion {
ui_designs: Vec<UIDesignSuggestionTreeNode>,
}
fn ui_design_suggestion_json_schema() -> Result<serde_json::Value, String> {
strict_json_schema::<UIDesignSuggestionResponse>()
strict_json_schema::<UIDesignSuggestion>()
}
const MAX_REFERENCES: usize = 4;
#[cfg(test)]
mod tests {
use super::*;
fn assert_no_refs(value: &serde_json::Value) {
match value {
serde_json::Value::Object(object) => {
assert!(!object.contains_key("$ref"));
assert!(!object.contains_key("$defs"));
for child in object.values() {
assert_no_refs(child);
}
}
serde_json::Value::Array(values) => {
for child in values {
assert_no_refs(child);
}
}
_ => {}
}
}
#[test]
fn strict_schema_is_inline_and_requires_all_properties() {
let schema = ui_design_suggestion_json_schema().expect("schema");
assert_no_refs(&schema);
assert_eq!(schema["required"], serde_json::json!(["suggestions"]));
let required = schema["properties"]["suggestions"]["items"]["required"]
.as_array()
.expect("item required array")
.iter()
.filter_map(serde_json::Value::as_str)
.collect::<std::collections::BTreeSet<_>>();
assert_eq!(
required,
[
"ui_design_image_id",
"name",
"description",
"role",
"slave_to"
]
.into_iter()
.collect()
);
}
}
fn image_data_url(path: &Path, bytes: &[u8]) -> Result<String, String> {
let mime = match path.extension().and_then(|value| value.to_str()) {
Some("png") => "image/png",
@@ -126,34 +72,47 @@ fn image_data_url(path: &Path, bytes: &[u8]) -> Result<String, String> {
}
fn validate_suggestions(
suggestions: &[UIDesignSuggestion],
suggestions: &[UIDesignSuggestionTreeNode],
image_ids: &HashSet<UIDesignImageId>,
) -> Result<(), String> {
let mut seen = HashSet::new();
for suggestion in suggestions {
if !image_ids.contains(&suggestion.ui_design_image_id) {
return Err("LLM 返回了未知界面图 ID".to_string());
}
if !seen.insert(suggestion.ui_design_image_id.clone()) {
return Err("LLM 返回了重复界面图建议".to_string());
}
if suggestion.role == Some(UIDesignImageRole::Page) && suggestion.slave_to.is_some() {
return Err("主页面不能设置归属页面".to_string());
}
if let Some(host) = &suggestion.slave_to {
if host == &suggestion.ui_design_image_id || !image_ids.contains(host) {
return Err("界面图归属页面无效".to_string());
fn visit(
nodes: &[UIDesignSuggestionTreeNode],
is_root: bool,
image_ids: &HashSet<UIDesignImageId>,
seen: &mut HashSet<UIDesignImageId>,
) -> Result<(), String> {
for node in nodes {
if !image_ids.contains(&node.id) {
return Err("LLM 返回了未知界面图 ID".to_string());
}
if !seen.insert(node.id.clone()) {
return Err("LLM 返回了重复界面图建议".to_string());
}
if node.name.trim().is_empty() || node.description.trim().is_empty() {
return Err("界面图语义字段不能是空字符串".to_string());
}
if is_root {
if node.role != UIDesignImageRole::Page {
return Err("界面图树根节点必须是 Page".to_string());
}
} else if node.role == UIDesignImageRole::Page {
return Err("Page 不能作为其它界面图的子节点".to_string());
}
visit(&node.children, false, image_ids, seen)?;
}
Ok(())
}
Ok(())
visit(suggestions, true, image_ids, &mut seen)
}
#[tauri::command]
pub(crate) async fn suggest_ui_design_semantic_impl(
project_path: String,
state: State,
) -> Result<Vec<UIDesignSuggestion>, String> {
) -> Result<Vec<UIDesignSuggestionTreeNode>, String> {
if state.ui_design_images.is_empty() {
eprintln!("ui_design_suggestion.error stage=validate reason=no_images");
return Err("请先导入界面图".to_string());
@@ -167,7 +126,6 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
}
let root = Path::new(project_path.trim());
let mut parts = Vec::new();
let mut ids = HashSet::new();
for (id, image) in &state.ui_design_images {
ids.insert(id.clone());
@@ -187,12 +145,7 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
format!("读取界面图失败:{error}")
})?;
parts.push(LlmMessageContentPart::InputText {
text: format!(
"REFERENCE id={} metadata={} pixel_size={:?}",
id.as_str(),
image.metadata,
image.pixel_size
),
text: format!("REFERENCE id:{} metadata:{}", id.as_str(), image.metadata,),
});
parts.push(LlmMessageContentPart::InputImage {
image_url: image_data_url(&absolute, &bytes)?,
@@ -238,12 +191,12 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
eprintln!("ui_design_suggestion.error stage=parse_tool_call reason=missing_tool_call");
"LLM 未返回 suggest_ui_design_semantics 工具调用".to_string()
})?;
let suggestions = serde_json::from_str::<UIDesignSuggestionResponse>(&call.arguments)
let suggestions = serde_json::from_str::<UIDesignSuggestion>(&call.arguments)
.map_err(|error| {
eprintln!("ui_design_suggestion.error stage=parse_arguments error={error}");
format!("UI 语义建议工具参数无效:{error}")
})?
.suggestions;
.ui_designs;
validate_suggestions(&suggestions, &ids).map_err(|error| {
eprintln!("ui_design_suggestion.error stage=validate_result error={error}");
error
@@ -251,3 +204,89 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
// TODO: future single-page policy may reject multiple Page suggestions.
Ok(suggestions)
}
#[cfg(test)]
mod tests {
use super::*;
fn id(value: &str) -> UIDesignImageId {
UIDesignImageId::new(value).expect("valid image id")
}
fn node(
value: &str,
role: UIDesignImageRole,
children: Vec<UIDesignSuggestionTreeNode>,
) -> UIDesignSuggestionTreeNode {
UIDesignSuggestionTreeNode {
id: id(value),
name: value.to_string(),
description: format!("{value} description"),
role,
children,
}
}
fn image_ids(values: &[&str]) -> HashSet<UIDesignImageId> {
values.iter().map(|value| id(value)).collect()
}
#[test]
fn validates_page_tree() {
let suggestions = vec![node(
"page",
UIDesignImageRole::Page,
vec![node("section", UIDesignImageRole::Section, Vec::new())],
)];
assert!(validate_suggestions(&suggestions, &image_ids(&["page", "section"])).is_ok());
}
#[test]
fn rejects_duplicate_and_unknown_ids() {
let duplicate = vec![
node(
"page",
UIDesignImageRole::Page,
vec![node("section", UIDesignImageRole::Section, Vec::new())],
),
node("page", UIDesignImageRole::Page, Vec::new()),
];
assert!(validate_suggestions(&duplicate, &image_ids(&["page", "section"])).is_err());
let unknown = vec![node("missing", UIDesignImageRole::Page, Vec::new())];
assert!(validate_suggestions(&unknown, &image_ids(&["page"])).is_err());
}
#[test]
fn rejects_invalid_root_roles_and_nested_pages() {
let non_page_root = vec![node("section", UIDesignImageRole::Section, Vec::new())];
assert!(validate_suggestions(&non_page_root, &image_ids(&["section"])).is_err());
let nested_page = vec![node(
"page",
UIDesignImageRole::Page,
vec![node("nested", UIDesignImageRole::Page, Vec::new())],
)];
assert!(validate_suggestions(&nested_page, &image_ids(&["page", "nested"])).is_err());
}
#[test]
fn strict_schema_requires_tree_fields() {
let schema = ui_design_suggestion_json_schema().expect("schema");
assert_eq!(schema["required"], serde_json::json!(["ui_designs"]));
let node_schema = &schema["properties"]["ui_designs"]["items"];
let required = node_schema["required"]
.as_array()
.expect("node required")
.iter()
.filter_map(serde_json::Value::as_str)
.collect::<HashSet<_>>();
assert_eq!(
required,
["id", "name", "description", "role", "children"]
.into_iter()
.collect()
);
assert_eq!(node_schema["additionalProperties"], false);
}
}
@@ -13,6 +13,9 @@ pub(crate) fn strict_json_schema<T: JsonSchema>() -> Result<serde_json::Value, S
fn normalize_strict_schema(schema: &mut serde_json::Value) {
match schema {
serde_json::Value::Object(object) => {
if let Some(one_of) = object.remove("oneOf") {
object.insert("anyOf".to_string(), one_of);
}
if let Some(properties) = object.get("properties").and_then(|value| value.as_object()) {
object.insert(
"required".to_string(),
@@ -17,6 +17,7 @@ pub struct Node {
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub enum NodeSource {
System,
Human,
Llm,
}
@@ -11,7 +11,7 @@ use ts_rs::TS;
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct UITree {
pub src_ui_design: UIDesignImageId,
pub children: Vec<Node>,
pub root: Node,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
@@ -1,3 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type NodeSource = "Human" | "Llm";
export type NodeSource = "System" | "Human" | "Llm";
@@ -1,5 +0,0 @@
// This file is generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { UIDesignImageId } from "./UIDesignImageId";
import type { UIDesignImageRole } from "./UIDesignImageRole";
export type UIDesignSuggestion = { ui_design_image_id: UIDesignImageId, name: string | null, description: string | null, role: UIDesignImageRole | null, slave_to: UIDesignImageId | null, };
@@ -2,4 +2,4 @@
import type { Node } from "./Node";
import type { UIDesignImageId } from "./UIDesignImageId";
export type UITree = { src_ui_design: UIDesignImageId, children: Array<Node>, };
export type UITree = { src_ui_design: UIDesignImageId, root: Node, };