重构 UI 设计语义建议和结构:
- 替换 `UIDesignSuggestion` 为树型结构 `UIDesignSuggestionTreeNode`,统一描述界面图语义。 - 更新 UI 树 `UITree` 数据结构,新增 `root` 字段取代 `children`。 - 调整 `NodeSource` 枚举,添加新值 `System`。 - 修改语义建议逻辑:移除 `oneOf` 节点限制,强制所有字段不可省略。 - 增强验证和测试覆盖率,确保数据模型和树结构一致性。
This commit is contained in:
@@ -114,7 +114,7 @@ use windows::*;
|
||||
async fn suggest_ui_design_semantic(
|
||||
project_path: String,
|
||||
state: ui_editor::state::State,
|
||||
) -> Result<Vec<ui_editor::commands::UIDesignSuggestion>, String> {
|
||||
) -> Result<Vec<ui_editor::commands::UIDesignSuggestionTreeNode>, String> {
|
||||
ui_editor::commands::suggest_ui_design_semantic_impl(project_path, state).await
|
||||
}
|
||||
|
||||
|
||||
@@ -5,4 +5,4 @@ pub mod utils;
|
||||
pub(crate) use recognition::recognize_ui_impl;
|
||||
pub use recognition::RecognitionDTO;
|
||||
pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl;
|
||||
pub use ui_design_suggestion::UIDesignSuggestion;
|
||||
pub use ui_design_suggestion::UIDesignSuggestionTreeNode;
|
||||
|
||||
@@ -19,12 +19,28 @@ use ts_rs::TS;
|
||||
|
||||
const MAX_REFERENCES: usize = 4;
|
||||
|
||||
// TODO
|
||||
// * 头像、图标或物品图只识别为一个图片或头像组件。
|
||||
// * 文本和动态数值即使没有独立素材,也必须保留为组件。
|
||||
const SYSTEM_PROMPT: &str = r#"
|
||||
你是游戏 UI 结构识别器。当前请求只处理一个 Page 参考图,以及直接归属于该 Page 的 slave 参考图。
|
||||
第一张标记为 PAGE 的图片是唯一输出树的画布和归属页面;其它标记为 SLAVE 的图片只提供视觉上下文。
|
||||
本次调用只能调用 recognize_ui_structure 工具返回一个节点列表,不得返回树数组、树 ID 或其它页面的节点。
|
||||
每个节点必须使用 src_ui_design_image_id 指定坐标来源,并在该图片的原始像素坐标系中输出
|
||||
角色:
|
||||
你是游戏 UI 多图结构识别器。
|
||||
|
||||
任务:
|
||||
同时分析同一 UI 系统的全部参考图,建立UI树
|
||||
|
||||
用户会给你一些UI截图(它们从属于同一个UI系统)和对应的元数据, 请用给定的工具描述UI结构
|
||||
你是游戏 UI 结构识别器。
|
||||
|
||||
识别规则:
|
||||
* 只识别 UI,不识别场景人物、地形、建筑、光影和背景装饰。
|
||||
* 不创建没有独立功能的微小装饰Node。
|
||||
* 无法确定类型、层级、关系时,在 UnSure 中写明原因。
|
||||
* source=user、reviewStatus=confirmed 或 semanticSource=user 的组件不得被 AI 覆盖。
|
||||
* 每个Node必须使用 src_ui_design_image_id 指定坐标来源(所属UI design图),并在该图片的原始像素坐标系中输出
|
||||
global_pos_x_px、global_pos_y_px、width_px、height_px;坐标相对于来源图片左上角
|
||||
* 多张图都是在描述同一个UI系统, 必须合并共用公共的框架/层次, 禁止简单每个图一个树, 坐标等有矛盾的地方父级优先
|
||||
* 面向用户的字段如名称描述等请用中文
|
||||
"#;
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
@@ -82,8 +98,6 @@ struct RecognitionNode {
|
||||
global_pos_y_px: u32,
|
||||
width_px: u32,
|
||||
height_px: u32,
|
||||
/// 视觉证据来源;只允许当前 PAGE 或其直接 slave,不能用来改变树归属或坐标系。
|
||||
#[schemars(with = "String")]
|
||||
src_ui_design_image_id: UIDesignImageId,
|
||||
local_anchor: Anchor,
|
||||
name: String,
|
||||
@@ -306,6 +320,29 @@ fn validate_source_image_ids(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn assert_no_one_of(value: &serde_json::Value) {
|
||||
match value {
|
||||
serde_json::Value::Object(object) => {
|
||||
assert!(!object.contains_key("oneOf"));
|
||||
for child in object.values() {
|
||||
assert_no_one_of(child);
|
||||
}
|
||||
}
|
||||
serde_json::Value::Array(values) => {
|
||||
for child in values {
|
||||
assert_no_one_of(child);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_schema_rewrites_union_variants_for_provider() {
|
||||
let schema = recognition_json_schema().expect("recognition schema");
|
||||
assert_no_one_of(&schema);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preset_anchor_ranges_use_top_left_image_coordinates() {
|
||||
let (min, max) = anchor_ranges(&Anchor::Preset(PresetAnchor {
|
||||
@@ -529,10 +566,23 @@ pub(crate) async fn recognize_ui_impl(
|
||||
convert_node(node, page, source_image, &state.ui_design_images, root_rect)
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
// Tree identity is assigned by Rust from the Page image, never chosen by the model.
|
||||
let root = LayoutNode {
|
||||
id: random_node_id()?,
|
||||
transform: Transform::stretch(),
|
||||
metadata: NodeMetadata {
|
||||
name: "页面根节点".to_string(),
|
||||
description: String::new(),
|
||||
status: NodeStatus::Passed,
|
||||
source: NodeSource::System,
|
||||
},
|
||||
// TODO: root components can host page-level background content once component preview is supported.
|
||||
components: Vec::new(),
|
||||
children,
|
||||
};
|
||||
// Tree identity and root identity are assigned by Rust, never chosen by the model.
|
||||
ui_trees.push(UITree {
|
||||
src_ui_design: page_id,
|
||||
children,
|
||||
root,
|
||||
});
|
||||
}
|
||||
Ok(RecognitionDTO { ui_trees })
|
||||
|
||||
+132
-93
@@ -17,8 +17,8 @@ const SYSTEM_PROMPT: &str = r#"
|
||||
请识别这些 UI 参考图的界面语义,并调用 suggest_ui_design_semantics 工具返回结果。
|
||||
不要在工具调用外输出 JSON、Markdown、代码围栏、注释、额外字段或解释文字。
|
||||
|
||||
字段含义和 null 规则:
|
||||
- ui_design_image_id:必填,必须逐字匹配当前输入参考图的 id。它标识“这条建议属于哪张图”,不是新 ID,不能为 null。
|
||||
字段含义:
|
||||
- id:必填,必须逐字匹配当前输入参考图的 id。它标识“这条建议属于哪张图”,不是新 ID,不能为 null。
|
||||
- name:要写入该图片 metadata 的简短、可读名称;
|
||||
- description:要写入该图片 metadata 的简短语义描述,例如“带底部导航的主游戏页面”。
|
||||
- role:要写入该图片 metadata 的界面角色。
|
||||
@@ -28,90 +28,36 @@ const SYSTEM_PROMPT: &str = r#"
|
||||
State 表示同一界面的状态变体,
|
||||
Scrolled 表示滚动或分页后的内容,
|
||||
Detail 表示局部详情或补充证据。
|
||||
- slave_to:要写入该图片 metadata 的归属页面 id。只有当当前图片明显是另一张输入图的局部/子界面时才填写那个宿主图的 id;
|
||||
Page 不得设置宿主。
|
||||
- children:该节点直接包含的子界面图。根节点必须是 Page,子节点不能是 Page;没有子节点时返回空数组。
|
||||
|
||||
所有字段都必须出现;nullable 字段使用 JSON null 表示“不要修改该字段”,不要省略字段。
|
||||
每张参考图最多返回一条建议,且不得重复 ui_design_image_id。
|
||||
以下UI设计图中的metadata部分信息已确定, 请补全未确定/为空的信息
|
||||
所有字段都必须出现,禁止省略字段、使用 null 或返回空字符串。每张参考图最多出现一次;id 必须逐字匹配输入图片 id。
|
||||
以下 UI 设计图中的 metadata 部分信息已确定,请根据已有信息补全语义.
|
||||
"#;
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct UIDesignSuggestion {
|
||||
pub ui_design_image_id: UIDesignImageId,
|
||||
#[schemars(required)]
|
||||
pub name: Option<String>,
|
||||
#[schemars(required)]
|
||||
pub description: Option<String>,
|
||||
#[schemars(required)]
|
||||
pub role: Option<UIDesignImageRole>,
|
||||
#[schemars(required)]
|
||||
pub slave_to: Option<UIDesignImageId>,
|
||||
pub struct UIDesignSuggestionTreeNode {
|
||||
pub id: UIDesignImageId,
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
pub role: UIDesignImageRole,
|
||||
pub children: Vec<UIDesignSuggestionTreeNode>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct UIDesignSuggestionResponse {
|
||||
suggestions: Vec<UIDesignSuggestion>,
|
||||
struct UIDesignSuggestion {
|
||||
ui_designs: Vec<UIDesignSuggestionTreeNode>,
|
||||
}
|
||||
|
||||
fn ui_design_suggestion_json_schema() -> Result<serde_json::Value, String> {
|
||||
strict_json_schema::<UIDesignSuggestionResponse>()
|
||||
strict_json_schema::<UIDesignSuggestion>()
|
||||
}
|
||||
|
||||
const MAX_REFERENCES: usize = 4;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn assert_no_refs(value: &serde_json::Value) {
|
||||
match value {
|
||||
serde_json::Value::Object(object) => {
|
||||
assert!(!object.contains_key("$ref"));
|
||||
assert!(!object.contains_key("$defs"));
|
||||
for child in object.values() {
|
||||
assert_no_refs(child);
|
||||
}
|
||||
}
|
||||
serde_json::Value::Array(values) => {
|
||||
for child in values {
|
||||
assert_no_refs(child);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_schema_is_inline_and_requires_all_properties() {
|
||||
let schema = ui_design_suggestion_json_schema().expect("schema");
|
||||
assert_no_refs(&schema);
|
||||
assert_eq!(schema["required"], serde_json::json!(["suggestions"]));
|
||||
let required = schema["properties"]["suggestions"]["items"]["required"]
|
||||
.as_array()
|
||||
.expect("item required array")
|
||||
.iter()
|
||||
.filter_map(serde_json::Value::as_str)
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(
|
||||
required,
|
||||
[
|
||||
"ui_design_image_id",
|
||||
"name",
|
||||
"description",
|
||||
"role",
|
||||
"slave_to"
|
||||
]
|
||||
.into_iter()
|
||||
.collect()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn image_data_url(path: &Path, bytes: &[u8]) -> Result<String, String> {
|
||||
let mime = match path.extension().and_then(|value| value.to_str()) {
|
||||
Some("png") => "image/png",
|
||||
@@ -126,34 +72,47 @@ fn image_data_url(path: &Path, bytes: &[u8]) -> Result<String, String> {
|
||||
}
|
||||
|
||||
fn validate_suggestions(
|
||||
suggestions: &[UIDesignSuggestion],
|
||||
suggestions: &[UIDesignSuggestionTreeNode],
|
||||
image_ids: &HashSet<UIDesignImageId>,
|
||||
) -> Result<(), String> {
|
||||
let mut seen = HashSet::new();
|
||||
for suggestion in suggestions {
|
||||
if !image_ids.contains(&suggestion.ui_design_image_id) {
|
||||
return Err("LLM 返回了未知界面图 ID".to_string());
|
||||
}
|
||||
if !seen.insert(suggestion.ui_design_image_id.clone()) {
|
||||
return Err("LLM 返回了重复界面图建议".to_string());
|
||||
}
|
||||
if suggestion.role == Some(UIDesignImageRole::Page) && suggestion.slave_to.is_some() {
|
||||
return Err("主页面不能设置归属页面".to_string());
|
||||
}
|
||||
if let Some(host) = &suggestion.slave_to {
|
||||
if host == &suggestion.ui_design_image_id || !image_ids.contains(host) {
|
||||
return Err("界面图归属页面无效".to_string());
|
||||
|
||||
fn visit(
|
||||
nodes: &[UIDesignSuggestionTreeNode],
|
||||
is_root: bool,
|
||||
image_ids: &HashSet<UIDesignImageId>,
|
||||
seen: &mut HashSet<UIDesignImageId>,
|
||||
) -> Result<(), String> {
|
||||
for node in nodes {
|
||||
if !image_ids.contains(&node.id) {
|
||||
return Err("LLM 返回了未知界面图 ID".to_string());
|
||||
}
|
||||
if !seen.insert(node.id.clone()) {
|
||||
return Err("LLM 返回了重复界面图建议".to_string());
|
||||
}
|
||||
if node.name.trim().is_empty() || node.description.trim().is_empty() {
|
||||
return Err("界面图语义字段不能是空字符串".to_string());
|
||||
}
|
||||
if is_root {
|
||||
if node.role != UIDesignImageRole::Page {
|
||||
return Err("界面图树根节点必须是 Page".to_string());
|
||||
}
|
||||
} else if node.role == UIDesignImageRole::Page {
|
||||
return Err("Page 不能作为其它界面图的子节点".to_string());
|
||||
}
|
||||
visit(&node.children, false, image_ids, seen)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Ok(())
|
||||
|
||||
visit(suggestions, true, image_ids, &mut seen)
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub(crate) async fn suggest_ui_design_semantic_impl(
|
||||
project_path: String,
|
||||
state: State,
|
||||
) -> Result<Vec<UIDesignSuggestion>, String> {
|
||||
) -> Result<Vec<UIDesignSuggestionTreeNode>, String> {
|
||||
if state.ui_design_images.is_empty() {
|
||||
eprintln!("ui_design_suggestion.error stage=validate reason=no_images");
|
||||
return Err("请先导入界面图".to_string());
|
||||
@@ -167,7 +126,6 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
|
||||
}
|
||||
let root = Path::new(project_path.trim());
|
||||
let mut parts = Vec::new();
|
||||
|
||||
let mut ids = HashSet::new();
|
||||
for (id, image) in &state.ui_design_images {
|
||||
ids.insert(id.clone());
|
||||
@@ -187,12 +145,7 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
|
||||
format!("读取界面图失败:{error}")
|
||||
})?;
|
||||
parts.push(LlmMessageContentPart::InputText {
|
||||
text: format!(
|
||||
"REFERENCE id={} metadata={} pixel_size={:?}",
|
||||
id.as_str(),
|
||||
image.metadata,
|
||||
image.pixel_size
|
||||
),
|
||||
text: format!("REFERENCE id:{} metadata:{}", id.as_str(), image.metadata,),
|
||||
});
|
||||
parts.push(LlmMessageContentPart::InputImage {
|
||||
image_url: image_data_url(&absolute, &bytes)?,
|
||||
@@ -238,12 +191,12 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
|
||||
eprintln!("ui_design_suggestion.error stage=parse_tool_call reason=missing_tool_call");
|
||||
"LLM 未返回 suggest_ui_design_semantics 工具调用".to_string()
|
||||
})?;
|
||||
let suggestions = serde_json::from_str::<UIDesignSuggestionResponse>(&call.arguments)
|
||||
let suggestions = serde_json::from_str::<UIDesignSuggestion>(&call.arguments)
|
||||
.map_err(|error| {
|
||||
eprintln!("ui_design_suggestion.error stage=parse_arguments error={error}");
|
||||
format!("UI 语义建议工具参数无效:{error}")
|
||||
})?
|
||||
.suggestions;
|
||||
.ui_designs;
|
||||
validate_suggestions(&suggestions, &ids).map_err(|error| {
|
||||
eprintln!("ui_design_suggestion.error stage=validate_result error={error}");
|
||||
error
|
||||
@@ -251,3 +204,89 @@ pub(crate) async fn suggest_ui_design_semantic_impl(
|
||||
// TODO: future single-page policy may reject multiple Page suggestions.
|
||||
Ok(suggestions)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn id(value: &str) -> UIDesignImageId {
|
||||
UIDesignImageId::new(value).expect("valid image id")
|
||||
}
|
||||
|
||||
fn node(
|
||||
value: &str,
|
||||
role: UIDesignImageRole,
|
||||
children: Vec<UIDesignSuggestionTreeNode>,
|
||||
) -> UIDesignSuggestionTreeNode {
|
||||
UIDesignSuggestionTreeNode {
|
||||
id: id(value),
|
||||
name: value.to_string(),
|
||||
description: format!("{value} description"),
|
||||
role,
|
||||
children,
|
||||
}
|
||||
}
|
||||
|
||||
fn image_ids(values: &[&str]) -> HashSet<UIDesignImageId> {
|
||||
values.iter().map(|value| id(value)).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validates_page_tree() {
|
||||
let suggestions = vec![node(
|
||||
"page",
|
||||
UIDesignImageRole::Page,
|
||||
vec![node("section", UIDesignImageRole::Section, Vec::new())],
|
||||
)];
|
||||
assert!(validate_suggestions(&suggestions, &image_ids(&["page", "section"])).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_duplicate_and_unknown_ids() {
|
||||
let duplicate = vec![
|
||||
node(
|
||||
"page",
|
||||
UIDesignImageRole::Page,
|
||||
vec![node("section", UIDesignImageRole::Section, Vec::new())],
|
||||
),
|
||||
node("page", UIDesignImageRole::Page, Vec::new()),
|
||||
];
|
||||
assert!(validate_suggestions(&duplicate, &image_ids(&["page", "section"])).is_err());
|
||||
|
||||
let unknown = vec![node("missing", UIDesignImageRole::Page, Vec::new())];
|
||||
assert!(validate_suggestions(&unknown, &image_ids(&["page"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_root_roles_and_nested_pages() {
|
||||
let non_page_root = vec![node("section", UIDesignImageRole::Section, Vec::new())];
|
||||
assert!(validate_suggestions(&non_page_root, &image_ids(&["section"])).is_err());
|
||||
|
||||
let nested_page = vec![node(
|
||||
"page",
|
||||
UIDesignImageRole::Page,
|
||||
vec![node("nested", UIDesignImageRole::Page, Vec::new())],
|
||||
)];
|
||||
assert!(validate_suggestions(&nested_page, &image_ids(&["page", "nested"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_schema_requires_tree_fields() {
|
||||
let schema = ui_design_suggestion_json_schema().expect("schema");
|
||||
assert_eq!(schema["required"], serde_json::json!(["ui_designs"]));
|
||||
let node_schema = &schema["properties"]["ui_designs"]["items"];
|
||||
let required = node_schema["required"]
|
||||
.as_array()
|
||||
.expect("node required")
|
||||
.iter()
|
||||
.filter_map(serde_json::Value::as_str)
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
required,
|
||||
["id", "name", "description", "role", "children"]
|
||||
.into_iter()
|
||||
.collect()
|
||||
);
|
||||
assert_eq!(node_schema["additionalProperties"], false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,9 @@ pub(crate) fn strict_json_schema<T: JsonSchema>() -> Result<serde_json::Value, S
|
||||
fn normalize_strict_schema(schema: &mut serde_json::Value) {
|
||||
match schema {
|
||||
serde_json::Value::Object(object) => {
|
||||
if let Some(one_of) = object.remove("oneOf") {
|
||||
object.insert("anyOf".to_string(), one_of);
|
||||
}
|
||||
if let Some(properties) = object.get("properties").and_then(|value| value.as_object()) {
|
||||
object.insert(
|
||||
"required".to_string(),
|
||||
|
||||
@@ -17,6 +17,7 @@ pub struct Node {
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub enum NodeSource {
|
||||
System,
|
||||
Human,
|
||||
Llm,
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ use ts_rs::TS;
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct UITree {
|
||||
pub src_ui_design: UIDesignImageId,
|
||||
pub children: Vec<Node>,
|
||||
pub root: Node,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
|
||||
export type NodeSource = "Human" | "Llm";
|
||||
export type NodeSource = "System" | "Human" | "Llm";
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
// This file is generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { UIDesignImageId } from "./UIDesignImageId";
|
||||
import type { UIDesignImageRole } from "./UIDesignImageRole";
|
||||
|
||||
export type UIDesignSuggestion = { ui_design_image_id: UIDesignImageId, name: string | null, description: string | null, role: UIDesignImageRole | null, slave_to: UIDesignImageId | null, };
|
||||
@@ -2,4 +2,4 @@
|
||||
import type { Node } from "./Node";
|
||||
import type { UIDesignImageId } from "./UIDesignImageId";
|
||||
|
||||
export type UITree = { src_ui_design: UIDesignImageId, children: Array<Node>, };
|
||||
export type UITree = { src_ui_design: UIDesignImageId, root: Node, };
|
||||
|
||||
Reference in New Issue
Block a user