实现UI自动分离命令

新增分离树、批次处理与sidecar状态

接入Raw GPT Image 2与视觉绑定重试

注册Tauri命令并补充识别组件草稿
This commit is contained in:
2026-09-08 18:07:27 +08:00
parent b2d4690f94
commit 4cc7f4dd64
5 changed files with 906 additions and 5 deletions
@@ -328,6 +328,15 @@ async fn recognize_ui(
ui_editor::commands::recognize_ui_impl(project_path, state).await
}
#[tauri::command]
async fn separate_ui(
project_path: String,
asset_id: String,
state: ui_editor::state::State,
) -> Result<ui_editor::commands::SeparationDTO, String> {
ui_editor::commands::separate_ui_impl(project_path, asset_id, state).await
}
#[tauri::command]
async fn merge_ui(state: ui_editor::state::State) -> Result<ui_editor::commands::MergeDTO, String> {
ui_editor::commands::merge_ui_impl(state).await
@@ -2559,6 +2568,7 @@ fn main() {
check_ui_editor_font_glyph_coverage,
suggest_ui_design_semantic,
recognize_ui,
separate_ui,
merge_ui,
bind_components,
load_ui_design_state,
@@ -1,6 +1,7 @@
pub mod binding;
pub mod merge;
pub mod recognition;
pub mod separation;
pub mod ui_design_suggestion;
pub mod utils;
@@ -10,5 +11,7 @@ pub use merge::MergeDTO;
pub(crate) use merge::{merge_ui_impl, merge_ui_impl_with_provider};
pub use recognition::RecognitionDTO;
pub(crate) use recognition::{recognize_ui_impl, recognize_ui_impl_with_provider};
pub(crate) use separation::separate_ui_impl;
pub use separation::SeparationDTO;
pub(crate) use ui_design_suggestion::suggest_ui_design_semantic_impl;
pub use ui_design_suggestion::UIDesignSuggestionTreeNode;
@@ -4,6 +4,7 @@ use crate::ui_editor::commands::utils::{
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm,
strict_json_schema,
};
use crate::ui_editor::component::Component;
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
use crate::ui_editor::layout::control_layout::ControlLayout;
use crate::ui_editor::layout::dimension::UIRect;
@@ -46,6 +47,8 @@ const SYSTEM_PROMPT: &str = r#"
* 由于每个截图未必是完整的, 可能是局部的, 每棵树描述清楚每个截图上UI的层次结构即可
* 不同树的共用框架/层次/...请使用使用相同的名称描述. 不同状态/变体名称使用相同的前缀, 用后缀区别
* 粒度要求: 尽可能细致, 最小单元举例: 进度条的底槽、填充和外框; slider的底槽, dragger等
* 为每个节点直接返回完整 components。纯容器返回空数组;需要从设计图自动分离图片的 Image component 必须令 target_graphic 为 null。文本内容和组件类型完全由视觉判断,不调用或依赖 OCR。
* 每个节点当前最多返回一个 Image component 和一个 Text component。
"#;
@@ -109,6 +112,7 @@ struct RecognitionNode {
description: String,
children: Vec<RecognitionNode>,
confidence: Confidence,
components: Vec<Component>,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
@@ -312,10 +316,7 @@ fn convert_node(
allow_llm_edit_component: true,
source: NodeSource::Llm,
},
// V1 有意把识别结果限定为“结构草稿”:组件绑定属于后续独立阶段。
// 因此空组件不是丢失数据,而是等待 visual-binding 阶段补齐 Image/Text。
// 约定见 docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md。
components: Vec::new(),
components: source.components.clone(),
children_display_mode: ChildrenDisplayMode::Stack,
children,
})
@@ -323,6 +324,30 @@ fn convert_node(
fn validate_confidence(nodes: &[RecognitionNode]) -> Result<(), String> {
for node in nodes {
let image_count = node
.components
.iter()
.filter(|component| matches!(component, Component::Image(_)))
.count();
let text_count = node
.components
.iter()
.filter(|component| matches!(component, Component::Text(_)))
.count();
if image_count > 1 || text_count > 1 {
return Err("单个节点当前最多包含一个 Image 和一个 Text component".to_string());
}
if node.components.iter().any(|component| {
matches!(
component,
Component::Image(crate::ui_editor::component::image::ImageComponent {
target_graphic: Some(_),
..
})
)
}) {
return Err("识别阶段不能返回已绑定的 SpriteAssetId".to_string());
}
if let Confidence::UnSure(reason) = &node.confidence {
if reason.trim().is_empty() {
return Err("UnSure 必须包含审阅原因".to_string());
@@ -387,6 +412,7 @@ mod tests {
description: String::new(),
children: Vec::new(),
confidence: Confidence::Confident,
components: Vec::new(),
}
}
@@ -490,6 +516,7 @@ mod tests {
description: String::new(),
children: Vec::new(),
confidence: Confidence::UnSure(String::new()),
components: Vec::new(),
};
assert!(validate_confidence(&[node]).is_err());
}
File diff suppressed because it is too large Load Diff
@@ -196,7 +196,7 @@ pub(crate) fn generate_ui_design_code_at(
})
}
fn generated_file_stem(asset_id: &str) -> String {
pub(crate) fn generated_file_stem(asset_id: &str) -> String {
let mut stem = String::new();
for character in asset_id.chars() {
if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') {