放开界面图识别根节点限制

允许非 Page 界面图作为独立识别上下文并生成 UI 树

保留 Page 与直接 slave 的同组识别关系

将语义建议的 LLM 校验失败统一包装为无效响应详情
This commit is contained in:
2026-08-18 13:53:40 +08:00
parent 117148816c
commit 8f4f07f05f
@@ -304,7 +304,7 @@ fn validate_tree_image_ids(
let mut seen = HashSet::new();
for tree in trees {
if !allowed.contains(&tree.src_ui_design_image_id) {
return Err("LLM 返回了不属于当前 Page 上下文的界面图 ID".to_string());
return Err("LLM 返回了不属于当前识别上下文的界面图 ID".to_string());
}
if !seen.insert(tree.src_ui_design_image_id.clone()) {
return Err("LLM 为同一界面图返回了重复 UI 树".to_string());
@@ -312,7 +312,7 @@ fn validate_tree_image_ids(
validate_confidence(&tree.children)?;
}
if seen.len() != allowed.len() {
return Err("LLM 未为当前 Page 上下文的每张界面图返回 UI 树".to_string());
return Err("LLM 未为当前识别上下文的每张界面图返回 UI 树".to_string());
}
Ok(())
}
@@ -460,24 +460,31 @@ mod tests {
}
}
fn page_image_ids(state: &State) -> Vec<UIDesignImageId> {
fn recognition_root_image_ids(state: &State) -> Vec<UIDesignImageId> {
state
.ui_design_images
.iter()
.filter_map(|(id, image)| {
(image.metadata.role
== Some(crate::ui_editor::resource::ui_design_image::UIDesignImageRole::Page))
.then(|| id.clone())
let is_page = image.metadata.role
== Some(crate::ui_editor::resource::ui_design_image::UIDesignImageRole::Page);
let is_direct_slave_of_page = image.metadata.slave_to.as_ref().is_some_and(|parent| {
state
.ui_design_images
.get(parent)
.and_then(|parent_image| parent_image.metadata.role)
== Some(crate::ui_editor::resource::ui_design_image::UIDesignImageRole::Page)
});
(is_page || !is_direct_slave_of_page).then(|| id.clone())
})
.collect()
}
fn slave_image_ids(state: &State, page_id: &UIDesignImageId) -> Vec<UIDesignImageId> {
fn slave_image_ids(state: &State, root_id: &UIDesignImageId) -> Vec<UIDesignImageId> {
state
.ui_design_images
.iter()
.filter_map(|(id, image)| {
(image.metadata.slave_to.as_ref() == Some(page_id)).then(|| id.clone())
(image.metadata.slave_to.as_ref() == Some(root_id)).then(|| id.clone())
})
.collect()
}
@@ -497,10 +504,10 @@ pub(crate) async fn recognize_ui_impl(
);
return Err("界面图最多 4 张".to_string());
}
let page_ids = page_image_ids(&state);
if page_ids.is_empty() {
eprintln!("ui_recognition.error stage=validate reason=no_page");
return Err("至少需要一张 role 为 Page 的界面图".to_string());
let root_ids = recognition_root_image_ids(&state);
if root_ids.is_empty() {
eprintln!("ui_recognition.error stage=validate reason=no_root_image");
return Err("至少需要一张可作为识别上下文根的界面图".to_string());
}
let client = build_game_creator_llm_client_from_config().map_err(|error| {
eprintln!("ui_recognition.error stage=build_client error={error}");
@@ -511,12 +518,12 @@ pub(crate) async fn recognize_ui_impl(
error
})?;
let root = Path::new(project_path.trim());
let mut ui_trees = Vec::with_capacity(page_ids.len());
for page_id in page_ids {
let mut context_ids = vec![page_id.clone()];
context_ids.extend(slave_image_ids(&state, &page_id));
// 当前只支持 Page 直接 slave;嵌套 slave 关系的识别批次待后续设计。
// TODO: 支持 slave_to 链的递归分组和独立识别
let mut ui_trees = Vec::with_capacity(root_ids.len());
for root_id in root_ids {
let mut context_ids = vec![root_id.clone()];
context_ids.extend(slave_image_ids(&state, &root_id));
// Page 仍可把直接 slave 参考图放在同一识别上下文;没有 Page
// 归属关系的其它界面图各自作为独立上下文根
let mut parts = Vec::with_capacity(context_ids.len() * 2);
for (index, context_id) in context_ids.iter().enumerate() {
let image = state
@@ -526,16 +533,16 @@ pub(crate) async fn recognize_ui_impl(
let absolute =
crate::project::resolve_local_project_path(root, &image.path).map_err(|error| {
eprintln!(
"ui_recognition.error stage=resolve_image page={} image={} error={error}",
page_id.as_str(),
"ui_recognition.error stage=resolve_image root={} image={} error={error}",
root_id.as_str(),
context_id.as_str()
);
error
})?;
let bytes = std::fs::read(&absolute).map_err(|error| {
eprintln!(
"ui_recognition.error stage=read_image page={} image={} error={error}",
page_id.as_str(),
"ui_recognition.error stage=read_image root={} image={} error={error}",
root_id.as_str(),
context_id.as_str()
);
format!("读取界面图失败:{error}")
@@ -543,7 +550,7 @@ pub(crate) async fn recognize_ui_impl(
parts.push(LlmMessageContentPart::InputText {
text: format!(
"{} id={} pixel_size={:?}",
if index == 0 { "PAGE" } else { "SLAVE" },
if index == 0 { "ROOT" } else { "SLAVE" },
context_id.as_str(),
image.pixel_size
),
@@ -554,7 +561,7 @@ pub(crate) async fn recognize_ui_impl(
}
let tool = LlmFunctionTool::new(
"recognize_ui_structure",
"识别当前 Page 及其直接 slave 参考图,并为每张输入图片各返回一棵 UI 树",
"识别当前界面图上下文,并为每张输入图片各返回一棵 UI 树",
schema.clone(),
)
.with_strict(true);
@@ -570,14 +577,14 @@ pub(crate) async fn recognize_ui_impl(
.await
.map_err(|error| {
eprintln!(
"ui_recognition.error stage=llm_request page={} error={error}",
page_id.as_str()
"ui_recognition.error stage=llm_request root={} error={error}",
root_id.as_str()
);
format!("UI 结构识别失败(Page {}):{error}", page_id.as_str())
format!("UI 结构识别失败(根界面图 {}):{error}", root_id.as_str())
})?;
eprintln!(
"ui_recognition.llm_output page={} text={:?} tool_calls={:?}",
page_id.as_str(),
"ui_recognition.llm_output root={} text={:?} tool_calls={:?}",
root_id.as_str(),
response.text,
response.tool_calls
);
@@ -587,26 +594,26 @@ pub(crate) async fn recognize_ui_impl(
.find(|call| call.name == "recognize_ui_structure")
.ok_or_else(|| {
eprintln!(
"ui_recognition.error stage=parse_tool_call page={} reason=missing_tool_call",
page_id.as_str()
"ui_recognition.error stage=parse_tool_call root={} reason=missing_tool_call",
root_id.as_str()
);
format!(
"Page {} 的 LLM 未返回 recognize_ui_structure 工具调用",
page_id.as_str()
"根界面图 {} 的 LLM 未返回 recognize_ui_structure 工具调用",
root_id.as_str()
)
})?;
let parsed =
serde_json::from_str::<RecognitionResponse>(&call.arguments).map_err(|error| {
eprintln!(
"ui_recognition.error stage=parse_arguments page={} error={error}",
page_id.as_str()
"ui_recognition.error stage=parse_arguments root={} error={error}",
root_id.as_str()
);
format!("Page {} 的识别工具参数无效:{error}", page_id.as_str())
format!("根界面图 {} 的识别工具参数无效:{error}", root_id.as_str())
})?;
validate_tree_image_ids(&parsed.trees, &context_ids).map_err(|error| {
eprintln!(
"ui_recognition.error stage=validate_trees page={} error={error}",
page_id.as_str()
"ui_recognition.error stage=validate_trees root={} error={error}",
root_id.as_str()
);
error
})?;