放开界面图识别根节点限制
允许非 Page 界面图作为独立识别上下文并生成 UI 树 保留 Page 与直接 slave 的同组识别关系 将语义建议的 LLM 校验失败统一包装为无效响应详情
This commit is contained in:
@@ -304,7 +304,7 @@ fn validate_tree_image_ids(
|
||||
let mut seen = HashSet::new();
|
||||
for tree in trees {
|
||||
if !allowed.contains(&tree.src_ui_design_image_id) {
|
||||
return Err("LLM 返回了不属于当前 Page 上下文的界面图 ID".to_string());
|
||||
return Err("LLM 返回了不属于当前识别上下文的界面图 ID".to_string());
|
||||
}
|
||||
if !seen.insert(tree.src_ui_design_image_id.clone()) {
|
||||
return Err("LLM 为同一界面图返回了重复 UI 树".to_string());
|
||||
@@ -312,7 +312,7 @@ fn validate_tree_image_ids(
|
||||
validate_confidence(&tree.children)?;
|
||||
}
|
||||
if seen.len() != allowed.len() {
|
||||
return Err("LLM 未为当前 Page 上下文的每张界面图返回 UI 树".to_string());
|
||||
return Err("LLM 未为当前识别上下文的每张界面图返回 UI 树".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -460,24 +460,31 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn page_image_ids(state: &State) -> Vec<UIDesignImageId> {
|
||||
fn recognition_root_image_ids(state: &State) -> Vec<UIDesignImageId> {
|
||||
state
|
||||
.ui_design_images
|
||||
.iter()
|
||||
.filter_map(|(id, image)| {
|
||||
(image.metadata.role
|
||||
== Some(crate::ui_editor::resource::ui_design_image::UIDesignImageRole::Page))
|
||||
.then(|| id.clone())
|
||||
let is_page = image.metadata.role
|
||||
== Some(crate::ui_editor::resource::ui_design_image::UIDesignImageRole::Page);
|
||||
let is_direct_slave_of_page = image.metadata.slave_to.as_ref().is_some_and(|parent| {
|
||||
state
|
||||
.ui_design_images
|
||||
.get(parent)
|
||||
.and_then(|parent_image| parent_image.metadata.role)
|
||||
== Some(crate::ui_editor::resource::ui_design_image::UIDesignImageRole::Page)
|
||||
});
|
||||
(is_page || !is_direct_slave_of_page).then(|| id.clone())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn slave_image_ids(state: &State, page_id: &UIDesignImageId) -> Vec<UIDesignImageId> {
|
||||
fn slave_image_ids(state: &State, root_id: &UIDesignImageId) -> Vec<UIDesignImageId> {
|
||||
state
|
||||
.ui_design_images
|
||||
.iter()
|
||||
.filter_map(|(id, image)| {
|
||||
(image.metadata.slave_to.as_ref() == Some(page_id)).then(|| id.clone())
|
||||
(image.metadata.slave_to.as_ref() == Some(root_id)).then(|| id.clone())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -497,10 +504,10 @@ pub(crate) async fn recognize_ui_impl(
|
||||
);
|
||||
return Err("界面图最多 4 张".to_string());
|
||||
}
|
||||
let page_ids = page_image_ids(&state);
|
||||
if page_ids.is_empty() {
|
||||
eprintln!("ui_recognition.error stage=validate reason=no_page");
|
||||
return Err("至少需要一张 role 为 Page 的界面图".to_string());
|
||||
let root_ids = recognition_root_image_ids(&state);
|
||||
if root_ids.is_empty() {
|
||||
eprintln!("ui_recognition.error stage=validate reason=no_root_image");
|
||||
return Err("至少需要一张可作为识别上下文根的界面图".to_string());
|
||||
}
|
||||
let client = build_game_creator_llm_client_from_config().map_err(|error| {
|
||||
eprintln!("ui_recognition.error stage=build_client error={error}");
|
||||
@@ -511,12 +518,12 @@ pub(crate) async fn recognize_ui_impl(
|
||||
error
|
||||
})?;
|
||||
let root = Path::new(project_path.trim());
|
||||
let mut ui_trees = Vec::with_capacity(page_ids.len());
|
||||
for page_id in page_ids {
|
||||
let mut context_ids = vec![page_id.clone()];
|
||||
context_ids.extend(slave_image_ids(&state, &page_id));
|
||||
// 当前只支持 Page 的直接 slave;嵌套 slave 关系的识别批次待后续设计。
|
||||
// TODO: 支持 slave_to 链的递归分组和独立识别。
|
||||
let mut ui_trees = Vec::with_capacity(root_ids.len());
|
||||
for root_id in root_ids {
|
||||
let mut context_ids = vec![root_id.clone()];
|
||||
context_ids.extend(slave_image_ids(&state, &root_id));
|
||||
// Page 仍可把直接 slave 参考图放在同一识别上下文;没有 Page
|
||||
// 归属关系的其它界面图各自作为独立上下文根。
|
||||
let mut parts = Vec::with_capacity(context_ids.len() * 2);
|
||||
for (index, context_id) in context_ids.iter().enumerate() {
|
||||
let image = state
|
||||
@@ -526,16 +533,16 @@ pub(crate) async fn recognize_ui_impl(
|
||||
let absolute =
|
||||
crate::project::resolve_local_project_path(root, &image.path).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=resolve_image page={} image={} error={error}",
|
||||
page_id.as_str(),
|
||||
"ui_recognition.error stage=resolve_image root={} image={} error={error}",
|
||||
root_id.as_str(),
|
||||
context_id.as_str()
|
||||
);
|
||||
error
|
||||
})?;
|
||||
let bytes = std::fs::read(&absolute).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=read_image page={} image={} error={error}",
|
||||
page_id.as_str(),
|
||||
"ui_recognition.error stage=read_image root={} image={} error={error}",
|
||||
root_id.as_str(),
|
||||
context_id.as_str()
|
||||
);
|
||||
format!("读取界面图失败:{error}")
|
||||
@@ -543,7 +550,7 @@ pub(crate) async fn recognize_ui_impl(
|
||||
parts.push(LlmMessageContentPart::InputText {
|
||||
text: format!(
|
||||
"{} id={} pixel_size={:?}",
|
||||
if index == 0 { "PAGE" } else { "SLAVE" },
|
||||
if index == 0 { "ROOT" } else { "SLAVE" },
|
||||
context_id.as_str(),
|
||||
image.pixel_size
|
||||
),
|
||||
@@ -554,7 +561,7 @@ pub(crate) async fn recognize_ui_impl(
|
||||
}
|
||||
let tool = LlmFunctionTool::new(
|
||||
"recognize_ui_structure",
|
||||
"识别当前 Page 及其直接 slave 参考图,并为每张输入图片各返回一棵 UI 树",
|
||||
"识别当前界面图上下文,并为每张输入图片各返回一棵 UI 树",
|
||||
schema.clone(),
|
||||
)
|
||||
.with_strict(true);
|
||||
@@ -570,14 +577,14 @@ pub(crate) async fn recognize_ui_impl(
|
||||
.await
|
||||
.map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=llm_request page={} error={error}",
|
||||
page_id.as_str()
|
||||
"ui_recognition.error stage=llm_request root={} error={error}",
|
||||
root_id.as_str()
|
||||
);
|
||||
format!("UI 结构识别失败(Page {}):{error}", page_id.as_str())
|
||||
format!("UI 结构识别失败(根界面图 {}):{error}", root_id.as_str())
|
||||
})?;
|
||||
eprintln!(
|
||||
"ui_recognition.llm_output page={} text={:?} tool_calls={:?}",
|
||||
page_id.as_str(),
|
||||
"ui_recognition.llm_output root={} text={:?} tool_calls={:?}",
|
||||
root_id.as_str(),
|
||||
response.text,
|
||||
response.tool_calls
|
||||
);
|
||||
@@ -587,26 +594,26 @@ pub(crate) async fn recognize_ui_impl(
|
||||
.find(|call| call.name == "recognize_ui_structure")
|
||||
.ok_or_else(|| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=parse_tool_call page={} reason=missing_tool_call",
|
||||
page_id.as_str()
|
||||
"ui_recognition.error stage=parse_tool_call root={} reason=missing_tool_call",
|
||||
root_id.as_str()
|
||||
);
|
||||
format!(
|
||||
"Page {} 的 LLM 未返回 recognize_ui_structure 工具调用",
|
||||
page_id.as_str()
|
||||
"根界面图 {} 的 LLM 未返回 recognize_ui_structure 工具调用",
|
||||
root_id.as_str()
|
||||
)
|
||||
})?;
|
||||
let parsed =
|
||||
serde_json::from_str::<RecognitionResponse>(&call.arguments).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=parse_arguments page={} error={error}",
|
||||
page_id.as_str()
|
||||
"ui_recognition.error stage=parse_arguments root={} error={error}",
|
||||
root_id.as_str()
|
||||
);
|
||||
format!("Page {} 的识别工具参数无效:{error}", page_id.as_str())
|
||||
format!("根界面图 {} 的识别工具参数无效:{error}", root_id.as_str())
|
||||
})?;
|
||||
validate_tree_image_ids(&parsed.trees, &context_ids).map_err(|error| {
|
||||
eprintln!(
|
||||
"ui_recognition.error stage=validate_trees page={} error={error}",
|
||||
page_id.as_str()
|
||||
"ui_recognition.error stage=validate_trees root={} error={error}",
|
||||
root_id.as_str()
|
||||
);
|
||||
error
|
||||
})?;
|
||||
|
||||
Reference in New Issue
Block a user