优化 UI 分离工作流程的提示生成与历史处理
- 调整 initial_history 构建逻辑,增强系统与用户消息的一致性。 - 简化 SeparationNode 提示生成逻辑,仅保留节点 ID 与注释。 - 明确 extracted_area 的权威性为 processed 图像,并更新生成提示的规则。
This commit is contained in:
@@ -42,12 +42,8 @@ pub struct SeparationNode {
|
||||
impl SeparationNode {
|
||||
pub fn as_prompt(&self) -> String {
|
||||
format!(
|
||||
"node_id={} area=({}, {}, {}, {}) {}",
|
||||
"node_id={} note: {}",
|
||||
self.id.as_str(),
|
||||
self.global_pos_x_px,
|
||||
self.global_pos_y_px,
|
||||
self.width_px,
|
||||
self.height_px,
|
||||
self.note.as_prompt()
|
||||
)
|
||||
}
|
||||
|
||||
@@ -33,7 +33,6 @@ pub(super) fn gen_extract_prompt(separation_notes: Vec<SeparationNote>) -> Strin
|
||||
pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String {
|
||||
let binding_system_prompt = format!(
|
||||
r#"
|
||||
You are working under a UI elements separation workflow.
|
||||
You will be given a src UI design image and a processed image, where some ui elements are separated.
|
||||
Here were the separation requirements:
|
||||
```
|
||||
@@ -42,6 +41,10 @@ pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String {
|
||||
You need to recognize and review the separation using the given tool.
|
||||
field notes:
|
||||
* extracted_area MUST be the recognized area from the processed image, INSTEAD OF from the src image.
|
||||
The processed image is the only authoritative image for extracted_area.
|
||||
Return the pixel bounding box of the extracted element as it appears in the processed image.
|
||||
Do not copy, infer, or reuse the source node rectangle.
|
||||
The src image is only for identifying which semantic UI element belongs to to_node.
|
||||
|
||||
these node need handle:
|
||||
"#
|
||||
|
||||
+20
-12
@@ -259,6 +259,7 @@ async fn visual_binding(
|
||||
e.to_string()
|
||||
})?
|
||||
.llm;
|
||||
|
||||
let client =
|
||||
build_game_creator_llm_client_from_llm_config(&llm_config, "llm").map_err(|e| {
|
||||
app_log!("ui_separation.error stage=visual_binding reason=build_client error={e}");
|
||||
@@ -274,16 +275,23 @@ async fn visual_binding(
|
||||
schema,
|
||||
)
|
||||
.with_strict(true);
|
||||
let base_prompt = gen_binding_prompt(nodes.to_vec());
|
||||
let initial_history = vec![LlmMessage::user_multimodal(vec![
|
||||
LlmMessageContentPart::InputText { text: base_prompt },
|
||||
LlmMessageContentPart::InputImage {
|
||||
image_url: source_url.clone(),
|
||||
},
|
||||
LlmMessageContentPart::InputImage {
|
||||
image_url: processed_url.clone(),
|
||||
},
|
||||
])];
|
||||
let initial_history = vec![
|
||||
LlmMessage::system(gen_binding_prompt(nodes.to_vec())),
|
||||
LlmMessage::user_multimodal(vec![
|
||||
LlmMessageContentPart::InputText {
|
||||
text: "processed image:".to_string(),
|
||||
},
|
||||
LlmMessageContentPart::InputImage {
|
||||
image_url: processed_url.clone(),
|
||||
},
|
||||
LlmMessageContentPart::InputText {
|
||||
text: "src image:".to_string(),
|
||||
},
|
||||
LlmMessageContentPart::InputImage {
|
||||
image_url: source_url.clone(),
|
||||
},
|
||||
]),
|
||||
];
|
||||
let result = run_with_repair_history(
|
||||
2,
|
||||
initial_history,
|
||||
@@ -518,13 +526,13 @@ pub(crate) async fn separate_ui_impl(
|
||||
for decision in &binding.decisions {
|
||||
if let BindingDecision::Ok {
|
||||
to_node,
|
||||
extracted_area: separated_image_area,
|
||||
extracted_area,
|
||||
} = decision
|
||||
{
|
||||
let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str()));
|
||||
match cut_processed_image(
|
||||
processed_path.clone(),
|
||||
*separated_image_area,
|
||||
*extracted_area,
|
||||
cut_path.clone(),
|
||||
)
|
||||
.await
|
||||
|
||||
Reference in New Issue
Block a user