diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model.rs index 16b617403..985e8fda3 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/model.rs @@ -42,12 +42,8 @@ pub struct SeparationNode { impl SeparationNode { pub fn as_prompt(&self) -> String { format!( - "node_id={} area=({}, {}, {}, {}) {}", + "node_id={} note: {}", self.id.as_str(), - self.global_pos_x_px, - self.global_pos_y_px, - self.width_px, - self.height_px, self.note.as_prompt() ) } diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs index d6df93e9f..ab32085b0 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt.rs @@ -33,7 +33,6 @@ pub(super) fn gen_extract_prompt(separation_notes: Vec) -> Strin pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String { let binding_system_prompt = format!( r#" - You are working under a UI elements separation workflow. You will be given a src UI design image and a processed image, where some ui elements are separated. Here were the separation requirements: ``` @@ -42,6 +41,10 @@ pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String { You need to recognize and review the separation using the given tool. field notes: * extracted_area MUST be the recognized area from the processed image, INSTEAD OF from the src image. + The processed image is the only authoritative image for extracted_area. + Return the pixel bounding box of the extracted element as it appears in the processed image. + Do not copy, infer, or reuse the source node rectangle. + The src image is only for identifying which semantic UI element belongs to to_node. these node need handle: "# diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs index 1081ff2fc..2c3a629bb 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/workflow.rs @@ -259,6 +259,7 @@ async fn visual_binding( e.to_string() })? .llm; + let client = build_game_creator_llm_client_from_llm_config(&llm_config, "llm").map_err(|e| { app_log!("ui_separation.error stage=visual_binding reason=build_client error={e}"); @@ -274,16 +275,23 @@ async fn visual_binding( schema, ) .with_strict(true); - let base_prompt = gen_binding_prompt(nodes.to_vec()); - let initial_history = vec![LlmMessage::user_multimodal(vec![ - LlmMessageContentPart::InputText { text: base_prompt }, - LlmMessageContentPart::InputImage { - image_url: source_url.clone(), - }, - LlmMessageContentPart::InputImage { - image_url: processed_url.clone(), - }, - ])]; + let initial_history = vec![ + LlmMessage::system(gen_binding_prompt(nodes.to_vec())), + LlmMessage::user_multimodal(vec![ + LlmMessageContentPart::InputText { + text: "processed image:".to_string(), + }, + LlmMessageContentPart::InputImage { + image_url: processed_url.clone(), + }, + LlmMessageContentPart::InputText { + text: "src image:".to_string(), + }, + LlmMessageContentPart::InputImage { + image_url: source_url.clone(), + }, + ]), + ]; let result = run_with_repair_history( 2, initial_history, @@ -518,13 +526,13 @@ pub(crate) async fn separate_ui_impl( for decision in &binding.decisions { if let BindingDecision::Ok { to_node, - extracted_area: separated_image_area, + extracted_area, } = decision { let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str())); match cut_processed_image( processed_path.clone(), - *separated_image_area, + *extracted_area, cut_path.clone(), ) .await