From 7c7d55dcabb8b79c031cd7bd1c9804b040df33a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BE=B7=E5=AE=87?= Date: Fri, 11 Sep 2026 14:54:00 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=8F=90=E5=8F=96=E6=8F=90?= =?UTF-8?q?=E7=A4=BA=E8=AF=8D=E7=94=9F=E6=88=90=E9=80=BB=E8=BE=91=EF=BC=8C?= =?UTF-8?q?=E7=A7=BB=E9=99=A4=E6=96=87=E6=9C=AC=E6=8B=BC=E6=8E=A5=E6=96=B9?= =?UTF-8?q?=E5=BC=8F=EF=BC=8C=E6=96=B0=E5=A2=9E=20YAML=20=E5=BA=8F?= =?UTF-8?q?=E5=88=97=E5=8C=96=E7=BB=93=E6=9E=84=E4=B8=8E=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/ui_editor/commands/separation/mod.rs | 31 --- .../commands/separation/prompt/extract.rs | 178 ++++++++++++------ ...案】UI编辑器自动切分素材工作流-2026-09-08.md | 1 + 3 files changed, 124 insertions(+), 86 deletions(-) diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs index 6b8545876..6e8a0f6fa 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/mod.rs @@ -252,37 +252,6 @@ mod tests { assert!(next_image_batch(&separation, &separation.trees[0]).is_empty()); } - #[test] - fn next_extract_prompt_contains_previous_rework_notes() { - let node = SeparationNode { - id: NodeId::new("image").unwrap(), - kind: SeparationNodeKind::ImageTarget, - global_pos_x_px: 0, - global_pos_y_px: 0, - width_px: 1, - height_px: 1, - note: SeparationNote { - description: "图标".to_string(), - rework_notes: vec!["不要带父背景".to_string()], - }, - children: vec![], - rework_count: 0, - }; - let tree = SeparationTree { - src_ui_design: UIDesignImageId::new("page").unwrap(), - root: node.clone(), - root_extractable: true, - }; - let separation = SeparationState { - schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(), - trees: vec![tree.clone()], - bound: vec![], - problematic_nodes: vec![], - }; - let prompt = super::prompt::gen_extract_prompt(&separation, &tree, &[&node]); - assert!(prompt.contains("previous rework notes:\n- 不要带父背景")); - } - #[test] fn binding_validation_rejects_overlong_rework_note() { let node = SeparationNode { diff --git a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/extract.rs b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/extract.rs index be839b24b..e0059e2b9 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/extract.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/ui_editor/commands/separation/prompt/extract.rs @@ -1,8 +1,45 @@ +use crate::ui_editor::commands::separation::model::{SeparationState, SeparationTree}; use crate::ui_editor::commands::separation::{SeparationNode, SeparationNodeKind}; +use crate::ui_editor::utils::NodeId; +use serde::Serialize; use std::collections::HashSet; -use crate::ui_editor::commands::separation::model::{SeparationState, SeparationTree}; -use crate::ui_editor::utils::NodeId; +#[derive(Serialize)] +struct ExtractPromptDocument { + ui_layer_tree: ExtractPromptNode, +} + +#[derive(Serialize)] +struct ExtractPromptNode { + index: usize, + status: ExtractPromptStatus, + rect: ExtractPromptRect, + description: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + rework_notes: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + children: Vec, +} + +#[derive(Serialize)] +enum ExtractPromptStatus { + #[serde(rename = "OUTPUT_THIS_TURN")] + OutputThisTurn, + #[serde(rename = "DONE")] + Done, + #[serde(rename = "CONTEXT_ONLY")] + ContextOnly, + #[serde(rename = "REMOVE_ONLY")] + RemoveOnly, +} + +#[derive(Serialize)] +struct ExtractPromptRect { + x: u32, + y: u32, + width: u32, + height: u32, +} pub(crate) fn gen_extract_prompt( state: &SeparationState, @@ -36,66 +73,97 @@ pub(crate) fn gen_extract_prompt( .map(|node| node.node_id.clone()), ) .collect::>(); - let mut lines = Vec::new(); - let mut index = 1usize; - append_context_lines( - &tree.root, - 0, - true, - &target_ids, - &terminal_ids, - &mut index, - &mut lines, - ); - result.push_str(&lines.join("\n")); + let mut index = 1; + let document = ExtractPromptDocument { + ui_layer_tree: project_node(&tree.root, &target_ids, &terminal_ids, &mut index), + }; + let yaml = serde_yaml::to_string(&document) + .expect("UI separation extract prompt projection must be serializable"); + + result.push_str("```yaml\n"); + result.push_str(&yaml); + result.push_str("```\n"); result } -fn append_context_lines( +fn project_node( node: &SeparationNode, - depth: usize, - is_root: bool, target_ids: &HashSet, terminal_ids: &HashSet, index: &mut usize, - lines: &mut Vec, -) { - let status = match node.kind { - SeparationNodeKind::TextRemovalOnly => "REMOVE_ONLY", - SeparationNodeKind::PureContainer => "CONTEXT_ONLY", - SeparationNodeKind::ImageTarget => { - if target_ids.contains(&node.id) { - "OUTPUT_THIS_TURN" - } else if terminal_ids.contains(&node.id) { - "DONE" - } else { - "CONTEXT_ONLY" - } - } - }; - let role = if is_root { "root" } else { "child" }; - lines.push(format!( - "{}. [{}] depth={} role={} rect=({}, {}, {}, {}) {}", - *index, - status, - depth, - role, - node.global_pos_x_px, - node.global_pos_y_px, - node.width_px, - node.height_px, - node.note.as_prompt() - )); +) -> ExtractPromptNode { + let current_index = *index; *index += 1; - for child in &node.children { - append_context_lines( - child, - depth + 1, - false, - target_ids, - terminal_ids, - index, - lines, - ); + + let status = match node.kind { + SeparationNodeKind::TextRemovalOnly => ExtractPromptStatus::RemoveOnly, + SeparationNodeKind::PureContainer => ExtractPromptStatus::ContextOnly, + SeparationNodeKind::ImageTarget if target_ids.contains(&node.id) => { + ExtractPromptStatus::OutputThisTurn + } + SeparationNodeKind::ImageTarget if terminal_ids.contains(&node.id) => { + ExtractPromptStatus::Done + } + SeparationNodeKind::ImageTarget => ExtractPromptStatus::ContextOnly, + }; + + let children = node + .children + .iter() + .map(|child| project_node(child, target_ids, terminal_ids, index)) + .collect(); + + ExtractPromptNode { + index: current_index, + status, + rect: ExtractPromptRect { + x: node.global_pos_x_px, + y: node.global_pos_y_px, + width: node.width_px, + height: node.height_px, + }, + description: node.note.description.clone(), + rework_notes: node.note.rework_notes.clone(), + children, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ui_editor::commands::separation::model::SeparationNote; + + #[test] + fn projects_source_node_to_prompt_view() { + let node = SeparationNode { + id: NodeId::new("image").unwrap(), + kind: SeparationNodeKind::ImageTarget, + global_pos_x_px: 1, + global_pos_y_px: 2, + width_px: 3, + height_px: 4, + note: SeparationNote { + description: "按钮".to_string(), + rework_notes: vec!["保留圆角".to_string()], + }, + children: vec![], + rework_count: 0, + }; + let mut index = 1; + let target_ids = HashSet::from([node.id.clone()]); + let projected = project_node(&node, &target_ids, &HashSet::new(), &mut index); + + assert_eq!(projected.index, 1); + assert!(matches!( + projected.status, + ExtractPromptStatus::OutputThisTurn + )); + assert_eq!(projected.rect.x, 1); + assert_eq!(projected.rect.y, 2); + assert_eq!(projected.rect.width, 3); + assert_eq!(projected.rect.height, 4); + assert_eq!(projected.description, "按钮"); + assert_eq!(projected.rework_notes, vec!["保留圆角".to_string()]); + assert!(projected.children.is_empty()); } } diff --git a/docs/technical/【技术方案】UI编辑器自动切分素材工作流-2026-09-08.md b/docs/technical/【技术方案】UI编辑器自动切分素材工作流-2026-09-08.md index de3f0c774..5c756464b 100644 --- a/docs/technical/【技术方案】UI编辑器自动切分素材工作流-2026-09-08.md +++ b/docs/technical/【技术方案】UI编辑器自动切分素材工作流-2026-09-08.md @@ -45,6 +45,7 @@ Rust 在接收并校验工具结果后，才把这两个枚举变体映射为正 - image-edit 直接使用原始 UI design PNG，不再生成或发送绿色框、紫色填充等额外辅助输入图。提取 prompt 直接描述完整页面分层清单和当前 batch 状态。 - 追加清单区分本轮 Image 输出目标、已完成 Image、仅作父子/遮挡上下文的 Image，以及只需从父图片移除的 Text。清单使用人类可读的编号、name/description、位置和层级，不向 image-edit 暴露 opaque NodeId。 +- 提取 prompt 先把 separation tree 投影为小型 YAML 视图，再注入固定规则文本：每个节点只包含展示编号、状态、`x/y/width/height` 矩形、描述、可选返工意见和递归 children；YAML 不携带 opaque NodeId，树的嵌套关系替代 `depth/role` 字段。 - 由于 raw endpoint 每次只返回一张 PNG，prompt 要求 image-edit 输出透明 atlas：本轮图片层可以移动和缩放，放置在不会互相遮挡的位置；视觉模型返回每层在 processed 图中的实际区域。源节点矩形只用于语义定位，不用于裁切区域推断。 - 请求尺寸始终使用源 UI design 尺寸；Raw GPT Image 2 API 保证返回相同尺寸，客户端不额外做尺寸拒绝检查。 - 处理图解码/写入和 cut 裁切属于本地 CPU/文件操作，放入独立的 `spawn_blocking` 任务；image-edit 与 visual binding 网络请求仍运行在 async future 中。