完善 UI 自动分离返工意见链路

记录视觉模型 NeedRework 意见并注入后续提取提示同步返工次数、problematic 终态和 512 字符校验补充 separation DTO 类型、回归测试与技术方案
2026-09-09 16:29:54 +08:00
parent 8a0dcf8be7
commit 6e4f9a6f1f
10 changed files with 151 additions and 9 deletions
@@ -116,13 +116,109 @@ mod tests {
            height_px: 1,
            note: SeparationNote {
                description: "image".to_string(),
-                text_note: String::new(),
+                rework_notes: Vec::new(),
            },
            children: vec![],
            rework_count: 0,
        };
        assert!(validate_binding_response(&BindingResp { decisions: vec![] }, &[&node]).is_err());
    }
+
+    #[test]
+    fn separation_note_prompt_keeps_rework_notes_in_order() {
+        let without_notes = SeparationNote {
+            description: "按钮".to_string(),
+            rework_notes: Vec::new(),
+        };
+        assert_eq!(without_notes.as_prompt(), "desc: 按钮");
+
+        let with_notes = SeparationNote {
+            description: "按钮".to_string(),
+            rework_notes: vec!["保留圆角".to_string(), "去掉阴影".to_string()],
+        };
+        assert_eq!(
+            with_notes.as_prompt(),
+            "desc: 按钮\nprevious rework notes:\n- 保留圆角\n- 去掉阴影"
+        );
+    }
+
+    #[test]
+    fn need_rework_appends_note_and_final_attempt_becomes_problematic() {
+        let image = Component::Image(ImageComponent {
+            target_graphic: None,
+            image_type: ImageType::Simple {
+                preserve_aspect: false,
+            },
+        });
+        let mut separation = construct_separation_state(&state(node(
+            "root",
+            vec![],
+            vec![node("image", vec![image], vec![])],
+        )));
+        let id = NodeId::new("image").unwrap();
+        let paths = HashMap::new();
+
+        for note in ["第一次意见", "第二次意见", "最后一次意见"] {
+            apply_batch_patch(
+                &mut separation,
+                0,
+                &[BindingDecision::NeedRework {
+                    to_node: id.clone(),
+                    problem_description: note.to_string(),
+                }],
+                &paths,
+            )
+            .unwrap();
+        }
+
+        let node = &separation.trees[0].root.children[0];
+        assert_eq!(
+            node.note.rework_notes,
+            ["第一次意见", "第二次意见", "最后一次意见"]
+        );
+        assert_eq!(separation.problematic_nodes.len(), 1);
+        assert_eq!(
+            separation.problematic_nodes[0].rework_count,
+            MAX_REWORK_COUNT
+        );
+        assert_eq!(node.rework_count, MAX_REWORK_COUNT);
+        assert!(next_leaf_batch(&separation, &separation.trees[0]).is_empty());
+    }
+
+    #[test]
+    fn next_extract_prompt_contains_previous_rework_notes() {
+        let note = SeparationNote {
+            description: "图标".to_string(),
+            rework_notes: vec!["不要带父背景".to_string()],
+        };
+        let prompt = super::prompt::gen_extract_prompt(vec![note]);
+        assert!(prompt.contains("previous rework notes:\n- 不要带父背景"));
+    }
+
+    #[test]
+    fn binding_validation_rejects_overlong_rework_note() {
+        let node = SeparationNode {
+            id: NodeId::new("image").unwrap(),
+            global_pos_x_px: 0,
+            global_pos_y_px: 0,
+            width_px: 1,
+            height_px: 1,
+            note: SeparationNote::default(),
+            children: vec![],
+            rework_count: 0,
+        };
+        let decision = BindingDecision::NeedRework {
+            to_node: node.id.clone(),
+            problem_description: "x".repeat(MAX_REWORK_NOTE_CHARS + 1),
+        };
+        assert!(validate_binding_response(
+            &BindingResp {
+                decisions: vec![decision]
+            },
+            &[&node]
+        )
+        .is_err());
+    }
    #[test]
    fn sidecar_name_uses_asset_id_digest() {
        let dir = separation_sidecar_dir(Path::new("/tmp/project"), "ui:1").unwrap();
@@ -5,16 +5,25 @@ use ts_rs::TS;

 pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1";
 pub const MAX_REWORK_COUNT: u32 = 3;
+pub const MAX_REWORK_NOTE_CHARS: usize = 512;

 #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)]
 #[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
 pub struct SeparationNote {
    pub description: String,
-    pub text_note: String,
+    pub rework_notes: Vec<String>,
 }
 impl SeparationNote {
    pub fn as_prompt(&self) -> String {
-        format!("desc: {} {}", self.description, self.text_note)
+        let mut prompt = format!("desc: {}", self.description);
+        if !self.rework_notes.is_empty() {
+            prompt.push_str("\nprevious rework notes:");
+            for note in &self.rework_notes {
+                prompt.push_str("\n- ");
+                prompt.push_str(note);
+            }
+        }
+        prompt
    }
 }

@@ -63,7 +63,7 @@ fn collect_todo_nodes(
            height_px: h,
            note: SeparationNote {
                description: node_description(node),
-                text_note: String::new(),
+                rework_notes: Vec::new(),
            },
            children,
            rework_count: 0,
@@ -102,7 +102,7 @@ pub fn construct_separation_state(state: &State) -> SeparationState {
                    height_px: h,
                    note: SeparationNote {
                        description: node_description(&tree.root),
-                        text_note: String::new(),
+                        rework_notes: Vec::new(),
                    },
                    children,
                    rework_count: 0,
@@ -217,6 +217,11 @@ pub fn validate_binding_response(
            if problem_description.trim().is_empty() {
                return Err("NeedRework 必须包含问题描述".to_string());
            }
+            if problem_description.chars().count() > MAX_REWORK_NOTE_CHARS {
+                return Err(format!(
+                    "NeedRework 问题描述不能超过 {MAX_REWORK_NOTE_CHARS} 个字符"
+                ));
+            }
        }
    }
    if seen.len() != expected.len() {
@@ -66,15 +66,15 @@ pub fn apply_batch_patch(
                to_node,
                problem_description,
            } => {
+                append_rework_note(&mut tree.root, to_node, problem_description);
                let count = rework_counts.get(to_node).copied().unwrap_or(0) + 1;
+                increment_rework_count(&mut tree.root, to_node, count);
                if count >= MAX_REWORK_COUNT {
                    state.problematic_nodes.push(ProblematicNode {
                        node_id: to_node.clone(),
                        problem_description: problem_description.clone(),
                        rework_count: count,
                    });
-                } else {
-                    increment_rework_count(&mut tree.root, to_node, count);
                }
            }
        }
@@ -89,6 +89,16 @@ pub fn apply_batch_patch(
    Ok(())
 }

+fn append_rework_note(node: &mut SeparationNode, id: &NodeId, note: &str) -> bool {
+    if node.id == *id {
+        node.note.rework_notes.push(note.to_string());
+        return true;
+    }
+    node.children
+        .iter_mut()
+        .any(|child| append_rework_note(child, id, note))
+}
+
 fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) {
    if node.id == *id {
        node.rework_count = count;
@@ -512,7 +522,7 @@ pub(crate) async fn separate_ui_impl(
            for decision in &binding.decisions {
                if let BindingDecision::Ok {
                    to_node,
-                  extracted_area: separated_image_area,
+                    extracted_area: separated_image_area,
                } = decision
                {
                    let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str()));
@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { NodeId } from "./NodeId";
+
+export type BoundNode = { node_id: NodeId, cut_image_path: string, };
@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { NodeId } from "./NodeId";
+
+export type ProblematicNode = { node_id: NodeId, problem_description: string, rework_count: number, };
@@ -0,0 +1,5 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { BoundNode } from "./BoundNode";
+import type { ProblematicNode } from "./ProblematicNode";
+
+export type SeparationDTO = { bound_nodes: Array<BoundNode>, problematic_nodes: Array<ProblematicNode>, };
@@ -0,0 +1,5 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { NodeId } from "./NodeId";
+import type { SeparationNote } from "./SeparationNote";
+
+export type SeparationNode = { id: NodeId, global_pos_x_px: number, global_pos_y_px: number, width_px: number, height_px: number, note: SeparationNote, children: Array<SeparationNode>, rework_count: number, };
@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type SeparationNote = { description: string, rework_notes: Array<string>, };
@@ -35,7 +35,8 @@
  `spawn_blocking` 任务；image-edit 与 visual binding 网络请求仍运行在 async future 中。
 - 视觉 binding 输入源图与处理图，必须为当前 batch 每个节点恰好返回一次 `Ok` 或 `NeedRework`。
 - `Ok` 返回 `NodeId + BindingArea`；Rust 仅校验 NodeId、区域边界和非零尺寸，不检查与原节点框的偏差，也不要求区域不重叠。
- `NeedRework` 携带短问题描述。结构化工具调用失败时使用可复用 repair harness，把错误反馈给模型并额外请求一次；image-edit 不使用该 harness。
+- `NeedRework` 携带短问题描述（最多 512 个 Unicode 字符）；通过校验后按产生顺序追加到目标 `SeparationNode.note.rework_notes`，下一次该节点进入 image-edit 时全部意见会注入提取 prompt。结构化工具调用失败时使用可复用 repair harness，把错误反馈给模型并额外请求一次；image-edit 不使用该 harness。
+- 达到返工上限时仍先保留最后一条视觉模型意见，再把节点追加到 problematic；网络、IO、裁切等基础设施错误不写入节点意见。
 - 达到模块级重做常量后，节点移入 problematic；不中断整条工作流，最终统一通知用户。image-edit、图像写入或裁切失败保留当前 state 并返回错误，不自动把整批标记为 problematic。
 - 父节点背景重建由 image-edit 模型完成，不由 Rust 硬编码重建算法完成。