WIP: UI编辑器自动分图层切图标 #304

Draft
k88936 wants to merge 100 commits from feat/ui-editor-auto-seperation into master
10 changed files with 151 additions and 9 deletions
Showing only changes of commit 6e4f9a6f1f - Show all commits
@@ -116,13 +116,109 @@ mod tests {
height_px: 1,
note: SeparationNote {
description: "image".to_string(),
text_note: String::new(),
rework_notes: Vec::new(),
},
children: vec![],
rework_count: 0,
};
assert!(validate_binding_response(&BindingResp { decisions: vec![] }, &[&node]).is_err());
}
#[test]
fn separation_note_prompt_keeps_rework_notes_in_order() {
let without_notes = SeparationNote {
description: "按钮".to_string(),
rework_notes: Vec::new(),
};
assert_eq!(without_notes.as_prompt(), "desc: 按钮");
let with_notes = SeparationNote {
description: "按钮".to_string(),
rework_notes: vec!["保留圆角".to_string(), "去掉阴影".to_string()],
};
assert_eq!(
with_notes.as_prompt(),
"desc: 按钮\nprevious rework notes:\n- 保留圆角\n- 去掉阴影"
);
}
#[test]
fn need_rework_appends_note_and_final_attempt_becomes_problematic() {
let image = Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
preserve_aspect: false,
},
});
let mut separation = construct_separation_state(&state(node(
"root",
vec![],
vec![node("image", vec![image], vec![])],
)));
let id = NodeId::new("image").unwrap();
let paths = HashMap::new();
for note in ["第一次意见", "第二次意见", "最后一次意见"] {
apply_batch_patch(
&mut separation,
0,
&[BindingDecision::NeedRework {
to_node: id.clone(),
problem_description: note.to_string(),
}],
&paths,
)
.unwrap();
}
let node = &separation.trees[0].root.children[0];
assert_eq!(
node.note.rework_notes,
["第一次意见", "第二次意见", "最后一次意见"]
);
assert_eq!(separation.problematic_nodes.len(), 1);
assert_eq!(
separation.problematic_nodes[0].rework_count,
MAX_REWORK_COUNT
);
assert_eq!(node.rework_count, MAX_REWORK_COUNT);
assert!(next_leaf_batch(&separation, &separation.trees[0]).is_empty());
}
#[test]
fn next_extract_prompt_contains_previous_rework_notes() {
let note = SeparationNote {
description: "图标".to_string(),
rework_notes: vec!["不要带父背景".to_string()],
};
let prompt = super::prompt::gen_extract_prompt(vec![note]);
assert!(prompt.contains("previous rework notes:\n- 不要带父背景"));
}
#[test]
fn binding_validation_rejects_overlong_rework_note() {
let node = SeparationNode {
id: NodeId::new("image").unwrap(),
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
note: SeparationNote::default(),
children: vec![],
rework_count: 0,
};
let decision = BindingDecision::NeedRework {
to_node: node.id.clone(),
problem_description: "x".repeat(MAX_REWORK_NOTE_CHARS + 1),
};
assert!(validate_binding_response(
&BindingResp {
decisions: vec![decision]
},
&[&node]
)
.is_err());
}
#[test]
fn sidecar_name_uses_asset_id_digest() {
let dir = separation_sidecar_dir(Path::new("/tmp/project"), "ui:1").unwrap();
@@ -5,16 +5,25 @@ use ts_rs::TS;
pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1";
pub const MAX_REWORK_COUNT: u32 = 3;
pub const MAX_REWORK_NOTE_CHARS: usize = 512;
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationNote {
pub description: String,
pub text_note: String,
pub rework_notes: Vec<String>,
}
impl SeparationNote {
pub fn as_prompt(&self) -> String {
format!("desc: {} {}", self.description, self.text_note)
let mut prompt = format!("desc: {}", self.description);
if !self.rework_notes.is_empty() {
prompt.push_str("\nprevious rework notes:");
for note in &self.rework_notes {
prompt.push_str("\n- ");
prompt.push_str(note);
}
}
prompt
}
}
@@ -63,7 +63,7 @@ fn collect_todo_nodes(
height_px: h,
note: SeparationNote {
description: node_description(node),
text_note: String::new(),
rework_notes: Vec::new(),
},
children,
rework_count: 0,
@@ -102,7 +102,7 @@ pub fn construct_separation_state(state: &State) -> SeparationState {
height_px: h,
note: SeparationNote {
description: node_description(&tree.root),
text_note: String::new(),
rework_notes: Vec::new(),
},
children,
rework_count: 0,
@@ -217,6 +217,11 @@ pub fn validate_binding_response(
if problem_description.trim().is_empty() {
return Err("NeedRework 必须包含问题描述".to_string());
}
if problem_description.chars().count() > MAX_REWORK_NOTE_CHARS {
return Err(format!(
"NeedRework 问题描述不能超过 {MAX_REWORK_NOTE_CHARS} 个字符"
));
}
}
}
if seen.len() != expected.len() {
@@ -66,15 +66,15 @@ pub fn apply_batch_patch(
to_node,
problem_description,
} => {
append_rework_note(&mut tree.root, to_node, problem_description);
let count = rework_counts.get(to_node).copied().unwrap_or(0) + 1;
increment_rework_count(&mut tree.root, to_node, count);
if count >= MAX_REWORK_COUNT {
state.problematic_nodes.push(ProblematicNode {
node_id: to_node.clone(),
problem_description: problem_description.clone(),
rework_count: count,
});
} else {
increment_rework_count(&mut tree.root, to_node, count);
}
}
}
@@ -89,6 +89,16 @@ pub fn apply_batch_patch(
Ok(())
}
fn append_rework_note(node: &mut SeparationNode, id: &NodeId, note: &str) -> bool {
if node.id == *id {
node.note.rework_notes.push(note.to_string());
return true;
}
node.children
.iter_mut()
.any(|child| append_rework_note(child, id, note))
}
fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) {
if node.id == *id {
node.rework_count = count;
@@ -512,7 +522,7 @@ pub(crate) async fn separate_ui_impl(
for decision in &binding.decisions {
if let BindingDecision::Ok {
to_node,
extracted_area: separated_image_area,
extracted_area: separated_image_area,
} = decision
{
let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str()));
@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { NodeId } from "./NodeId";
export type BoundNode = { node_id: NodeId, cut_image_path: string, };
@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { NodeId } from "./NodeId";
export type ProblematicNode = { node_id: NodeId, problem_description: string, rework_count: number, };
@@ -0,0 +1,5 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { BoundNode } from "./BoundNode";
import type { ProblematicNode } from "./ProblematicNode";
export type SeparationDTO = { bound_nodes: Array<BoundNode>, problematic_nodes: Array<ProblematicNode>, };
@@ -0,0 +1,5 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { NodeId } from "./NodeId";
import type { SeparationNote } from "./SeparationNote";
export type SeparationNode = { id: NodeId, global_pos_x_px: number, global_pos_y_px: number, width_px: number, height_px: number, note: SeparationNote, children: Array<SeparationNode>, rework_count: number, };
@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type SeparationNote = { description: string, rework_notes: Array<string>, };
@@ -35,7 +35,8 @@
`spawn_blocking` 任务;image-edit 与 visual binding 网络请求仍运行在 async future 中。
- 视觉 binding 输入源图与处理图,必须为当前 batch 每个节点恰好返回一次 `Ok``NeedRework`
- `Ok` 返回 `NodeId + BindingArea`Rust 仅校验 NodeId、区域边界和非零尺寸,不检查与原节点框的偏差,也不要求区域不重叠。
- `NeedRework` 携带短问题描述。结构化工具调用失败时使用可复用 repair harness,把错误反馈给模型并额外请求一次;image-edit 不使用该 harness。
- `NeedRework` 携带短问题描述(最多 512 个 Unicode 字符);通过校验后按产生顺序追加到目标 `SeparationNode.note.rework_notes`,下一次该节点进入 image-edit 时全部意见会注入提取 prompt。结构化工具调用失败时使用可复用 repair harness,把错误反馈给模型并额外请求一次;image-edit 不使用该 harness。
- 达到返工上限时仍先保留最后一条视觉模型意见,再把节点追加到 problematic;网络、IO、裁切等基础设施错误不写入节点意见。
- 达到模块级重做常量后,节点移入 problematic;不中断整条工作流,最终统一通知用户。image-edit、图像写入或裁切失败保留当前 state 并返回错误,不自动把整批标记为 problematic。
- 父节点背景重建由 image-edit 模型完成,不由 Rust 硬编码重建算法完成。