重构UI素材切分树与批次选择
移除文本遮罩和标记图生成流程,图片编辑直接读取原始界面图 引入节点类型并保留文字移除上下文,按前序DFS与面积预算选择图片批次 补充根节点、重叠节点、文字过滤和超面积推进测试并同步TypeScript类型
This commit is contained in:
@@ -1,364 +0,0 @@
|
||||
use super::model::SeparationNode;
|
||||
use base64::Engine as _;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Instant;
|
||||
|
||||
const MARKER_LINE_WIDTH: u32 = 2;
|
||||
const PURPLE_FILL: image::Rgba<u8> = image::Rgba([180, 0, 180, 120]);
|
||||
|
||||
pub async fn build_marked_image(
|
||||
source_url: String,
|
||||
nodes: Vec<SeparationNode>,
|
||||
target: PathBuf,
|
||||
) -> Result<String, String> {
|
||||
let node_count = nodes.len();
|
||||
let started = Instant::now();
|
||||
let result = match tokio::task::spawn_blocking(move || {
|
||||
let blocking_started = Instant::now();
|
||||
let result = build_marked_image_blocking(&source_url, &nodes, &target);
|
||||
app_log!(
|
||||
"ui_separation.marker.blocking_timing outcome={} elapsed_ms={} nodes={}",
|
||||
if result.is_ok() { "ok" } else { "error" },
|
||||
blocking_started.elapsed().as_millis(),
|
||||
node_count
|
||||
);
|
||||
result
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(error) => Err(format!("构建标记图任务失败:{error}")),
|
||||
};
|
||||
app_log!(
|
||||
"ui_separation.marker.timing outcome={} elapsed_ms={} nodes={}",
|
||||
if result.is_ok() { "ok" } else { "error" },
|
||||
started.elapsed().as_millis(),
|
||||
node_count
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
fn build_marked_image_blocking(
|
||||
source_url: &str,
|
||||
nodes: &[SeparationNode],
|
||||
target: &Path,
|
||||
) -> Result<String, String> {
|
||||
let encoded = source_url
|
||||
.split_once(',')
|
||||
.map(|(_, d)| d)
|
||||
.ok_or_else(|| "源图 data URL 无效".to_string())?;
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(encoded)
|
||||
.map_err(|e| format!("解码源图失败:{e}"))?;
|
||||
let mut image = image::load_from_memory(&bytes)
|
||||
.map_err(|e| format!("读取源图失败:{e}"))?
|
||||
.to_rgba8();
|
||||
let width = image.width();
|
||||
let height = image.height();
|
||||
// 先填充紫色重建区域,随后绘制绿色框,确保绿色框位于最上层。
|
||||
for node in nodes {
|
||||
for child in &node.children {
|
||||
fill_rect(
|
||||
&mut image,
|
||||
child.global_pos_x_px,
|
||||
child.global_pos_y_px,
|
||||
child.width_px,
|
||||
child.height_px,
|
||||
PURPLE_FILL,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
}
|
||||
for mask in &node.text_mask_areas {
|
||||
fill_rect(
|
||||
&mut image,
|
||||
mask.global_pos_x_px,
|
||||
mask.global_pos_y_px,
|
||||
mask.width_px,
|
||||
mask.height_px,
|
||||
PURPLE_FILL,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
}
|
||||
}
|
||||
for node in nodes {
|
||||
draw_frame(
|
||||
&mut image,
|
||||
node.global_pos_x_px,
|
||||
node.global_pos_y_px,
|
||||
node.width_px,
|
||||
node.height_px,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
}
|
||||
let image = image::DynamicImage::ImageRgba8(image);
|
||||
image
|
||||
.save_with_format(target, image::ImageFormat::Png)
|
||||
.map_err(|e| format!("写入标记图失败:{e}"))?;
|
||||
let mut png = Vec::new();
|
||||
image
|
||||
.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
|
||||
.map_err(|e| format!("编码标记图失败:{e}"))?;
|
||||
Ok(format!(
|
||||
"data:image/png;base64,{}",
|
||||
base64::engine::general_purpose::STANDARD.encode(png)
|
||||
))
|
||||
}
|
||||
|
||||
fn clipped_rect(
|
||||
x: u32,
|
||||
y: u32,
|
||||
w: u32,
|
||||
h: u32,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Option<(u32, u32, u32, u32)> {
|
||||
if width == 0 || height == 0 || w == 0 || h == 0 {
|
||||
return None;
|
||||
}
|
||||
if x >= width || y >= height {
|
||||
return None;
|
||||
}
|
||||
let x0 = x;
|
||||
let y0 = y;
|
||||
let x1 = x.saturating_add(w).min(width).saturating_sub(1);
|
||||
let y1 = y.saturating_add(h).min(height).saturating_sub(1);
|
||||
(x0 <= x1 && y0 <= y1).then_some((x0, y0, x1, y1))
|
||||
}
|
||||
|
||||
fn fill_rect(
|
||||
image: &mut image::RgbaImage,
|
||||
x: u32,
|
||||
y: u32,
|
||||
w: u32,
|
||||
h: u32,
|
||||
color: image::Rgba<u8>,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) {
|
||||
let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else {
|
||||
return;
|
||||
};
|
||||
for yy in y0..=y1 {
|
||||
for xx in x0..=x1 {
|
||||
image.put_pixel(xx, yy, color);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn draw_frame(
|
||||
image: &mut image::RgbaImage,
|
||||
x: u32,
|
||||
y: u32,
|
||||
w: u32,
|
||||
h: u32,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) {
|
||||
let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else {
|
||||
return;
|
||||
};
|
||||
let green = image::Rgba([0, 255, 0, 255]);
|
||||
draw_line(
|
||||
image,
|
||||
(x0, y0),
|
||||
(x1, y0),
|
||||
green,
|
||||
MARKER_LINE_WIDTH,
|
||||
(x0, y0, x1, y1),
|
||||
);
|
||||
draw_line(
|
||||
image,
|
||||
(x0, y1),
|
||||
(x1, y1),
|
||||
green,
|
||||
MARKER_LINE_WIDTH,
|
||||
(x0, y0, x1, y1),
|
||||
);
|
||||
draw_line(
|
||||
image,
|
||||
(x0, y0),
|
||||
(x0, y1),
|
||||
green,
|
||||
MARKER_LINE_WIDTH,
|
||||
(x0, y0, x1, y1),
|
||||
);
|
||||
draw_line(
|
||||
image,
|
||||
(x1, y0),
|
||||
(x1, y1),
|
||||
green,
|
||||
MARKER_LINE_WIDTH,
|
||||
(x0, y0, x1, y1),
|
||||
);
|
||||
draw_line(
|
||||
image,
|
||||
(x0, y0),
|
||||
(x1, y1),
|
||||
green,
|
||||
MARKER_LINE_WIDTH,
|
||||
(x0, y0, x1, y1),
|
||||
);
|
||||
draw_line(
|
||||
image,
|
||||
(x1, y0),
|
||||
(x0, y1),
|
||||
green,
|
||||
MARKER_LINE_WIDTH,
|
||||
(x0, y0, x1, y1),
|
||||
);
|
||||
}
|
||||
|
||||
fn draw_line(
|
||||
image: &mut image::RgbaImage,
|
||||
start: (u32, u32),
|
||||
end: (u32, u32),
|
||||
color: image::Rgba<u8>,
|
||||
line_width: u32,
|
||||
bounds: (u32, u32, u32, u32),
|
||||
) {
|
||||
let mut x = start.0 as i64;
|
||||
let mut y = start.1 as i64;
|
||||
let target_x = end.0 as i64;
|
||||
let target_y = end.1 as i64;
|
||||
let dx = (target_x - x).abs();
|
||||
let sx = if x < target_x { 1 } else { -1 };
|
||||
let dy = -(target_y - y).abs();
|
||||
let sy = if y < target_y { 1 } else { -1 };
|
||||
let mut error = dx + dy;
|
||||
|
||||
loop {
|
||||
draw_brush(image, x, y, color, line_width, bounds);
|
||||
if x == target_x && y == target_y {
|
||||
break;
|
||||
}
|
||||
let twice_error = error * 2;
|
||||
if twice_error >= dy {
|
||||
error += dy;
|
||||
x += sx;
|
||||
}
|
||||
if twice_error <= dx {
|
||||
error += dx;
|
||||
y += sy;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn draw_brush(
|
||||
image: &mut image::RgbaImage,
|
||||
x: i64,
|
||||
y: i64,
|
||||
color: image::Rgba<u8>,
|
||||
line_width: u32,
|
||||
bounds: (u32, u32, u32, u32),
|
||||
) {
|
||||
let (x0, y0, x1, y1) = bounds;
|
||||
let line_width = line_width.max(1) as i64;
|
||||
let before = (line_width - 1) / 2;
|
||||
let after = line_width / 2;
|
||||
let min_x = (x - before).max(x0 as i64);
|
||||
let max_x = (x + after).min(x1 as i64);
|
||||
let min_y = (y - before).max(y0 as i64);
|
||||
let max_y = (y + after).min(y1 as i64);
|
||||
for yy in min_y..=max_y {
|
||||
for xx in min_x..=max_x {
|
||||
image.put_pixel(xx as u32, yy as u32, color);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ui_editor::commands::separation::{SeparationNote, TextMaskArea};
|
||||
|
||||
#[test]
|
||||
fn draw_frame_adds_green_cross_corner_lines() {
|
||||
let mut image = image::RgbaImage::from_pixel(8, 6, image::Rgba([1, 2, 3, 255]));
|
||||
draw_frame(&mut image, 1, 1, 5, 3, 8, 6);
|
||||
let green = image::Rgba([0, 255, 0, 255]);
|
||||
|
||||
for &(x, y) in &[(1, 1), (5, 1), (1, 3), (5, 3), (3, 2)] {
|
||||
assert_eq!(*image.get_pixel(x, y), green, "pixel ({x}, {y})");
|
||||
}
|
||||
assert_eq!(*image.get_pixel(3, 1), green);
|
||||
assert_eq!(*image.get_pixel(3, 3), green);
|
||||
assert_eq!(*image.get_pixel(2, 2), green);
|
||||
assert_eq!(*image.get_pixel(4, 2), green);
|
||||
assert_eq!(*image.get_pixel(0, 0), image::Rgba([1, 2, 3, 255]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn draw_frame_keeps_cross_inside_clipped_rect() {
|
||||
let mut image = image::RgbaImage::from_pixel(4, 4, image::Rgba([1, 2, 3, 255]));
|
||||
draw_frame(&mut image, 2, 2, 4, 4, 4, 4);
|
||||
let green = image::Rgba([0, 255, 0, 255]);
|
||||
for y in 2..4 {
|
||||
for x in 2..4 {
|
||||
assert_eq!(*image.get_pixel(x, y), green);
|
||||
}
|
||||
}
|
||||
assert_eq!(*image.get_pixel(1, 1), image::Rgba([1, 2, 3, 255]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clipped_rect_ignores_rectangles_starting_outside_image() {
|
||||
assert_eq!(clipped_rect(4, 0, 1, 1, 4, 4), None);
|
||||
assert_eq!(clipped_rect(0, 4, 1, 1, 4, 4), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_mask_is_purple_before_green_frame() {
|
||||
let mut image = image::RgbaImage::from_pixel(8, 8, image::Rgba([1, 2, 3, 255]));
|
||||
let node = SeparationNode {
|
||||
id: crate::ui_editor::utils::NodeId::new("image").unwrap(),
|
||||
global_pos_x_px: 1,
|
||||
global_pos_y_px: 1,
|
||||
width_px: 6,
|
||||
height_px: 6,
|
||||
note: SeparationNote::default(),
|
||||
text_mask_areas: vec![
|
||||
TextMaskArea {
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 1,
|
||||
height_px: 1,
|
||||
},
|
||||
TextMaskArea {
|
||||
global_pos_x_px: 1,
|
||||
global_pos_y_px: 1,
|
||||
width_px: 1,
|
||||
height_px: 1,
|
||||
},
|
||||
],
|
||||
children: vec![],
|
||||
rework_count: 0,
|
||||
};
|
||||
for mask in &node.text_mask_areas {
|
||||
fill_rect(
|
||||
&mut image,
|
||||
mask.global_pos_x_px,
|
||||
mask.global_pos_y_px,
|
||||
mask.width_px,
|
||||
mask.height_px,
|
||||
PURPLE_FILL,
|
||||
8,
|
||||
8,
|
||||
);
|
||||
}
|
||||
draw_frame(
|
||||
&mut image,
|
||||
node.global_pos_x_px,
|
||||
node.global_pos_y_px,
|
||||
node.width_px,
|
||||
node.height_px,
|
||||
8,
|
||||
8,
|
||||
);
|
||||
assert_eq!(*image.get_pixel(0, 0), PURPLE_FILL);
|
||||
assert_eq!(*image.get_pixel(1, 1), image::Rgba([0, 255, 0, 255]));
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,10 @@
|
||||
mod area;
|
||||
mod marker;
|
||||
mod model;
|
||||
mod persistence;
|
||||
mod prompt;
|
||||
mod tree;
|
||||
mod workflow;
|
||||
|
||||
pub(crate) use marker::build_marked_image;
|
||||
pub use model::*;
|
||||
pub use persistence::*;
|
||||
pub use tree::*;
|
||||
@@ -110,7 +108,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn construction_attaches_text_mask_to_nearest_unbound_image() {
|
||||
fn construction_keeps_text_as_removal_only_context() {
|
||||
let image = Component::Image(ImageComponent {
|
||||
target_graphic: None,
|
||||
image_type: ImageType::Simple {
|
||||
@@ -130,8 +128,8 @@ mod tests {
|
||||
let result = construct_separation_state(&state(root));
|
||||
let outer = &result.trees[0].root.children[0];
|
||||
assert_eq!(outer.id.as_str(), "outer-image");
|
||||
assert_eq!(outer.text_mask_areas.len(), 1);
|
||||
assert!(outer.children.is_empty());
|
||||
assert_eq!(outer.kind, SeparationNodeKind::ImageTarget);
|
||||
assert_eq!(outer.children[0].kind, SeparationNodeKind::TextRemovalOnly);
|
||||
|
||||
let nested_root = node(
|
||||
"root",
|
||||
@@ -148,12 +146,12 @@ mod tests {
|
||||
);
|
||||
let nested = construct_separation_state(&state(nested_root));
|
||||
let inner = &nested.trees[0].root.children[0].children[0];
|
||||
assert_eq!(inner.text_mask_areas.len(), 1);
|
||||
assert!(nested.trees[0].root.children[0].text_mask_areas.is_empty());
|
||||
assert_eq!(inner.kind, SeparationNodeKind::ImageTarget);
|
||||
assert_eq!(inner.children[0].kind, SeparationNodeKind::TextRemovalOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn root_image_receives_text_mask() {
|
||||
fn root_image_keeps_text_as_removal_only_context() {
|
||||
let root = node(
|
||||
"root-image",
|
||||
Some(Component::Image(ImageComponent {
|
||||
@@ -169,12 +167,16 @@ mod tests {
|
||||
)],
|
||||
);
|
||||
let result = construct_separation_state(&state(root));
|
||||
assert_eq!(result.trees[0].root.text_mask_areas.len(), 1);
|
||||
assert_eq!(
|
||||
result.trees[0].root.children[0].kind,
|
||||
SeparationNodeKind::TextRemovalOnly
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn binding_validation_requires_exact_batch_coverage() {
|
||||
let node = SeparationNode {
|
||||
id: NodeId::new("image").unwrap(),
|
||||
kind: SeparationNodeKind::ImageTarget,
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 1,
|
||||
@@ -183,7 +185,6 @@ mod tests {
|
||||
description: "image".to_string(),
|
||||
rework_notes: Vec::new(),
|
||||
},
|
||||
text_mask_areas: vec![],
|
||||
children: vec![],
|
||||
rework_count: 0,
|
||||
};
|
||||
@@ -248,7 +249,7 @@ mod tests {
|
||||
MAX_REWORK_COUNT
|
||||
);
|
||||
assert_eq!(node.rework_count, MAX_REWORK_COUNT);
|
||||
assert!(next_leaf_batch(&separation, &separation.trees[0]).is_empty());
|
||||
assert!(next_image_batch(&separation, &separation.trees[0]).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -265,12 +266,12 @@ mod tests {
|
||||
fn binding_validation_rejects_overlong_rework_note() {
|
||||
let node = SeparationNode {
|
||||
id: NodeId::new("image").unwrap(),
|
||||
kind: SeparationNodeKind::ImageTarget,
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 1,
|
||||
height_px: 1,
|
||||
note: SeparationNote::default(),
|
||||
text_mask_areas: vec![],
|
||||
children: vec![],
|
||||
rework_count: 0,
|
||||
};
|
||||
@@ -292,6 +293,7 @@ mod tests {
|
||||
assert!(dir.to_string_lossy().contains("ui_1-"));
|
||||
assert!(dir.to_string_lossy().ends_with("-separation"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn patch_collects_bound_and_keeps_tree_topology() {
|
||||
let image = Component::Image(ImageComponent {
|
||||
@@ -322,37 +324,37 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_selection_greedily_skips_overlapping_leaves() {
|
||||
fn batch_selection_keeps_dfs_order_even_when_rectangles_overlap() {
|
||||
let a = SeparationNode {
|
||||
id: NodeId::new("a").unwrap(),
|
||||
kind: SeparationNodeKind::ImageTarget,
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 10,
|
||||
height_px: 10,
|
||||
note: SeparationNote::default(),
|
||||
text_mask_areas: vec![],
|
||||
children: vec![],
|
||||
rework_count: 0,
|
||||
};
|
||||
let b = SeparationNode {
|
||||
id: NodeId::new("b").unwrap(),
|
||||
kind: SeparationNodeKind::ImageTarget,
|
||||
global_pos_x_px: 5,
|
||||
global_pos_y_px: 5,
|
||||
width_px: 10,
|
||||
height_px: 10,
|
||||
note: SeparationNote::default(),
|
||||
text_mask_areas: vec![],
|
||||
children: vec![],
|
||||
rework_count: 0,
|
||||
};
|
||||
let c = SeparationNode {
|
||||
id: NodeId::new("c").unwrap(),
|
||||
kind: SeparationNodeKind::ImageTarget,
|
||||
global_pos_x_px: 20,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 5,
|
||||
height_px: 5,
|
||||
note: SeparationNote::default(),
|
||||
text_mask_areas: vec![],
|
||||
children: vec![],
|
||||
rework_count: 0,
|
||||
};
|
||||
@@ -360,12 +362,12 @@ mod tests {
|
||||
src_ui_design: UIDesignImageId::new("page").unwrap(),
|
||||
root: SeparationNode {
|
||||
id: NodeId::new("root").unwrap(),
|
||||
kind: SeparationNodeKind::PureContainer,
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 100,
|
||||
height_px: 100,
|
||||
note: SeparationNote::default(),
|
||||
text_mask_areas: vec![],
|
||||
children: vec![a, b, c],
|
||||
rework_count: 0,
|
||||
},
|
||||
@@ -377,13 +379,134 @@ mod tests {
|
||||
bound: vec![],
|
||||
problematic_nodes: vec![],
|
||||
};
|
||||
let batch = next_leaf_batch(&state, &tree);
|
||||
let batch = next_image_batch(&state, &tree);
|
||||
assert_eq!(
|
||||
batch
|
||||
.iter()
|
||||
.map(|node| node.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["a", "c"]
|
||||
vec!["a", "b", "c"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_selection_skips_text_removal_only_nodes() {
|
||||
let image = Component::Image(ImageComponent {
|
||||
target_graphic: None,
|
||||
image_type: ImageType::Simple {
|
||||
preserve_aspect: false,
|
||||
},
|
||||
});
|
||||
let text = Component::Text(TextComponent::new("标题"));
|
||||
let separation = construct_separation_state(&state(node(
|
||||
"root",
|
||||
None,
|
||||
vec![
|
||||
node("text", Some(text), vec![]),
|
||||
node("image", Some(image), vec![]),
|
||||
],
|
||||
)));
|
||||
|
||||
let batch = next_image_batch(&separation, &separation.trees[0]);
|
||||
assert_eq!(
|
||||
batch
|
||||
.iter()
|
||||
.map(|node| node.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["image"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_selection_includes_extractable_root_before_children() {
|
||||
let image = Component::Image(ImageComponent {
|
||||
target_graphic: None,
|
||||
image_type: ImageType::Simple {
|
||||
preserve_aspect: false,
|
||||
},
|
||||
});
|
||||
let separation = construct_separation_state(&state(node(
|
||||
"root-image",
|
||||
Some(image.clone()),
|
||||
vec![node("child-image", Some(image), vec![])],
|
||||
)));
|
||||
|
||||
let batch = next_image_batch(&separation, &separation.trees[0]);
|
||||
assert_eq!(
|
||||
batch
|
||||
.iter()
|
||||
.map(|node| node.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["root-image", "child-image"]
|
||||
);
|
||||
}
|
||||
|
||||
fn image_separation_node(id: &str, width_px: u32, height_px: u32) -> SeparationNode {
|
||||
SeparationNode {
|
||||
id: NodeId::new(id).unwrap(),
|
||||
kind: SeparationNodeKind::ImageTarget,
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px,
|
||||
height_px,
|
||||
note: SeparationNote::default(),
|
||||
children: vec![],
|
||||
rework_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn separation_with_children(
|
||||
children: Vec<SeparationNode>,
|
||||
) -> (SeparationState, SeparationTree) {
|
||||
let tree = SeparationTree {
|
||||
src_ui_design: UIDesignImageId::new("page").unwrap(),
|
||||
root: SeparationNode {
|
||||
id: NodeId::new("root").unwrap(),
|
||||
kind: SeparationNodeKind::PureContainer,
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 100,
|
||||
height_px: 100,
|
||||
note: SeparationNote::default(),
|
||||
children,
|
||||
rework_count: 0,
|
||||
},
|
||||
root_extractable: false,
|
||||
};
|
||||
let separation = SeparationState {
|
||||
schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(),
|
||||
trees: vec![tree.clone()],
|
||||
bound: vec![],
|
||||
problematic_nodes: vec![],
|
||||
};
|
||||
(separation, tree)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_selection_stops_before_second_node_that_exceeds_area_budget() {
|
||||
let first_area = IMAGE_EDIT_AREA_LIMIT_PX / 2 + 1;
|
||||
let first = image_separation_node("first", first_area as u32, 1);
|
||||
let second = image_separation_node("second", first_area as u32, 1);
|
||||
let (separation, tree) = separation_with_children(vec![first, second]);
|
||||
|
||||
let batch = next_image_batch(&separation, &tree);
|
||||
assert_eq!(
|
||||
batch
|
||||
.iter()
|
||||
.map(|node| node.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["first"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_selection_accepts_first_oversized_node_to_guarantee_progress() {
|
||||
let oversized =
|
||||
image_separation_node("oversized", (IMAGE_EDIT_AREA_LIMIT_PX + 1) as u32, 1);
|
||||
let (separation, tree) = separation_with_children(vec![oversized]);
|
||||
|
||||
let batch = next_image_batch(&separation, &tree);
|
||||
assert_eq!(batch.len(), 1);
|
||||
assert_eq!(batch[0].id.as_str(), "oversized");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,11 @@ pub use node::*;
|
||||
pub use note::*;
|
||||
pub use result::*;
|
||||
|
||||
pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1";
|
||||
pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v2";
|
||||
pub const MAX_REWORK_COUNT: u32 = 3;
|
||||
pub const MAX_REWORK_NOTE_CHARS: usize = 512;
|
||||
pub const IMAGE_EDIT_MAX_DIMENSION_PX: u64 = 2880;
|
||||
pub const IMAGE_EDIT_AREA_UTILIZATION_PERCENT: u64 = 80;
|
||||
pub const IMAGE_EDIT_AREA_LIMIT_PX: u64 =
|
||||
IMAGE_EDIT_MAX_DIMENSION_PX * IMAGE_EDIT_MAX_DIMENSION_PX * IMAGE_EDIT_AREA_UTILIZATION_PERCENT
|
||||
/ 100;
|
||||
|
||||
+6
-7
@@ -2,25 +2,24 @@ use crate::ui_editor::utils::{NodeId, UIDesignImageId};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use ts_rs::TS;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct TextMaskArea {
|
||||
pub global_pos_x_px: u32,
|
||||
pub global_pos_y_px: u32,
|
||||
pub width_px: u32,
|
||||
pub height_px: u32,
|
||||
pub enum SeparationNodeKind {
|
||||
ImageTarget,
|
||||
TextRemovalOnly,
|
||||
PureContainer,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct SeparationNode {
|
||||
pub id: NodeId,
|
||||
pub kind: SeparationNodeKind,
|
||||
pub global_pos_x_px: u32,
|
||||
pub global_pos_y_px: u32,
|
||||
pub width_px: u32,
|
||||
pub height_px: u32,
|
||||
pub note: super::SeparationNote,
|
||||
pub text_mask_areas: Vec<TextMaskArea>,
|
||||
pub children: Vec<SeparationNode>,
|
||||
pub rework_count: u32,
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ use crate::ui_editor::component::{image::ImageComponent, Component};
|
||||
use crate::ui_editor::layout::node::Node;
|
||||
use crate::ui_editor::state::State;
|
||||
use crate::ui_editor::utils::NodeId;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
|
||||
fn is_unbound_image(node: &Node) -> bool {
|
||||
@@ -54,61 +53,41 @@ fn collect_todo_nodes(
|
||||
parent: &crate::ui_editor::layout::dimension::UIRect,
|
||||
ppu: f32,
|
||||
output: &mut Vec<SeparationNode>,
|
||||
text_masks: &mut HashMap<NodeId, Vec<TextMaskArea>>,
|
||||
) {
|
||||
let rect = node.layout.transform.resolve(parent);
|
||||
let mut children = Vec::new();
|
||||
for child in &node.children {
|
||||
collect_todo_nodes(child, &rect, ppu, &mut children, text_masks);
|
||||
collect_todo_nodes(child, &rect, ppu, &mut children);
|
||||
}
|
||||
if is_unbound_image(node) {
|
||||
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
|
||||
output.push(SeparationNode {
|
||||
id: node.id.clone(),
|
||||
global_pos_x_px: x,
|
||||
global_pos_y_px: y,
|
||||
width_px: w,
|
||||
height_px: h,
|
||||
note: SeparationNote {
|
||||
description: node_description(node),
|
||||
rework_notes: Vec::new(),
|
||||
},
|
||||
text_mask_areas: text_masks.remove(&node.id).unwrap_or_default(),
|
||||
children,
|
||||
rework_count: 0,
|
||||
});
|
||||
let kind = if is_unbound_image(node) {
|
||||
Some(SeparationNodeKind::ImageTarget)
|
||||
} else if has_text_component(node) && !has_image_component(node) {
|
||||
Some(SeparationNodeKind::TextRemovalOnly)
|
||||
} else {
|
||||
output.extend(children);
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_text_masks(
|
||||
node: &Node,
|
||||
parent: &crate::ui_editor::layout::dimension::UIRect,
|
||||
ppu: f32,
|
||||
nearest_image: Option<NodeId>,
|
||||
output: &mut HashMap<NodeId, Vec<TextMaskArea>>,
|
||||
) {
|
||||
let node_is_image = is_unbound_image(node);
|
||||
let nearest_image = if node_is_image {
|
||||
Some(node.id.clone())
|
||||
} else {
|
||||
nearest_image
|
||||
None
|
||||
};
|
||||
if has_text_component(node) && !has_image_component(node) {
|
||||
if let Some(image_id) = nearest_image.clone() {
|
||||
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
|
||||
output.entry(image_id).or_default().push(TextMaskArea {
|
||||
if let Some(kind) = kind {
|
||||
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
|
||||
if w > 0 && h > 0 {
|
||||
output.push(SeparationNode {
|
||||
id: node.id.clone(),
|
||||
kind,
|
||||
global_pos_x_px: x,
|
||||
global_pos_y_px: y,
|
||||
width_px: w,
|
||||
height_px: h,
|
||||
note: SeparationNote {
|
||||
description: node_description(node),
|
||||
rework_notes: Vec::new(),
|
||||
},
|
||||
children,
|
||||
rework_count: 0,
|
||||
});
|
||||
} else {
|
||||
output.extend(children);
|
||||
}
|
||||
}
|
||||
let rect = node.layout.transform.resolve(parent);
|
||||
for child in &node.children {
|
||||
collect_text_masks(child, &rect, ppu, nearest_image.clone(), output);
|
||||
} else {
|
||||
output.extend(children);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,11 +101,9 @@ pub fn construct_separation_state(state: &State) -> SeparationState {
|
||||
let size = image.pixel_size / ppu;
|
||||
let root_rect =
|
||||
crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size);
|
||||
let mut text_masks = HashMap::new();
|
||||
collect_text_masks(&tree.root, &root_rect, ppu, None, &mut text_masks);
|
||||
let mut children = Vec::new();
|
||||
for child in &tree.root.children {
|
||||
collect_todo_nodes(child, &root_rect, ppu, &mut children, &mut text_masks);
|
||||
collect_todo_nodes(child, &root_rect, ppu, &mut children);
|
||||
}
|
||||
let root_extractable = is_unbound_image(&tree.root);
|
||||
if !root_extractable && children.is_empty() {
|
||||
@@ -137,6 +114,13 @@ pub fn construct_separation_state(state: &State) -> SeparationState {
|
||||
src_ui_design: tree.src_ui_design.clone(),
|
||||
root: SeparationNode {
|
||||
id: tree.root.id.clone(),
|
||||
kind: if root_extractable {
|
||||
SeparationNodeKind::ImageTarget
|
||||
} else if has_text_component(&tree.root) && !has_image_component(&tree.root) {
|
||||
SeparationNodeKind::TextRemovalOnly
|
||||
} else {
|
||||
SeparationNodeKind::PureContainer
|
||||
},
|
||||
global_pos_x_px: x,
|
||||
global_pos_y_px: y,
|
||||
width_px: w,
|
||||
@@ -145,7 +129,6 @@ pub fn construct_separation_state(state: &State) -> SeparationState {
|
||||
description: node_description(&tree.root),
|
||||
rework_notes: Vec::new(),
|
||||
},
|
||||
text_mask_areas: text_masks.remove(&tree.root.id).unwrap_or_default(),
|
||||
children,
|
||||
rework_count: 0,
|
||||
},
|
||||
@@ -170,62 +153,55 @@ fn terminal_ids(state: &SeparationState) -> HashSet<NodeId> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn logical_leaves<'a>(
|
||||
fn collect_dfs_batch<'a>(
|
||||
node: &'a SeparationNode,
|
||||
extractable: bool,
|
||||
is_root: bool,
|
||||
root_extractable: bool,
|
||||
terminal: &HashSet<NodeId>,
|
||||
output: &mut Vec<&'a SeparationNode>,
|
||||
) {
|
||||
let is_terminal = terminal.contains(&node.id);
|
||||
let children_terminal = node
|
||||
.children
|
||||
.iter()
|
||||
.all(|child| terminal.contains(&child.id));
|
||||
if extractable && !is_terminal && children_terminal {
|
||||
output.push(node);
|
||||
return;
|
||||
selected: &mut Vec<&'a SeparationNode>,
|
||||
area: &mut u64,
|
||||
) -> bool {
|
||||
let is_target = matches!(node.kind, SeparationNodeKind::ImageTarget)
|
||||
&& (!is_root || root_extractable)
|
||||
&& !terminal.contains(&node.id);
|
||||
if is_target {
|
||||
let node_area = u64::from(node.width_px).saturating_mul(u64::from(node.height_px));
|
||||
let would_exceed = area.saturating_add(node_area) > IMAGE_EDIT_AREA_LIMIT_PX;
|
||||
if selected.is_empty() || !would_exceed {
|
||||
selected.push(node);
|
||||
*area = area.saturating_add(node_area);
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
for child in &node.children {
|
||||
logical_leaves(child, true, terminal, output);
|
||||
if collect_dfs_batch(child, false, root_extractable, terminal, selected, area) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn overlaps(a: &SeparationNode, b: &SeparationNode) -> bool {
|
||||
let ax1 = a.global_pos_x_px as u64 + a.width_px as u64;
|
||||
let ay1 = a.global_pos_y_px as u64 + a.height_px as u64;
|
||||
let bx1 = b.global_pos_x_px as u64 + b.width_px as u64;
|
||||
let by1 = b.global_pos_y_px as u64 + b.height_px as u64;
|
||||
let width = ax1
|
||||
.min(bx1)
|
||||
.saturating_sub(a.global_pos_x_px.max(b.global_pos_x_px) as u64);
|
||||
let height = ay1
|
||||
.min(by1)
|
||||
.saturating_sub(a.global_pos_y_px.max(b.global_pos_y_px) as u64);
|
||||
width > 0 && height > 0
|
||||
}
|
||||
|
||||
pub fn next_leaf_batch<'a>(
|
||||
pub fn next_image_batch<'a>(
|
||||
state: &SeparationState,
|
||||
tree: &'a SeparationTree,
|
||||
) -> Vec<&'a SeparationNode> {
|
||||
let terminal = terminal_ids(state);
|
||||
let mut candidates = Vec::new();
|
||||
logical_leaves(
|
||||
let mut selected = Vec::new();
|
||||
let mut area = 0;
|
||||
collect_dfs_batch(
|
||||
&tree.root,
|
||||
true,
|
||||
tree.root_extractable,
|
||||
&terminal,
|
||||
&mut candidates,
|
||||
&mut selected,
|
||||
&mut area,
|
||||
);
|
||||
let mut selected: Vec<&'a SeparationNode> = Vec::new();
|
||||
for candidate in candidates {
|
||||
if selected.iter().all(|other| !overlaps(candidate, other)) {
|
||||
selected.push(candidate);
|
||||
}
|
||||
}
|
||||
app_log!(
|
||||
"ui_separation.batch_selected image_id={} leaf_nodes={}",
|
||||
"ui_separation.batch_selected image_id={} image_nodes={} area_px={}",
|
||||
tree.src_ui_design.as_str(),
|
||||
selected.len()
|
||||
selected.len(),
|
||||
area
|
||||
);
|
||||
selected
|
||||
}
|
||||
@@ -236,6 +212,7 @@ pub fn validate_binding_response(
|
||||
) -> Result<(), String> {
|
||||
let expected = batch
|
||||
.iter()
|
||||
.filter(|node| matches!(node.kind, SeparationNodeKind::ImageTarget))
|
||||
.map(|node| node.id.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
let mut seen = HashSet::new();
|
||||
@@ -251,11 +228,7 @@ pub fn validate_binding_response(
|
||||
if !seen.insert(node_id.clone()) {
|
||||
return Err(format!("视觉绑定重复返回节点:{}", node_id.as_str()));
|
||||
}
|
||||
if let BindingDecision::NeedRework {
|
||||
advice,
|
||||
..
|
||||
} = decision
|
||||
{
|
||||
if let BindingDecision::NeedRework { advice, .. } = decision {
|
||||
if advice.trim().is_empty() {
|
||||
return Err("NeedRework 必须包含问题描述".to_string());
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ use serde::Deserialize;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Instant;
|
||||
|
||||
pub fn apply_batch_patch(
|
||||
state: &mut SeparationState,
|
||||
tree_index: usize,
|
||||
@@ -36,7 +37,7 @@ pub fn apply_batch_patch(
|
||||
.trees
|
||||
.get(tree_index)
|
||||
.ok_or_else(|| "separation tree 索引无效".to_string())?;
|
||||
next_leaf_batch(state, tree)
|
||||
next_image_batch(state, tree)
|
||||
.into_iter()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
@@ -160,13 +161,13 @@ async fn raw_image_edit_inner(
|
||||
.and_then(|v| v.strip_suffix(";base64"))
|
||||
.unwrap_or("image/png");
|
||||
if !mime.eq_ignore_ascii_case("image/png") {
|
||||
return Err("图片分离请求只支持 PNG 标记图".to_string());
|
||||
return Err("图片分离请求只支持 PNG 源图".to_string());
|
||||
}
|
||||
let image_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(data.trim())
|
||||
.map_err(|error| format!("解码标记图失败:{error}"))?;
|
||||
.map_err(|error| format!("解码源图失败:{error}"))?;
|
||||
if image_bytes.is_empty() {
|
||||
return Err("标记图不能为空".to_string());
|
||||
return Err("源图不能为空".to_string());
|
||||
}
|
||||
let client = crate::http_client::agc_main_site_client_builder()
|
||||
.build()
|
||||
@@ -481,7 +482,7 @@ pub(crate) async fn separate_ui_impl(
|
||||
let Some(current_tree) = separation.trees.get(tree_index) else {
|
||||
break;
|
||||
};
|
||||
let batch_nodes = next_leaf_batch(&separation, current_tree)
|
||||
let batch_nodes = next_image_batch(&separation, current_tree)
|
||||
.into_iter()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
@@ -496,8 +497,7 @@ pub(crate) async fn separate_ui_impl(
|
||||
break;
|
||||
}
|
||||
let batch = batch_nodes.iter().collect::<Vec<_>>();
|
||||
let prompt =
|
||||
gen_extract_prompt(batch.iter().map(|n| n.note.clone()).collect::<Vec<_>>());
|
||||
let prompt = gen_extract_prompt(&separation, current_tree, &batch);
|
||||
app_log!(
|
||||
"ui_separation.batch_start tree_index={} batch_index={} nodes={} prompt_chars={} rework_total={}",
|
||||
tree_index,
|
||||
@@ -506,28 +506,9 @@ pub(crate) async fn separate_ui_impl(
|
||||
prompt.chars().count(),
|
||||
batch.iter().map(|node| node.rework_count).sum::<u32>()
|
||||
);
|
||||
let marker_path = sidecar.join(format!("marked-{}.png", separation.bound.len()));
|
||||
let marked_url = match build_marked_image(
|
||||
source_url.clone(),
|
||||
batch_nodes.clone(),
|
||||
marker_path,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
app_log!(
|
||||
"ui_separation.error stage=mark_image tree_index={} batch_index={} error={error}",
|
||||
tree_index,
|
||||
batch_index
|
||||
);
|
||||
write_separation_state(&state_path, &separation)?;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let processed_url = match raw_image_edit(
|
||||
&session,
|
||||
&marked_url,
|
||||
&source_url,
|
||||
&prompt,
|
||||
image.pixel_size.x as u32,
|
||||
image.pixel_size.y as u32,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { NodeId } from "./NodeId";
|
||||
import type { SeparationNodeKind } from "./SeparationNodeKind";
|
||||
import type { SeparationNote } from "./SeparationNote";
|
||||
import type { TextMaskArea } from "./TextMaskArea";
|
||||
|
||||
export type SeparationNode = { id: NodeId, global_pos_x_px: number, global_pos_y_px: number, width_px: number, height_px: number, note: SeparationNote, text_mask_areas: Array<TextMaskArea>, children: Array<SeparationNode>, rework_count: number, };
|
||||
export type SeparationNode = { id: NodeId, kind: SeparationNodeKind, global_pos_x_px: number, global_pos_y_px: number, width_px: number, height_px: number, note: SeparationNote, children: Array<SeparationNode>, rework_count: number, };
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
|
||||
export type SeparationNodeKind = "ImageTarget" | "TextRemovalOnly" | "PureContainer";
|
||||
@@ -1,3 +0,0 @@
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
|
||||
export type TextMaskArea = { global_pos_x_px: number, global_pos_y_px: number, width_px: number, height_px: number, };
|
||||
Reference in New Issue
Block a user