重构UI素材切分树与批次选择

移除文本遮罩和标记图生成流程,图片编辑直接读取原始界面图

引入节点类型并保留文字移除上下文,按前序DFS与面积预算选择图片批次

补充根节点、重叠节点、文字过滤和超面积推进测试并同步TypeScript类型
This commit is contained in:
2026-09-11 12:07:19 +08:00
parent 13701d0144
commit 1db7597141
9 changed files with 231 additions and 514 deletions
@@ -1,364 +0,0 @@
use super::model::SeparationNode;
use base64::Engine as _;
use std::path::Path;
use std::path::PathBuf;
use std::time::Instant;
const MARKER_LINE_WIDTH: u32 = 2;
const PURPLE_FILL: image::Rgba<u8> = image::Rgba([180, 0, 180, 120]);
pub async fn build_marked_image(
source_url: String,
nodes: Vec<SeparationNode>,
target: PathBuf,
) -> Result<String, String> {
let node_count = nodes.len();
let started = Instant::now();
let result = match tokio::task::spawn_blocking(move || {
let blocking_started = Instant::now();
let result = build_marked_image_blocking(&source_url, &nodes, &target);
app_log!(
"ui_separation.marker.blocking_timing outcome={} elapsed_ms={} nodes={}",
if result.is_ok() { "ok" } else { "error" },
blocking_started.elapsed().as_millis(),
node_count
);
result
})
.await
{
Ok(result) => result,
Err(error) => Err(format!("构建标记图任务失败:{error}")),
};
app_log!(
"ui_separation.marker.timing outcome={} elapsed_ms={} nodes={}",
if result.is_ok() { "ok" } else { "error" },
started.elapsed().as_millis(),
node_count
);
result
}
fn build_marked_image_blocking(
source_url: &str,
nodes: &[SeparationNode],
target: &Path,
) -> Result<String, String> {
let encoded = source_url
.split_once(',')
.map(|(_, d)| d)
.ok_or_else(|| "源图 data URL 无效".to_string())?;
let bytes = base64::engine::general_purpose::STANDARD
.decode(encoded)
.map_err(|e| format!("解码源图失败:{e}"))?;
let mut image = image::load_from_memory(&bytes)
.map_err(|e| format!("读取源图失败:{e}"))?
.to_rgba8();
let width = image.width();
let height = image.height();
// 先填充紫色重建区域,随后绘制绿色框,确保绿色框位于最上层。
for node in nodes {
for child in &node.children {
fill_rect(
&mut image,
child.global_pos_x_px,
child.global_pos_y_px,
child.width_px,
child.height_px,
PURPLE_FILL,
width,
height,
);
}
for mask in &node.text_mask_areas {
fill_rect(
&mut image,
mask.global_pos_x_px,
mask.global_pos_y_px,
mask.width_px,
mask.height_px,
PURPLE_FILL,
width,
height,
);
}
}
for node in nodes {
draw_frame(
&mut image,
node.global_pos_x_px,
node.global_pos_y_px,
node.width_px,
node.height_px,
width,
height,
);
}
let image = image::DynamicImage::ImageRgba8(image);
image
.save_with_format(target, image::ImageFormat::Png)
.map_err(|e| format!("写入标记图失败:{e}"))?;
let mut png = Vec::new();
image
.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
.map_err(|e| format!("编码标记图失败:{e}"))?;
Ok(format!(
"data:image/png;base64,{}",
base64::engine::general_purpose::STANDARD.encode(png)
))
}
fn clipped_rect(
x: u32,
y: u32,
w: u32,
h: u32,
width: u32,
height: u32,
) -> Option<(u32, u32, u32, u32)> {
if width == 0 || height == 0 || w == 0 || h == 0 {
return None;
}
if x >= width || y >= height {
return None;
}
let x0 = x;
let y0 = y;
let x1 = x.saturating_add(w).min(width).saturating_sub(1);
let y1 = y.saturating_add(h).min(height).saturating_sub(1);
(x0 <= x1 && y0 <= y1).then_some((x0, y0, x1, y1))
}
fn fill_rect(
image: &mut image::RgbaImage,
x: u32,
y: u32,
w: u32,
h: u32,
color: image::Rgba<u8>,
width: u32,
height: u32,
) {
let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else {
return;
};
for yy in y0..=y1 {
for xx in x0..=x1 {
image.put_pixel(xx, yy, color);
}
}
}
fn draw_frame(
image: &mut image::RgbaImage,
x: u32,
y: u32,
w: u32,
h: u32,
width: u32,
height: u32,
) {
let Some((x0, y0, x1, y1)) = clipped_rect(x, y, w, h, width, height) else {
return;
};
let green = image::Rgba([0, 255, 0, 255]);
draw_line(
image,
(x0, y0),
(x1, y0),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
draw_line(
image,
(x0, y1),
(x1, y1),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
draw_line(
image,
(x0, y0),
(x0, y1),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
draw_line(
image,
(x1, y0),
(x1, y1),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
draw_line(
image,
(x0, y0),
(x1, y1),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
draw_line(
image,
(x1, y0),
(x0, y1),
green,
MARKER_LINE_WIDTH,
(x0, y0, x1, y1),
);
}
fn draw_line(
image: &mut image::RgbaImage,
start: (u32, u32),
end: (u32, u32),
color: image::Rgba<u8>,
line_width: u32,
bounds: (u32, u32, u32, u32),
) {
let mut x = start.0 as i64;
let mut y = start.1 as i64;
let target_x = end.0 as i64;
let target_y = end.1 as i64;
let dx = (target_x - x).abs();
let sx = if x < target_x { 1 } else { -1 };
let dy = -(target_y - y).abs();
let sy = if y < target_y { 1 } else { -1 };
let mut error = dx + dy;
loop {
draw_brush(image, x, y, color, line_width, bounds);
if x == target_x && y == target_y {
break;
}
let twice_error = error * 2;
if twice_error >= dy {
error += dy;
x += sx;
}
if twice_error <= dx {
error += dx;
y += sy;
}
}
}
fn draw_brush(
image: &mut image::RgbaImage,
x: i64,
y: i64,
color: image::Rgba<u8>,
line_width: u32,
bounds: (u32, u32, u32, u32),
) {
let (x0, y0, x1, y1) = bounds;
let line_width = line_width.max(1) as i64;
let before = (line_width - 1) / 2;
let after = line_width / 2;
let min_x = (x - before).max(x0 as i64);
let max_x = (x + after).min(x1 as i64);
let min_y = (y - before).max(y0 as i64);
let max_y = (y + after).min(y1 as i64);
for yy in min_y..=max_y {
for xx in min_x..=max_x {
image.put_pixel(xx as u32, yy as u32, color);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ui_editor::commands::separation::{SeparationNote, TextMaskArea};
#[test]
fn draw_frame_adds_green_cross_corner_lines() {
let mut image = image::RgbaImage::from_pixel(8, 6, image::Rgba([1, 2, 3, 255]));
draw_frame(&mut image, 1, 1, 5, 3, 8, 6);
let green = image::Rgba([0, 255, 0, 255]);
for &(x, y) in &[(1, 1), (5, 1), (1, 3), (5, 3), (3, 2)] {
assert_eq!(*image.get_pixel(x, y), green, "pixel ({x}, {y})");
}
assert_eq!(*image.get_pixel(3, 1), green);
assert_eq!(*image.get_pixel(3, 3), green);
assert_eq!(*image.get_pixel(2, 2), green);
assert_eq!(*image.get_pixel(4, 2), green);
assert_eq!(*image.get_pixel(0, 0), image::Rgba([1, 2, 3, 255]));
}
#[test]
fn draw_frame_keeps_cross_inside_clipped_rect() {
let mut image = image::RgbaImage::from_pixel(4, 4, image::Rgba([1, 2, 3, 255]));
draw_frame(&mut image, 2, 2, 4, 4, 4, 4);
let green = image::Rgba([0, 255, 0, 255]);
for y in 2..4 {
for x in 2..4 {
assert_eq!(*image.get_pixel(x, y), green);
}
}
assert_eq!(*image.get_pixel(1, 1), image::Rgba([1, 2, 3, 255]));
}
#[test]
fn clipped_rect_ignores_rectangles_starting_outside_image() {
assert_eq!(clipped_rect(4, 0, 1, 1, 4, 4), None);
assert_eq!(clipped_rect(0, 4, 1, 1, 4, 4), None);
}
#[test]
fn text_mask_is_purple_before_green_frame() {
let mut image = image::RgbaImage::from_pixel(8, 8, image::Rgba([1, 2, 3, 255]));
let node = SeparationNode {
id: crate::ui_editor::utils::NodeId::new("image").unwrap(),
global_pos_x_px: 1,
global_pos_y_px: 1,
width_px: 6,
height_px: 6,
note: SeparationNote::default(),
text_mask_areas: vec![
TextMaskArea {
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
},
TextMaskArea {
global_pos_x_px: 1,
global_pos_y_px: 1,
width_px: 1,
height_px: 1,
},
],
children: vec![],
rework_count: 0,
};
for mask in &node.text_mask_areas {
fill_rect(
&mut image,
mask.global_pos_x_px,
mask.global_pos_y_px,
mask.width_px,
mask.height_px,
PURPLE_FILL,
8,
8,
);
}
draw_frame(
&mut image,
node.global_pos_x_px,
node.global_pos_y_px,
node.width_px,
node.height_px,
8,
8,
);
assert_eq!(*image.get_pixel(0, 0), PURPLE_FILL);
assert_eq!(*image.get_pixel(1, 1), image::Rgba([0, 255, 0, 255]));
}
}
@@ -1,12 +1,10 @@
mod area;
mod marker;
mod model;
mod persistence;
mod prompt;
mod tree;
mod workflow;
pub(crate) use marker::build_marked_image;
pub use model::*;
pub use persistence::*;
pub use tree::*;
@@ -110,7 +108,7 @@ mod tests {
}
#[test]
fn construction_attaches_text_mask_to_nearest_unbound_image() {
fn construction_keeps_text_as_removal_only_context() {
let image = Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
@@ -130,8 +128,8 @@ mod tests {
let result = construct_separation_state(&state(root));
let outer = &result.trees[0].root.children[0];
assert_eq!(outer.id.as_str(), "outer-image");
assert_eq!(outer.text_mask_areas.len(), 1);
assert!(outer.children.is_empty());
assert_eq!(outer.kind, SeparationNodeKind::ImageTarget);
assert_eq!(outer.children[0].kind, SeparationNodeKind::TextRemovalOnly);
let nested_root = node(
"root",
@@ -148,12 +146,12 @@ mod tests {
);
let nested = construct_separation_state(&state(nested_root));
let inner = &nested.trees[0].root.children[0].children[0];
assert_eq!(inner.text_mask_areas.len(), 1);
assert!(nested.trees[0].root.children[0].text_mask_areas.is_empty());
assert_eq!(inner.kind, SeparationNodeKind::ImageTarget);
assert_eq!(inner.children[0].kind, SeparationNodeKind::TextRemovalOnly);
}
#[test]
fn root_image_receives_text_mask() {
fn root_image_keeps_text_as_removal_only_context() {
let root = node(
"root-image",
Some(Component::Image(ImageComponent {
@@ -169,12 +167,16 @@ mod tests {
)],
);
let result = construct_separation_state(&state(root));
assert_eq!(result.trees[0].root.text_mask_areas.len(), 1);
assert_eq!(
result.trees[0].root.children[0].kind,
SeparationNodeKind::TextRemovalOnly
);
}
#[test]
fn binding_validation_requires_exact_batch_coverage() {
let node = SeparationNode {
id: NodeId::new("image").unwrap(),
kind: SeparationNodeKind::ImageTarget,
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
@@ -183,7 +185,6 @@ mod tests {
description: "image".to_string(),
rework_notes: Vec::new(),
},
text_mask_areas: vec![],
children: vec![],
rework_count: 0,
};
@@ -248,7 +249,7 @@ mod tests {
MAX_REWORK_COUNT
);
assert_eq!(node.rework_count, MAX_REWORK_COUNT);
assert!(next_leaf_batch(&separation, &separation.trees[0]).is_empty());
assert!(next_image_batch(&separation, &separation.trees[0]).is_empty());
}
#[test]
@@ -265,12 +266,12 @@ mod tests {
fn binding_validation_rejects_overlong_rework_note() {
let node = SeparationNode {
id: NodeId::new("image").unwrap(),
kind: SeparationNodeKind::ImageTarget,
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
note: SeparationNote::default(),
text_mask_areas: vec![],
children: vec![],
rework_count: 0,
};
@@ -292,6 +293,7 @@ mod tests {
assert!(dir.to_string_lossy().contains("ui_1-"));
assert!(dir.to_string_lossy().ends_with("-separation"));
}
#[test]
fn patch_collects_bound_and_keeps_tree_topology() {
let image = Component::Image(ImageComponent {
@@ -322,37 +324,37 @@ mod tests {
}
#[test]
fn batch_selection_greedily_skips_overlapping_leaves() {
fn batch_selection_keeps_dfs_order_even_when_rectangles_overlap() {
let a = SeparationNode {
id: NodeId::new("a").unwrap(),
kind: SeparationNodeKind::ImageTarget,
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 10,
height_px: 10,
note: SeparationNote::default(),
text_mask_areas: vec![],
children: vec![],
rework_count: 0,
};
let b = SeparationNode {
id: NodeId::new("b").unwrap(),
kind: SeparationNodeKind::ImageTarget,
global_pos_x_px: 5,
global_pos_y_px: 5,
width_px: 10,
height_px: 10,
note: SeparationNote::default(),
text_mask_areas: vec![],
children: vec![],
rework_count: 0,
};
let c = SeparationNode {
id: NodeId::new("c").unwrap(),
kind: SeparationNodeKind::ImageTarget,
global_pos_x_px: 20,
global_pos_y_px: 0,
width_px: 5,
height_px: 5,
note: SeparationNote::default(),
text_mask_areas: vec![],
children: vec![],
rework_count: 0,
};
@@ -360,12 +362,12 @@ mod tests {
src_ui_design: UIDesignImageId::new("page").unwrap(),
root: SeparationNode {
id: NodeId::new("root").unwrap(),
kind: SeparationNodeKind::PureContainer,
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 100,
height_px: 100,
note: SeparationNote::default(),
text_mask_areas: vec![],
children: vec![a, b, c],
rework_count: 0,
},
@@ -377,13 +379,134 @@ mod tests {
bound: vec![],
problematic_nodes: vec![],
};
let batch = next_leaf_batch(&state, &tree);
let batch = next_image_batch(&state, &tree);
assert_eq!(
batch
.iter()
.map(|node| node.id.as_str())
.collect::<Vec<_>>(),
vec!["a", "c"]
vec!["a", "b", "c"]
);
}
#[test]
fn batch_selection_skips_text_removal_only_nodes() {
let image = Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
preserve_aspect: false,
},
});
let text = Component::Text(TextComponent::new("标题"));
let separation = construct_separation_state(&state(node(
"root",
None,
vec![
node("text", Some(text), vec![]),
node("image", Some(image), vec![]),
],
)));
let batch = next_image_batch(&separation, &separation.trees[0]);
assert_eq!(
batch
.iter()
.map(|node| node.id.as_str())
.collect::<Vec<_>>(),
vec!["image"]
);
}
#[test]
fn batch_selection_includes_extractable_root_before_children() {
let image = Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
preserve_aspect: false,
},
});
let separation = construct_separation_state(&state(node(
"root-image",
Some(image.clone()),
vec![node("child-image", Some(image), vec![])],
)));
let batch = next_image_batch(&separation, &separation.trees[0]);
assert_eq!(
batch
.iter()
.map(|node| node.id.as_str())
.collect::<Vec<_>>(),
vec!["root-image", "child-image"]
);
}
fn image_separation_node(id: &str, width_px: u32, height_px: u32) -> SeparationNode {
SeparationNode {
id: NodeId::new(id).unwrap(),
kind: SeparationNodeKind::ImageTarget,
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px,
height_px,
note: SeparationNote::default(),
children: vec![],
rework_count: 0,
}
}
fn separation_with_children(
children: Vec<SeparationNode>,
) -> (SeparationState, SeparationTree) {
let tree = SeparationTree {
src_ui_design: UIDesignImageId::new("page").unwrap(),
root: SeparationNode {
id: NodeId::new("root").unwrap(),
kind: SeparationNodeKind::PureContainer,
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 100,
height_px: 100,
note: SeparationNote::default(),
children,
rework_count: 0,
},
root_extractable: false,
};
let separation = SeparationState {
schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(),
trees: vec![tree.clone()],
bound: vec![],
problematic_nodes: vec![],
};
(separation, tree)
}
#[test]
fn batch_selection_stops_before_second_node_that_exceeds_area_budget() {
let first_area = IMAGE_EDIT_AREA_LIMIT_PX / 2 + 1;
let first = image_separation_node("first", first_area as u32, 1);
let second = image_separation_node("second", first_area as u32, 1);
let (separation, tree) = separation_with_children(vec![first, second]);
let batch = next_image_batch(&separation, &tree);
assert_eq!(
batch
.iter()
.map(|node| node.id.as_str())
.collect::<Vec<_>>(),
vec!["first"]
);
}
#[test]
fn batch_selection_accepts_first_oversized_node_to_guarantee_progress() {
let oversized =
image_separation_node("oversized", (IMAGE_EDIT_AREA_LIMIT_PX + 1) as u32, 1);
let (separation, tree) = separation_with_children(vec![oversized]);
let batch = next_image_batch(&separation, &tree);
assert_eq!(batch.len(), 1);
assert_eq!(batch[0].id.as_str(), "oversized");
}
}
@@ -8,6 +8,11 @@ pub use node::*;
pub use note::*;
pub use result::*;
pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1";
pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v2";
pub const MAX_REWORK_COUNT: u32 = 3;
pub const MAX_REWORK_NOTE_CHARS: usize = 512;
pub const IMAGE_EDIT_MAX_DIMENSION_PX: u64 = 2880;
pub const IMAGE_EDIT_AREA_UTILIZATION_PERCENT: u64 = 80;
pub const IMAGE_EDIT_AREA_LIMIT_PX: u64 =
IMAGE_EDIT_MAX_DIMENSION_PX * IMAGE_EDIT_MAX_DIMENSION_PX * IMAGE_EDIT_AREA_UTILIZATION_PERCENT
/ 100;
@@ -2,25 +2,24 @@ use crate::ui_editor::utils::{NodeId, UIDesignImageId};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize, TS)]
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct TextMaskArea {
pub global_pos_x_px: u32,
pub global_pos_y_px: u32,
pub width_px: u32,
pub height_px: u32,
pub enum SeparationNodeKind {
ImageTarget,
TextRemovalOnly,
PureContainer,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationNode {
pub id: NodeId,
pub kind: SeparationNodeKind,
pub global_pos_x_px: u32,
pub global_pos_y_px: u32,
pub width_px: u32,
pub height_px: u32,
pub note: super::SeparationNote,
pub text_mask_areas: Vec<TextMaskArea>,
pub children: Vec<SeparationNode>,
pub rework_count: u32,
}
@@ -3,7 +3,6 @@ use crate::ui_editor::component::{image::ImageComponent, Component};
use crate::ui_editor::layout::node::Node;
use crate::ui_editor::state::State;
use crate::ui_editor::utils::NodeId;
use std::collections::HashMap;
use std::collections::HashSet;
fn is_unbound_image(node: &Node) -> bool {
@@ -54,61 +53,41 @@ fn collect_todo_nodes(
parent: &crate::ui_editor::layout::dimension::UIRect,
ppu: f32,
output: &mut Vec<SeparationNode>,
text_masks: &mut HashMap<NodeId, Vec<TextMaskArea>>,
) {
let rect = node.layout.transform.resolve(parent);
let mut children = Vec::new();
for child in &node.children {
collect_todo_nodes(child, &rect, ppu, &mut children, text_masks);
collect_todo_nodes(child, &rect, ppu, &mut children);
}
if is_unbound_image(node) {
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
output.push(SeparationNode {
id: node.id.clone(),
global_pos_x_px: x,
global_pos_y_px: y,
width_px: w,
height_px: h,
note: SeparationNote {
description: node_description(node),
rework_notes: Vec::new(),
},
text_mask_areas: text_masks.remove(&node.id).unwrap_or_default(),
children,
rework_count: 0,
});
let kind = if is_unbound_image(node) {
Some(SeparationNodeKind::ImageTarget)
} else if has_text_component(node) && !has_image_component(node) {
Some(SeparationNodeKind::TextRemovalOnly)
} else {
output.extend(children);
}
}
fn collect_text_masks(
node: &Node,
parent: &crate::ui_editor::layout::dimension::UIRect,
ppu: f32,
nearest_image: Option<NodeId>,
output: &mut HashMap<NodeId, Vec<TextMaskArea>>,
) {
let node_is_image = is_unbound_image(node);
let nearest_image = if node_is_image {
Some(node.id.clone())
} else {
nearest_image
None
};
if has_text_component(node) && !has_image_component(node) {
if let Some(image_id) = nearest_image.clone() {
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
output.entry(image_id).or_default().push(TextMaskArea {
if let Some(kind) = kind {
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
if w > 0 && h > 0 {
output.push(SeparationNode {
id: node.id.clone(),
kind,
global_pos_x_px: x,
global_pos_y_px: y,
width_px: w,
height_px: h,
note: SeparationNote {
description: node_description(node),
rework_notes: Vec::new(),
},
children,
rework_count: 0,
});
} else {
output.extend(children);
}
}
let rect = node.layout.transform.resolve(parent);
for child in &node.children {
collect_text_masks(child, &rect, ppu, nearest_image.clone(), output);
} else {
output.extend(children);
}
}
@@ -122,11 +101,9 @@ pub fn construct_separation_state(state: &State) -> SeparationState {
let size = image.pixel_size / ppu;
let root_rect =
crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size);
let mut text_masks = HashMap::new();
collect_text_masks(&tree.root, &root_rect, ppu, None, &mut text_masks);
let mut children = Vec::new();
for child in &tree.root.children {
collect_todo_nodes(child, &root_rect, ppu, &mut children, &mut text_masks);
collect_todo_nodes(child, &root_rect, ppu, &mut children);
}
let root_extractable = is_unbound_image(&tree.root);
if !root_extractable && children.is_empty() {
@@ -137,6 +114,13 @@ pub fn construct_separation_state(state: &State) -> SeparationState {
src_ui_design: tree.src_ui_design.clone(),
root: SeparationNode {
id: tree.root.id.clone(),
kind: if root_extractable {
SeparationNodeKind::ImageTarget
} else if has_text_component(&tree.root) && !has_image_component(&tree.root) {
SeparationNodeKind::TextRemovalOnly
} else {
SeparationNodeKind::PureContainer
},
global_pos_x_px: x,
global_pos_y_px: y,
width_px: w,
@@ -145,7 +129,6 @@ pub fn construct_separation_state(state: &State) -> SeparationState {
description: node_description(&tree.root),
rework_notes: Vec::new(),
},
text_mask_areas: text_masks.remove(&tree.root.id).unwrap_or_default(),
children,
rework_count: 0,
},
@@ -170,62 +153,55 @@ fn terminal_ids(state: &SeparationState) -> HashSet<NodeId> {
.collect()
}
fn logical_leaves<'a>(
fn collect_dfs_batch<'a>(
node: &'a SeparationNode,
extractable: bool,
is_root: bool,
root_extractable: bool,
terminal: &HashSet<NodeId>,
output: &mut Vec<&'a SeparationNode>,
) {
let is_terminal = terminal.contains(&node.id);
let children_terminal = node
.children
.iter()
.all(|child| terminal.contains(&child.id));
if extractable && !is_terminal && children_terminal {
output.push(node);
return;
selected: &mut Vec<&'a SeparationNode>,
area: &mut u64,
) -> bool {
let is_target = matches!(node.kind, SeparationNodeKind::ImageTarget)
&& (!is_root || root_extractable)
&& !terminal.contains(&node.id);
if is_target {
let node_area = u64::from(node.width_px).saturating_mul(u64::from(node.height_px));
let would_exceed = area.saturating_add(node_area) > IMAGE_EDIT_AREA_LIMIT_PX;
if selected.is_empty() || !would_exceed {
selected.push(node);
*area = area.saturating_add(node_area);
} else {
return true;
}
}
for child in &node.children {
logical_leaves(child, true, terminal, output);
if collect_dfs_batch(child, false, root_extractable, terminal, selected, area) {
return true;
}
}
false
}
fn overlaps(a: &SeparationNode, b: &SeparationNode) -> bool {
let ax1 = a.global_pos_x_px as u64 + a.width_px as u64;
let ay1 = a.global_pos_y_px as u64 + a.height_px as u64;
let bx1 = b.global_pos_x_px as u64 + b.width_px as u64;
let by1 = b.global_pos_y_px as u64 + b.height_px as u64;
let width = ax1
.min(bx1)
.saturating_sub(a.global_pos_x_px.max(b.global_pos_x_px) as u64);
let height = ay1
.min(by1)
.saturating_sub(a.global_pos_y_px.max(b.global_pos_y_px) as u64);
width > 0 && height > 0
}
pub fn next_leaf_batch<'a>(
pub fn next_image_batch<'a>(
state: &SeparationState,
tree: &'a SeparationTree,
) -> Vec<&'a SeparationNode> {
let terminal = terminal_ids(state);
let mut candidates = Vec::new();
logical_leaves(
let mut selected = Vec::new();
let mut area = 0;
collect_dfs_batch(
&tree.root,
true,
tree.root_extractable,
&terminal,
&mut candidates,
&mut selected,
&mut area,
);
let mut selected: Vec<&'a SeparationNode> = Vec::new();
for candidate in candidates {
if selected.iter().all(|other| !overlaps(candidate, other)) {
selected.push(candidate);
}
}
app_log!(
"ui_separation.batch_selected image_id={} leaf_nodes={}",
"ui_separation.batch_selected image_id={} image_nodes={} area_px={}",
tree.src_ui_design.as_str(),
selected.len()
selected.len(),
area
);
selected
}
@@ -236,6 +212,7 @@ pub fn validate_binding_response(
) -> Result<(), String> {
let expected = batch
.iter()
.filter(|node| matches!(node.kind, SeparationNodeKind::ImageTarget))
.map(|node| node.id.clone())
.collect::<HashSet<_>>();
let mut seen = HashSet::new();
@@ -251,11 +228,7 @@ pub fn validate_binding_response(
if !seen.insert(node_id.clone()) {
return Err(format!("视觉绑定重复返回节点:{}", node_id.as_str()));
}
if let BindingDecision::NeedRework {
advice,
..
} = decision
{
if let BindingDecision::NeedRework { advice, .. } = decision {
if advice.trim().is_empty() {
return Err("NeedRework 必须包含问题描述".to_string());
}
@@ -19,6 +19,7 @@ use serde::Deserialize;
use std::fs;
use std::path::{Path, PathBuf};
use std::time::Instant;
pub fn apply_batch_patch(
state: &mut SeparationState,
tree_index: usize,
@@ -36,7 +37,7 @@ pub fn apply_batch_patch(
.trees
.get(tree_index)
.ok_or_else(|| "separation tree 索引无效".to_string())?;
next_leaf_batch(state, tree)
next_image_batch(state, tree)
.into_iter()
.cloned()
.collect::<Vec<_>>()
@@ -160,13 +161,13 @@ async fn raw_image_edit_inner(
.and_then(|v| v.strip_suffix(";base64"))
.unwrap_or("image/png");
if !mime.eq_ignore_ascii_case("image/png") {
return Err("图片分离请求只支持 PNG 标记图".to_string());
return Err("图片分离请求只支持 PNG 源图".to_string());
}
let image_bytes = base64::engine::general_purpose::STANDARD
.decode(data.trim())
.map_err(|error| format!("解码标记图失败:{error}"))?;
.map_err(|error| format!("解码源图失败:{error}"))?;
if image_bytes.is_empty() {
return Err("标记图不能为空".to_string());
return Err("源图不能为空".to_string());
}
let client = crate::http_client::agc_main_site_client_builder()
.build()
@@ -481,7 +482,7 @@ pub(crate) async fn separate_ui_impl(
let Some(current_tree) = separation.trees.get(tree_index) else {
break;
};
let batch_nodes = next_leaf_batch(&separation, current_tree)
let batch_nodes = next_image_batch(&separation, current_tree)
.into_iter()
.cloned()
.collect::<Vec<_>>();
@@ -496,8 +497,7 @@ pub(crate) async fn separate_ui_impl(
break;
}
let batch = batch_nodes.iter().collect::<Vec<_>>();
let prompt =
gen_extract_prompt(batch.iter().map(|n| n.note.clone()).collect::<Vec<_>>());
let prompt = gen_extract_prompt(&separation, current_tree, &batch);
app_log!(
"ui_separation.batch_start tree_index={} batch_index={} nodes={} prompt_chars={} rework_total={}",
tree_index,
@@ -506,28 +506,9 @@ pub(crate) async fn separate_ui_impl(
prompt.chars().count(),
batch.iter().map(|node| node.rework_count).sum::<u32>()
);
let marker_path = sidecar.join(format!("marked-{}.png", separation.bound.len()));
let marked_url = match build_marked_image(
source_url.clone(),
batch_nodes.clone(),
marker_path,
)
.await
{
Ok(value) => value,
Err(error) => {
app_log!(
"ui_separation.error stage=mark_image tree_index={} batch_index={} error={error}",
tree_index,
batch_index
);
write_separation_state(&state_path, &separation)?;
return Err(error);
}
};
let processed_url = match raw_image_edit(
&session,
&marked_url,
&source_url,
&prompt,
image.pixel_size.x as u32,
image.pixel_size.y as u32,
@@ -1,6 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { NodeId } from "./NodeId";
import type { SeparationNodeKind } from "./SeparationNodeKind";
import type { SeparationNote } from "./SeparationNote";
import type { TextMaskArea } from "./TextMaskArea";
export type SeparationNode = { id: NodeId, global_pos_x_px: number, global_pos_y_px: number, width_px: number, height_px: number, note: SeparationNote, text_mask_areas: Array<TextMaskArea>, children: Array<SeparationNode>, rework_count: number, };
export type SeparationNode = { id: NodeId, kind: SeparationNodeKind, global_pos_x_px: number, global_pos_y_px: number, width_px: number, height_px: number, note: SeparationNote, children: Array<SeparationNode>, rework_count: number, };
@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type SeparationNodeKind = "ImageTarget" | "TextRemovalOnly" | "PureContainer";
@@ -1,3 +0,0 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type TextMaskArea = { global_pos_x_px: number, global_pos_y_px: number, width_px: number, height_px: number, };