WIP: UI编辑器自动分图层切图标 #304

Draft
k88936 wants to merge 100 commits from feat/ui-editor-auto-seperation into master
8 changed files with 991 additions and 900 deletions
Showing only changes of commit 109cc81740 - Show all commits
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,140 @@
mod model;
mod persistence;
mod prompt;
mod tree;
mod workflow;
pub use model::*;
pub use persistence::*;
pub use tree::*;
pub(crate) use workflow::separate_ui_impl;
#[cfg(test)]
mod tests {
use super::*;
use crate::ui_editor::component::image::{ImageComponent, ImageType};
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
use crate::ui_editor::layout::control_layout::ControlLayout;
use crate::ui_editor::layout::node::{NodeMetadata, NodeSource, StageStatus};
use crate::ui_editor::resource::ui_design_image::UIDesignImage;
use nalgebra::Vector2;
use std::collections::HashMap;
use typed_floats::tf32::StrictlyPositiveFinite;
fn node(id: &str, components: Vec<Component>, children: Vec<Node>) -> Node {
Node {
id: NodeId::new(id).unwrap(),
layout: ControlLayout::default(),
metadata: NodeMetadata {
name: id.to_string(),
description: String::new(),
layout_status: StageStatus::NoProblem,
components_status: StageStatus::NoProblem,
allow_llm_edit_layout: true,
allow_llm_edit_component: true,
source: NodeSource::Llm,
},
components,
children_display_mode: ChildrenDisplayMode::Stack,
children,
}
}
fn state(root: Node) -> State {
let image_id = UIDesignImageId::new("page").unwrap();
State {
ui_trees: vec![UITree {
src_ui_design: image_id.clone(),
root,
}],
ui_design_images: HashMap::from([(
image_id,
UIDesignImage {
metadata: crate::ui_editor::resource::ui_design_image::UIDesignImageMetadata {
name: "page".to_string(),
description: String::new(),
role: None,
slave_to: None,
},
path: "page.png".to_string(),
pixel_size: Vector2::new(100.0, 100.0),
pixels_per_unit: StrictlyPositiveFinite::new(1.0).unwrap(),
},
)]),
sprite_assets: HashMap::new(),
font_assets: HashMap::new(),
}
}
#[test]
fn construction_filters_pure_nodes_and_passes_children_through() {
let image = Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
preserve_aspect: false,
},
});
let root = node(
"root",
vec![],
vec![node(
"container",
vec![],
vec![node("image", vec![image], vec![])],
)],
);
let result = construct_separation_state(&state(root));
assert_eq!(
result.unprocessed_trees[0].root.children[0].id.as_str(),
"image"
);
}
#[test]
fn binding_validation_requires_exact_batch_coverage() {
let node = SeparationNode {
id: NodeId::new("image").unwrap(),
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
note: SeparationNote {
description: "image".to_string(),
text_note: String::new(),
},
children: vec![],
rework_count: 0,
};
assert!(validate_binding_response(&BindingResp { decisions: vec![] }, &[&node]).is_err());
}
#[test]
fn sidecar_name_uses_asset_id_digest() {
let dir = separation_sidecar_dir(Path::new("/tmp/project"), "ui:1").unwrap();
assert!(dir.to_string_lossy().contains("ui_1-"));
assert!(dir.to_string_lossy().ends_with("-separation"));
}
#[test]
fn patch_collects_bound_and_removes_leaf() {
let image = Component::Image(ImageComponent {
target_graphic: None,
image_type: ImageType::Simple {
preserve_aspect: false,
},
});
let mut state = construct_separation_state(&state(node(
"root",
vec![],
vec![node("image", vec![image], vec![])],
)));
let id = NodeId::new("image").unwrap();
let decisions = vec![BindingDecision::Ok {
to_node: id.clone(),
separated_image_area: BindingArea {
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: 1,
height_px: 1,
},
}];
let paths = HashMap::from([(id.clone(), "ui/.sidecar/cut.png".to_string())]);
apply_batch_patch(&mut state, 0, &decisions, &paths).unwrap();
assert_eq!(state.bound[0].node_id, id);
assert!(state.unprocessed_trees[0].root.children.is_empty());
}
}
@@ -0,0 +1,123 @@
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use ts_rs::TS;
pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1";
pub const MAX_REWORK_COUNT: u32 = 3;
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationNote {
pub description: String,
pub text_note: String,
}
impl SeparationNote {
pub fn as_prompt(&self) -> String {
format!("desc: {} {}", self.description, self.text_note)
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationNode {
pub id: NodeId,
pub global_pos_x_px: u32,
pub global_pos_y_px: u32,
pub width_px: u32,
pub height_px: u32,
pub note: SeparationNote,
pub children: Vec<SeparationNode>,
pub rework_count: u32,
}
impl SeparationNode {
pub fn as_prompt(&self) -> String {
format!(
"node_id={} area=({}, {}, {}, {}) {}",
self.id.as_str(),
self.global_pos_x_px,
self.global_pos_y_px,
self.width_px,
self.height_px,
self.note.as_prompt()
)
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationTree {
pub src_ui_design: UIDesignImageId,
pub root: SeparationNode,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct BoundNode {
pub node_id: NodeId,
pub cut_image_path: String,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct ProblematicNode {
pub node_id: NodeId,
pub problem_description: String,
pub rework_count: u32,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationState {
pub schema_version: String,
pub unprocessed_trees: Vec<SeparationTree>,
pub bound: Vec<BoundNode>,
pub problematic_nodes: Vec<ProblematicNode>,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
pub struct SeparationDTO {
pub bound_nodes: Vec<BoundNode>,
pub problematic_nodes: Vec<ProblematicNode>,
}
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub struct BindingArea {
pub global_pos_x_px: u32,
pub global_pos_y_px: u32,
pub width_px: u32,
pub height_px: u32,
}
impl BindingArea {
pub fn validate_in(&self, w: u32, h: u32) -> Result<(), String> {
if self.width_px == 0 || self.height_px == 0 {
return Err("BindingArea 宽度和高度必须大于 0".into());
}
if self
.global_pos_x_px
.checked_add(self.width_px)
.is_none_or(|v| v > w)
|| self
.global_pos_y_px
.checked_add(self.height_px)
.is_none_or(|v| v > h)
{
return Err("BindingArea 超出处理图边界".into());
}
Ok(())
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub enum BindingDecision {
Ok {
separated_image_area: BindingArea,
to_node: NodeId,
},
NeedRework {
problem_description: String,
to_node: NodeId,
},
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
pub struct BindingResp {
pub decisions: Vec<BindingDecision>,
}
@@ -0,0 +1,49 @@
use super::model::*;
use crate::ui_editor::commands::separation::*;
use std::fs;
use std::path::{Path, PathBuf};
pub fn separation_sidecar_dir(root: &Path, asset_id: &str) -> Result<PathBuf, String> {
if asset_id.trim().is_empty() || asset_id.trim() != asset_id {
return Err("UI 资源 ID 无效".to_string());
}
let dir = root.join("ui").join(format!(
".{}-separation",
crate::ui_editor::persistence::generated_file_stem(asset_id)
));
if !dir.starts_with(root) {
return Err("separation sidecar 路径越界".to_string());
}
Ok(dir)
}
pub fn write_separation_state(path: &Path, state: &SeparationState) -> Result<(), String> {
if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION {
return Err("不支持的 separation state schema".to_string());
}
let bytes = serde_json::to_vec_pretty(state)
.map_err(|error| format!("序列化 separation state 失败:{error}"))?;
let parent = path
.parent()
.ok_or_else(|| "separation state 路径缺少父目录".to_string())?;
fs::create_dir_all(parent).map_err(|error| format!("创建 separation sidecar 失败:{error}"))?;
let temporary = path.with_extension("json.tmp");
fs::write(&temporary, bytes).map_err(|error| format!("写入 separation state 失败:{error}"))?;
fs::rename(&temporary, path).map_err(|error| format!("安装 separation state 失败:{error}"))
}
pub fn read_separation_state(path: &Path) -> Result<SeparationState, String> {
let bytes = fs::read(path).map_err(|error| format!("读取 separation state 失败:{error}"))?;
let state: SeparationState = serde_json::from_slice(&bytes)
.map_err(|error| format!("解析 separation state 失败:{error}"))?;
if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION {
return Err("不支持的 separation state schema".to_string());
}
Ok(state)
}
pub fn separation_dto(state: &SeparationState) -> SeparationDTO {
SeparationDTO {
bound_nodes: state.bound.clone(),
problematic_nodes: state.problematic_nodes.clone(),
}
}
@@ -0,0 +1,52 @@
use crate::ui_editor::commands::separation::{SeparationNode, SeparationNote};
const SHARED_SEPARATION_REQ: &str = r#"
MUST hard edges; preserve no glow/blur beyond the exact visible shape.
NEVER keep its parent's background with it.
UI elements that needs to extract has been marked with GREEN line frames (only for mark purpose, NEVER wrap a frame in your extraction).
On some UI elements, there is some PURPLE filled area, they were removed UI elements, reconstruct the background under where they were.
"#;
pub(super) fn gen_extract_prompt(separation_notes: Vec<SeparationNote>) -> String {
let extract_system_prompt = format!(
r#"
This is a UI design image, not a normal photo/illustration. Extract it strictly as UI elements/layers, not as a generic foreground/background extraction.
Treat distinct UI element as its own layer with hard, clean, pixel-accurate edges and full transparency outside the element.
MUST keep each element at its original position on a transparent canvas.
{SHARED_SEPARATION_REQ}
here are UI elements to extract:
"#
);
let mut result = extract_system_prompt;
result.reserve(512);
for elem in separation_notes {
result.push_str(&elem.as_prompt());
result.push('\n');
}
result
}
pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String {
let binding_system_prompt = format!(
r#"
You are working under a UI elements separation workflow.
You will be given a src UI design image and a processed image, where some ui elements are separated.
Here were the separation requirements:
```
{SHARED_SEPARATION_REQ}
```
You need to recognize and review the separation:
these node need handle:
"#
);
let mut result = binding_system_prompt;
result.reserve(512);
for elem in nodes {
result.push_str(&elem.as_prompt());
result.push('\n');
}
result
}
@@ -0,0 +1,158 @@
use super::model::*;
use crate::ui_editor::component::{image::ImageComponent, Component};
use crate::ui_editor::layout::node::Node;
use crate::ui_editor::state::{State, UITree};
use crate::ui_editor::utils::NodeId;
fn is_unbound_image(node: &Node) -> bool {
node.components.iter().any(|component| {
matches!(
component,
Component::Image(ImageComponent {
target_graphic: None,
..
})
)
})
}
fn node_pixel_rect(
node: &Node,
parent: &crate::ui_editor::layout::dimension::UIRect,
ppu: f32,
) -> (u32, u32, u32, u32) {
let rect = node.layout.transform.resolve(parent);
let x = (rect.min.x * ppu).max(0.0).round() as u32;
let y = (rect.min.y * ppu).max(0.0).round() as u32;
let w = (rect.size.x * ppu).max(0.0).round() as u32;
let h = (rect.size.y * ppu).max(0.0).round() as u32;
(x, y, w, h)
}
fn node_description(node: &Node) -> String {
let name = node.metadata.name.trim();
let description = node.metadata.description.trim();
match (name.is_empty(), description.is_empty()) {
(true, true) => "未命名 UI 图片元素".to_string(),
(false, true) => name.to_string(),
(true, false) => description.to_string(),
(false, false) => format!("{name}{description}"),
}
}
fn collect_todo_nodes(
node: &Node,
parent: &crate::ui_editor::layout::dimension::UIRect,
ppu: f32,
output: &mut Vec<SeparationNode>,
) {
let mut children = Vec::new();
let rect = node.layout.transform.resolve(parent);
for child in &node.children {
collect_todo_nodes(child, &rect, ppu, &mut children);
}
if is_unbound_image(node) {
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
output.push(SeparationNode {
id: node.id.clone(),
global_pos_x_px: x,
global_pos_y_px: y,
width_px: w,
height_px: h,
note: SeparationNote {
description: node_description(node),
text_note: String::new(),
},
children,
rework_count: 0,
});
} else {
output.extend(children);
}
}
pub fn construct_separation_state(state: &State) -> SeparationState {
let unprocessed_trees = state
.ui_trees
.iter()
.filter_map(|tree| {
let image = state.ui_design_images.get(&tree.src_ui_design)?;
let ppu = image.pixels_per_unit.get();
let size = image.pixel_size / ppu;
let root_rect =
crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size);
let mut children = Vec::new();
collect_todo_nodes(&tree.root, &root_rect, ppu, &mut children);
(!children.is_empty()).then(|| SeparationTree {
src_ui_design: tree.src_ui_design.clone(),
root: SeparationNode {
id: tree.root.id.clone(),
global_pos_x_px: 0,
global_pos_y_px: 0,
width_px: image.pixel_size.x.max(0.0).round() as u32,
height_px: image.pixel_size.y.max(0.0).round() as u32,
note: SeparationNote::default(),
children,
rework_count: 0,
},
})
})
.collect();
SeparationState {
schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(),
unprocessed_trees,
bound: Vec::new(),
problematic_nodes: Vec::new(),
}
}
pub fn next_leaf_batch(tree: &SeparationTree) -> Vec<&SeparationNode> {
fn leaves<'a>(node: &'a SeparationNode, output: &mut Vec<&'a SeparationNode>) {
if node.children.is_empty() {
output.push(node);
} else {
for child in &node.children {
leaves(child, output);
}
}
}
let mut output = Vec::new();
leaves(&tree.root, &mut output);
output
}
pub fn validate_binding_response(
response: &BindingResp,
batch: &[&SeparationNode],
) -> Result<(), String> {
let expected = batch
.iter()
.map(|node| node.id.clone())
.collect::<std::collections::HashSet<_>>();
let mut seen = std::collections::HashSet::new();
for decision in &response.decisions {
let node_id = match decision {
BindingDecision::Ok { to_node, .. } | BindingDecision::NeedRework { to_node, .. } => {
to_node
}
};
if !expected.contains(node_id) {
return Err(format!("视觉绑定返回了未知节点:{}", node_id.as_str()));
}
if !seen.insert(node_id.clone()) {
return Err(format!("视觉绑定重复返回节点:{}", node_id.as_str()));
}
if let BindingDecision::NeedRework {
problem_description,
..
} = decision
{
if problem_description.trim().is_empty() {
return Err("NeedRework 必须包含问题描述".to_string());
}
}
}
if seen.len() != expected.len() {
return Err("视觉绑定未覆盖当前 batch 的全部节点".to_string());
}
Ok(())
}
@@ -0,0 +1,420 @@
use super::model::*;
use super::prompt::{gen_binding_prompt, gen_extract_prompt};
use crate::config::{build_game_creator_llm_client_from_llm_config, load_game_creator_app_config};
use crate::platform_session::current_platform_session;
use crate::ui_editor::commands::separation::*;
use crate::ui_editor::commands::utils::{
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm,
request_with_feedback, strict_json_schema,
};
use crate::ui_editor::state::State;
use crate::ui_editor::utils::NodeId;
use base64::Engine as _;
use image::ImageFormat;
use platform_llm::{
LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice,
};
use serde::Deserialize;
use std::fs;
use std::path::Path;
pub fn apply_batch_patch(
state: &mut SeparationState,
tree_index: usize,
decisions: &[BindingDecision],
cut_paths: &std::collections::HashMap<NodeId, String>,
) -> Result<(), String> {
let tree = state
.unprocessed_trees
.get_mut(tree_index)
.ok_or_else(|| "separation tree 索引无效".to_string())?;
let batch = next_leaf_batch(tree);
validate_binding_response(
&BindingResp {
decisions: decisions.to_vec(),
},
&batch,
)?;
let rework_counts = batch
.iter()
.map(|node| (node.id.clone(), node.rework_count))
.collect::<std::collections::HashMap<_, _>>();
let mut ids = std::collections::HashSet::new();
for decision in decisions {
match decision {
BindingDecision::Ok { to_node, .. } => {
let path = cut_paths
.get(to_node)
.ok_or_else(|| format!("缺少节点 {} 的 cut 图片", to_node.as_str()))?;
state.bound.push(BoundNode {
node_id: to_node.clone(),
cut_image_path: path.clone(),
});
ids.insert(to_node.clone());
}
BindingDecision::NeedRework {
to_node,
problem_description,
} => {
let count = rework_counts.get(to_node).copied().unwrap_or(0) + 1;
if count >= MAX_REWORK_COUNT {
state.problematic_nodes.push(ProblematicNode {
node_id: to_node.clone(),
problem_description: problem_description.clone(),
rework_count: count,
});
ids.insert(to_node.clone());
} else {
increment_rework_count(&mut tree.root, to_node, count);
}
}
}
}
remove_ids(&mut tree.root, &ids);
Ok(())
}
fn mark_batch_problematic(
state: &mut SeparationState,
tree_index: usize,
batch: &[&SeparationNode],
error: String,
) {
let ids = batch
.iter()
.map(|node| node.id.clone())
.collect::<std::collections::HashSet<_>>();
for node in batch {
state.problematic_nodes.push(ProblematicNode {
node_id: node.id.clone(),
problem_description: error.clone(),
rework_count: MAX_REWORK_COUNT,
});
}
if let Some(tree) = state.unprocessed_trees.get_mut(tree_index) {
remove_ids(&mut tree.root, &ids);
}
}
fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) {
if node.id == *id {
node.rework_count = count;
return;
}
for child in &mut node.children {
increment_rework_count(child, id, count);
}
}
#[derive(Deserialize)]
struct RawEditResponse {
data: Vec<RawEditItem>,
}
#[derive(Deserialize)]
struct RawEditItem {
b64_json: String,
}
async fn raw_image_edit(
session: &crate::platform_session::PlatformSessionSnapshot,
image_data_url: &str,
prompt: &str,
width: u32,
height: u32,
) -> Result<String, String> {
let (mime, data) = image_data_url
.split_once(",")
.ok_or_else(|| "界面图 data URL 无效".to_string())?;
let mime = mime
.strip_prefix("data:")
.and_then(|v| v.strip_suffix(";base64"))
.unwrap_or("image/png");
let client = crate::http_client::agc_main_site_client_builder()
.build()
.map_err(|e| format!("创建图片编辑客户端失败:{e}"))?;
let url = format!(
"{}/api/raw/v1/images/edit",
session.api_base_url.trim_end_matches('/')
);
let body = serde_json::json!({
"image": {"data": data, "mimeType": mime},
"prompt": prompt,
"width": width,
"height": height,
"output_format": "png",
"background": "transparent"
});
let response = crate::http_client::with_agc_main_site_marker(
client
.post(url)
.bearer_auth(&session.access_token)
.json(&body),
)
.send()
.await
.map_err(|e| format!("图片分离请求失败:{e}"))?;
if !response.status().is_success() {
return Err(format!("图片分离请求失败(HTTP {}", response.status()));
}
let payload = response
.json::<RawEditResponse>()
.await
.map_err(|e| format!("解析图片分离响应失败:{e}"))?;
payload
.data
.into_iter()
.next()
.map(|item| format!("data:image/png;base64,{}", item.b64_json))
.ok_or_else(|| "图片分离响应没有图像".to_string())
}
fn build_marked_image(
source_url: &str,
nodes: &[&SeparationNode],
target: &Path,
) -> Result<String, String> {
let encoded = source_url
.split_once(',')
.map(|(_, d)| d)
.ok_or_else(|| "源图 data URL 无效".to_string())?;
let bytes = base64::engine::general_purpose::STANDARD
.decode(encoded)
.map_err(|e| format!("解码源图失败:{e}"))?;
let mut image = image::load_from_memory(&bytes)
.map_err(|e| format!("读取源图失败:{e}"))?
.to_rgba8();
let width = image.width();
let height = image.height();
for node in nodes {
let x0 = node.global_pos_x_px.min(width.saturating_sub(1));
let y0 = node.global_pos_y_px.min(height.saturating_sub(1));
let x1 = node
.global_pos_x_px
.saturating_add(node.width_px)
.min(width)
.saturating_sub(1);
let y1 = node
.global_pos_y_px
.saturating_add(node.height_px)
.min(height)
.saturating_sub(1);
if x0 >= x1 || y0 >= y1 {
continue;
}
for x in x0..=x1 {
image.put_pixel(x, y0, image::Rgba([0, 255, 0, 255]));
image.put_pixel(x, y1, image::Rgba([0, 255, 0, 255]));
}
for y in y0..=y1 {
image.put_pixel(x0, y, image::Rgba([0, 255, 0, 255]));
image.put_pixel(x1, y, image::Rgba([0, 255, 0, 255]));
}
for y in y0..=y1 {
for x in x0..=x1 {
if x > x0 && x < x1 && y > y0 && y < y1 {
image.put_pixel(x, y, image::Rgba([180, 0, 180, 120]));
}
}
}
}
image::DynamicImage::ImageRgba8(image.clone())
.save_with_format(target, image::ImageFormat::Png)
.map_err(|e| format!("写入标记图失败:{e}"))?;
let mut png = Vec::new();
image::DynamicImage::ImageRgba8(image)
.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
.map_err(|e| format!("编码标记图失败:{e}"))?;
Ok(format!(
"data:image/png;base64,{}",
base64::engine::general_purpose::STANDARD.encode(png)
))
}
async fn visual_binding(
source_url: String,
processed_url: String,
nodes: &[&SeparationNode],
) -> Result<BindingResp, String> {
let llm_config = load_game_creator_app_config()
.map_err(|e| e.to_string())?
.llm;
let client = build_game_creator_llm_client_from_llm_config(&llm_config, "llm")
.map_err(|e| e.to_string())?;
let schema = strict_json_schema::<BindingResp>()?;
let tool = LlmFunctionTool::new(
"bind_ui_elements",
"确认处理图中的区域对应哪些 UI 节点",
schema,
)
.with_strict(true);
let base_prompt = gen_binding_prompt(nodes.to_vec());
request_with_feedback(
2,
|feedback| {
let prompt = feedback.map_or_else(
|| base_prompt.clone(),
|error| format!("{base_prompt}\n上一次输出错误:{error}\n请修正并完整返回。"),
);
let source_url = source_url.clone();
let processed_url = processed_url.clone();
let tool = tool.clone();
let client = client.clone();
let llm_config = llm_config.clone();
async move {
let request = LlmRunRequest::new(vec![
LlmMessage::system("你是 UI 图片视觉绑定器。只根据图像判断区域,不做 OCR。"),
LlmMessage::user_multimodal(vec![
LlmMessageContentPart::InputText { text: prompt },
LlmMessageContentPart::InputImage {
image_url: source_url,
},
LlmMessageContentPart::InputImage {
image_url: processed_url,
},
]),
])
.with_function_tools(vec![tool.clone()])
.with_tool_choice(LlmToolChoice::Required);
request_ui_editor_llm(&client, &llm_config, request)
.await
.map_err(|e| e.to_string())
.and_then(|response| {
response
.tool_calls
.into_iter()
.find(|call| call.name == "bind_ui_elements")
.map(|call| call.arguments)
.ok_or_else(|| "视觉绑定模型未返回工具调用".to_string())
})
.and_then(|arguments| parse_limited_llm_tool_arguments(&arguments))
.and_then(|args| {
serde_json::from_value::<BindingResp>(args)
.map_err(|e| format!("视觉绑定结果无效:{e}"))
})
.and_then(|parsed| Ok(parsed))
}
},
|value: &BindingResp| validate_binding_response(value, nodes),
)
.await
}
pub(crate) async fn separate_ui_impl(
project_path: String,
asset_id: String,
state: State,
) -> Result<SeparationDTO, String> {
let session = current_platform_session().ok_or_else(|| "请先登录平台账号".to_string())?;
let root = Path::new(project_path.trim());
let sidecar = separation_sidecar_dir(root, &asset_id)?;
fs::create_dir_all(&sidecar).map_err(|e| format!("创建 separation sidecar 失败:{e}"))?;
let state_path = sidecar.join("state.json");
let mut separation = if state_path.exists() {
read_separation_state(&state_path)?
} else {
construct_separation_state(&state)
};
for (tree_index, tree) in separation.unprocessed_trees.clone().iter().enumerate() {
let image = state
.ui_design_images
.get(&tree.src_ui_design)
.ok_or_else(|| "缺少源界面图".to_string())?;
let source_path = crate::project::resolve_local_project_path(root, &image.path)?;
let source_url = read_ui_reference_image_data_url(source_path).await?;
write_separation_state(&state_path, &separation)?;
loop {
let Some(current_tree) = separation.unprocessed_trees.get(tree_index) else {
break;
};
let batch_nodes = next_leaf_batch(current_tree)
.into_iter()
.cloned()
.collect::<Vec<_>>();
if batch_nodes.is_empty() {
break;
}
let batch = batch_nodes.iter().collect::<Vec<_>>();
let prompt =
gen_extract_prompt(batch.iter().map(|n| n.note.clone()).collect::<Vec<_>>());
let marker_path = sidecar.join(format!("marked-{}.png", separation.bound.len()));
let marked_url = match build_marked_image(&source_url, &batch, &marker_path) {
Ok(value) => value,
Err(error) => {
mark_batch_problematic(&mut separation, tree_index, &batch, error);
write_separation_state(&state_path, &separation)?;
continue;
}
};
let processed_url = match raw_image_edit(
&session,
&marked_url,
&prompt,
image.pixel_size.x as u32,
image.pixel_size.y as u32,
)
.await
{
Ok(value) => value,
Err(error) => {
mark_batch_problematic(&mut separation, tree_index, &batch, error);
write_separation_state(&state_path, &separation)?;
continue;
}
};
let processed_path = sidecar.join(format!("processed-{}.png", separation.bound.len()));
let processed_bytes = base64::engine::general_purpose::STANDARD
.decode(
processed_url
.split_once(',')
.map(|(_, d)| d)
.unwrap_or_default(),
)
.map_err(|e| e.to_string())?;
fs::write(&processed_path, processed_bytes).map_err(|e| e.to_string())?;
let binding = match visual_binding(source_url.clone(), processed_url, &batch).await {
Ok(value) => value,
Err(error) => {
mark_batch_problematic(&mut separation, tree_index, &batch, error);
write_separation_state(&state_path, &separation)?;
continue;
}
};
let mut cut_paths = std::collections::HashMap::new();
for decision in &binding.decisions {
if let BindingDecision::Ok {
to_node,
separated_image_area,
} = decision
{
let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str()));
cut_processed_image(&processed_path, separated_image_area, &cut_path)?;
cut_paths.insert(to_node.clone(), cut_path.to_string_lossy().to_string());
}
}
apply_batch_patch(&mut separation, tree_index, &binding.decisions, &cut_paths)?;
write_separation_state(&state_path, &separation)?;
}
}
let _ = fs::remove_file(state_path);
Ok(separation_dto(&separation))
}
fn cut_processed_image(source: &Path, area: &BindingArea, target: &Path) -> Result<(), String> {
let image = image::open(source).map_err(|e| format!("读取处理图失败:{e}"))?;
area.validate_in(image.width(), image.height())?;
let cropped = image.crop_imm(
area.global_pos_x_px,
area.global_pos_y_px,
area.width_px,
area.height_px,
);
cropped
.save_with_format(target, ImageFormat::Png)
.map_err(|e| format!("写入 cut 图片失败:{e}"))
}
fn remove_ids(node: &mut SeparationNode, ids: &std::collections::HashSet<NodeId>) {
node.children.retain(|child| !ids.contains(&child.id));
for child in &mut node.children {
remove_ids(child, ids);
}
}
@@ -4,6 +4,7 @@ use base64::Engine as _;
use platform_llm::{LlmClient, LlmError, LlmRunRequest, LlmRunResponse};
use schemars::JsonSchema;
use std::fs::File;
use std::future::Future;
use std::io::Read;
use std::path::{Path, PathBuf};
@@ -25,6 +26,33 @@ pub(crate) async fn request_ui_editor_llm(
request_game_creator_llm_text(client, llm, request).await
}
/// 结构化 LLM 请求的小型 repair harness:第一次请求或校验失败后,
/// 将错误反馈给模型并只额外重试一次。网络/模型调用本身的错误也会
/// 进入第二次请求的反馈文本;调用方负责在第二次失败后决定业务状态。
pub(crate) async fn request_with_feedback<T, Request, Fut, Validate>(
more_turn: usize,
request: Request,
validate: Validate,
) -> Result<T, String>
where
Request: Fn(Option<String>) -> Fut,
Fut: Future<Output = Result<T, String>>,
Validate: Fn(&T) -> Result<(), String>,
{
let mut feedback = None;
for attempt in 0..=more_turn {
let result = request(feedback.clone())
.await
.and_then(|value| validate(&value).map(|_| value));
match result {
Ok(value) => return Ok(value),
Err(error) if attempt < more_turn => feedback = Some(error),
Err(error) => return Err(error),
}
}
unreachable!("repair harness always returns within requested turns")
}
pub(crate) fn parse_limited_llm_tool_arguments(
arguments: &str,
) -> Result<serde_json::Value, String> {
@@ -144,6 +172,27 @@ mod tests {
);
}
#[tokio::test]
async fn feedback_harness_zero_more_turn_calls_once_without_feedback() {
let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let seen = calls.clone();
let result = request_with_feedback(
0,
move |feedback| {
let seen = seen.clone();
async move {
seen.lock().unwrap().push(feedback);
Ok::<_, String>(serde_json::json!({"ok": true}))
}
},
|_| Ok(()),
)
.await
.expect("single turn should succeed");
assert_eq!(result, serde_json::json!({"ok": true}));
assert_eq!(calls.lock().unwrap().as_slice(), &[None]);
}
#[test]
fn reference_image_rejects_file_over_five_mib_before_reading() {
let directory = tempfile::tempdir().expect("reference image fixture");