WIP: UI编辑器自动分图层切图标 #304
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,140 @@
|
||||
mod model;
|
||||
mod persistence;
|
||||
mod prompt;
|
||||
mod tree;
|
||||
mod workflow;
|
||||
|
||||
pub use model::*;
|
||||
pub use persistence::*;
|
||||
pub use tree::*;
|
||||
pub(crate) use workflow::separate_ui_impl;
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ui_editor::component::image::{ImageComponent, ImageType};
|
||||
use crate::ui_editor::layout::children_display_mode::ChildrenDisplayMode;
|
||||
use crate::ui_editor::layout::control_layout::ControlLayout;
|
||||
use crate::ui_editor::layout::node::{NodeMetadata, NodeSource, StageStatus};
|
||||
use crate::ui_editor::resource::ui_design_image::UIDesignImage;
|
||||
use nalgebra::Vector2;
|
||||
use std::collections::HashMap;
|
||||
use typed_floats::tf32::StrictlyPositiveFinite;
|
||||
|
||||
fn node(id: &str, components: Vec<Component>, children: Vec<Node>) -> Node {
|
||||
Node {
|
||||
id: NodeId::new(id).unwrap(),
|
||||
layout: ControlLayout::default(),
|
||||
metadata: NodeMetadata {
|
||||
name: id.to_string(),
|
||||
description: String::new(),
|
||||
layout_status: StageStatus::NoProblem,
|
||||
components_status: StageStatus::NoProblem,
|
||||
allow_llm_edit_layout: true,
|
||||
allow_llm_edit_component: true,
|
||||
source: NodeSource::Llm,
|
||||
},
|
||||
components,
|
||||
children_display_mode: ChildrenDisplayMode::Stack,
|
||||
children,
|
||||
}
|
||||
}
|
||||
fn state(root: Node) -> State {
|
||||
let image_id = UIDesignImageId::new("page").unwrap();
|
||||
State {
|
||||
ui_trees: vec![UITree {
|
||||
src_ui_design: image_id.clone(),
|
||||
root,
|
||||
}],
|
||||
ui_design_images: HashMap::from([(
|
||||
image_id,
|
||||
UIDesignImage {
|
||||
metadata: crate::ui_editor::resource::ui_design_image::UIDesignImageMetadata {
|
||||
name: "page".to_string(),
|
||||
description: String::new(),
|
||||
role: None,
|
||||
slave_to: None,
|
||||
},
|
||||
path: "page.png".to_string(),
|
||||
pixel_size: Vector2::new(100.0, 100.0),
|
||||
pixels_per_unit: StrictlyPositiveFinite::new(1.0).unwrap(),
|
||||
},
|
||||
)]),
|
||||
sprite_assets: HashMap::new(),
|
||||
font_assets: HashMap::new(),
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn construction_filters_pure_nodes_and_passes_children_through() {
|
||||
let image = Component::Image(ImageComponent {
|
||||
target_graphic: None,
|
||||
image_type: ImageType::Simple {
|
||||
preserve_aspect: false,
|
||||
},
|
||||
});
|
||||
let root = node(
|
||||
"root",
|
||||
vec![],
|
||||
vec![node(
|
||||
"container",
|
||||
vec![],
|
||||
vec![node("image", vec![image], vec![])],
|
||||
)],
|
||||
);
|
||||
let result = construct_separation_state(&state(root));
|
||||
assert_eq!(
|
||||
result.unprocessed_trees[0].root.children[0].id.as_str(),
|
||||
"image"
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
fn binding_validation_requires_exact_batch_coverage() {
|
||||
let node = SeparationNode {
|
||||
id: NodeId::new("image").unwrap(),
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 1,
|
||||
height_px: 1,
|
||||
note: SeparationNote {
|
||||
description: "image".to_string(),
|
||||
text_note: String::new(),
|
||||
},
|
||||
children: vec![],
|
||||
rework_count: 0,
|
||||
};
|
||||
assert!(validate_binding_response(&BindingResp { decisions: vec![] }, &[&node]).is_err());
|
||||
}
|
||||
#[test]
|
||||
fn sidecar_name_uses_asset_id_digest() {
|
||||
let dir = separation_sidecar_dir(Path::new("/tmp/project"), "ui:1").unwrap();
|
||||
assert!(dir.to_string_lossy().contains("ui_1-"));
|
||||
assert!(dir.to_string_lossy().ends_with("-separation"));
|
||||
}
|
||||
#[test]
|
||||
fn patch_collects_bound_and_removes_leaf() {
|
||||
let image = Component::Image(ImageComponent {
|
||||
target_graphic: None,
|
||||
image_type: ImageType::Simple {
|
||||
preserve_aspect: false,
|
||||
},
|
||||
});
|
||||
let mut state = construct_separation_state(&state(node(
|
||||
"root",
|
||||
vec![],
|
||||
vec![node("image", vec![image], vec![])],
|
||||
)));
|
||||
let id = NodeId::new("image").unwrap();
|
||||
let decisions = vec![BindingDecision::Ok {
|
||||
to_node: id.clone(),
|
||||
separated_image_area: BindingArea {
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 1,
|
||||
height_px: 1,
|
||||
},
|
||||
}];
|
||||
let paths = HashMap::from([(id.clone(), "ui/.sidecar/cut.png".to_string())]);
|
||||
apply_batch_patch(&mut state, 0, &decisions, &paths).unwrap();
|
||||
assert_eq!(state.bound[0].node_id, id);
|
||||
assert!(state.unprocessed_trees[0].root.children.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
use crate::ui_editor::utils::{NodeId, UIDesignImageId};
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use ts_rs::TS;
|
||||
|
||||
pub const SEPARATION_STATE_SCHEMA_VERSION: &str = "ui-editor-separation-state.v1";
|
||||
pub const MAX_REWORK_COUNT: u32 = 3;
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct SeparationNote {
|
||||
pub description: String,
|
||||
pub text_note: String,
|
||||
}
|
||||
impl SeparationNote {
|
||||
pub fn as_prompt(&self) -> String {
|
||||
format!("desc: {} {}", self.description, self.text_note)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct SeparationNode {
|
||||
pub id: NodeId,
|
||||
pub global_pos_x_px: u32,
|
||||
pub global_pos_y_px: u32,
|
||||
pub width_px: u32,
|
||||
pub height_px: u32,
|
||||
pub note: SeparationNote,
|
||||
pub children: Vec<SeparationNode>,
|
||||
pub rework_count: u32,
|
||||
}
|
||||
impl SeparationNode {
|
||||
pub fn as_prompt(&self) -> String {
|
||||
format!(
|
||||
"node_id={} area=({}, {}, {}, {}) {}",
|
||||
self.id.as_str(),
|
||||
self.global_pos_x_px,
|
||||
self.global_pos_y_px,
|
||||
self.width_px,
|
||||
self.height_px,
|
||||
self.note.as_prompt()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct SeparationTree {
|
||||
pub src_ui_design: UIDesignImageId,
|
||||
pub root: SeparationNode,
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct BoundNode {
|
||||
pub node_id: NodeId,
|
||||
pub cut_image_path: String,
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct ProblematicNode {
|
||||
pub node_id: NodeId,
|
||||
pub problem_description: String,
|
||||
pub rework_count: u32,
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct SeparationState {
|
||||
pub schema_version: String,
|
||||
pub unprocessed_trees: Vec<SeparationTree>,
|
||||
pub bound: Vec<BoundNode>,
|
||||
pub problematic_nodes: Vec<ProblematicNode>,
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, TS)]
|
||||
#[ts(export, export_to = concat!(env!("CARGO_MANIFEST_DIR"), "/../src/features/ui-editor/types/"))]
|
||||
pub struct SeparationDTO {
|
||||
pub bound_nodes: Vec<BoundNode>,
|
||||
pub problematic_nodes: Vec<ProblematicNode>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
pub struct BindingArea {
|
||||
pub global_pos_x_px: u32,
|
||||
pub global_pos_y_px: u32,
|
||||
pub width_px: u32,
|
||||
pub height_px: u32,
|
||||
}
|
||||
impl BindingArea {
|
||||
pub fn validate_in(&self, w: u32, h: u32) -> Result<(), String> {
|
||||
if self.width_px == 0 || self.height_px == 0 {
|
||||
return Err("BindingArea 宽度和高度必须大于 0".into());
|
||||
}
|
||||
if self
|
||||
.global_pos_x_px
|
||||
.checked_add(self.width_px)
|
||||
.is_none_or(|v| v > w)
|
||||
|| self
|
||||
.global_pos_y_px
|
||||
.checked_add(self.height_px)
|
||||
.is_none_or(|v| v > h)
|
||||
{
|
||||
return Err("BindingArea 超出处理图边界".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
pub enum BindingDecision {
|
||||
Ok {
|
||||
separated_image_area: BindingArea,
|
||||
to_node: NodeId,
|
||||
},
|
||||
NeedRework {
|
||||
problem_description: String,
|
||||
to_node: NodeId,
|
||||
},
|
||||
}
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)]
|
||||
pub struct BindingResp {
|
||||
pub decisions: Vec<BindingDecision>,
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
use super::model::*;
|
||||
use crate::ui_editor::commands::separation::*;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
pub fn separation_sidecar_dir(root: &Path, asset_id: &str) -> Result<PathBuf, String> {
|
||||
if asset_id.trim().is_empty() || asset_id.trim() != asset_id {
|
||||
return Err("UI 资源 ID 无效".to_string());
|
||||
}
|
||||
let dir = root.join("ui").join(format!(
|
||||
".{}-separation",
|
||||
crate::ui_editor::persistence::generated_file_stem(asset_id)
|
||||
));
|
||||
if !dir.starts_with(root) {
|
||||
return Err("separation sidecar 路径越界".to_string());
|
||||
}
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
pub fn write_separation_state(path: &Path, state: &SeparationState) -> Result<(), String> {
|
||||
if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION {
|
||||
return Err("不支持的 separation state schema".to_string());
|
||||
}
|
||||
let bytes = serde_json::to_vec_pretty(state)
|
||||
.map_err(|error| format!("序列化 separation state 失败:{error}"))?;
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| "separation state 路径缺少父目录".to_string())?;
|
||||
fs::create_dir_all(parent).map_err(|error| format!("创建 separation sidecar 失败:{error}"))?;
|
||||
let temporary = path.with_extension("json.tmp");
|
||||
fs::write(&temporary, bytes).map_err(|error| format!("写入 separation state 失败:{error}"))?;
|
||||
fs::rename(&temporary, path).map_err(|error| format!("安装 separation state 失败:{error}"))
|
||||
}
|
||||
|
||||
pub fn read_separation_state(path: &Path) -> Result<SeparationState, String> {
|
||||
let bytes = fs::read(path).map_err(|error| format!("读取 separation state 失败:{error}"))?;
|
||||
let state: SeparationState = serde_json::from_slice(&bytes)
|
||||
.map_err(|error| format!("解析 separation state 失败:{error}"))?;
|
||||
if state.schema_version != SEPARATION_STATE_SCHEMA_VERSION {
|
||||
return Err("不支持的 separation state schema".to_string());
|
||||
}
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
pub fn separation_dto(state: &SeparationState) -> SeparationDTO {
|
||||
SeparationDTO {
|
||||
bound_nodes: state.bound.clone(),
|
||||
problematic_nodes: state.problematic_nodes.clone(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
use crate::ui_editor::commands::separation::{SeparationNode, SeparationNote};
|
||||
|
||||
const SHARED_SEPARATION_REQ: &str = r#"
|
||||
|
||||
MUST hard edges; preserve no glow/blur beyond the exact visible shape.
|
||||
NEVER keep its parent's background with it.
|
||||
|
||||
UI elements that needs to extract has been marked with GREEN line frames (only for mark purpose, NEVER wrap a frame in your extraction).
|
||||
On some UI elements, there is some PURPLE filled area, they were removed UI elements, reconstruct the background under where they were.
|
||||
"#;
|
||||
pub(super) fn gen_extract_prompt(separation_notes: Vec<SeparationNote>) -> String {
|
||||
let extract_system_prompt = format!(
|
||||
r#"
|
||||
This is a UI design image, not a normal photo/illustration. Extract it strictly as UI elements/layers, not as a generic foreground/background extraction.
|
||||
Treat distinct UI element as its own layer with hard, clean, pixel-accurate edges and full transparency outside the element.
|
||||
|
||||
MUST keep each element at its original position on a transparent canvas.
|
||||
{SHARED_SEPARATION_REQ}
|
||||
here are UI elements to extract:
|
||||
|
||||
"#
|
||||
);
|
||||
let mut result = extract_system_prompt;
|
||||
result.reserve(512);
|
||||
for elem in separation_notes {
|
||||
result.push_str(&elem.as_prompt());
|
||||
result.push('\n');
|
||||
}
|
||||
result
|
||||
}
|
||||
pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String {
|
||||
let binding_system_prompt = format!(
|
||||
r#"
|
||||
You are working under a UI elements separation workflow.
|
||||
You will be given a src UI design image and a processed image, where some ui elements are separated.
|
||||
Here were the separation requirements:
|
||||
```
|
||||
{SHARED_SEPARATION_REQ}
|
||||
```
|
||||
You need to recognize and review the separation:
|
||||
|
||||
these node need handle:
|
||||
"#
|
||||
);
|
||||
let mut result = binding_system_prompt;
|
||||
result.reserve(512);
|
||||
for elem in nodes {
|
||||
result.push_str(&elem.as_prompt());
|
||||
result.push('\n');
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
use super::model::*;
|
||||
use crate::ui_editor::component::{image::ImageComponent, Component};
|
||||
use crate::ui_editor::layout::node::Node;
|
||||
use crate::ui_editor::state::{State, UITree};
|
||||
use crate::ui_editor::utils::NodeId;
|
||||
fn is_unbound_image(node: &Node) -> bool {
|
||||
node.components.iter().any(|component| {
|
||||
matches!(
|
||||
component,
|
||||
Component::Image(ImageComponent {
|
||||
target_graphic: None,
|
||||
..
|
||||
})
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn node_pixel_rect(
|
||||
node: &Node,
|
||||
parent: &crate::ui_editor::layout::dimension::UIRect,
|
||||
ppu: f32,
|
||||
) -> (u32, u32, u32, u32) {
|
||||
let rect = node.layout.transform.resolve(parent);
|
||||
let x = (rect.min.x * ppu).max(0.0).round() as u32;
|
||||
let y = (rect.min.y * ppu).max(0.0).round() as u32;
|
||||
let w = (rect.size.x * ppu).max(0.0).round() as u32;
|
||||
let h = (rect.size.y * ppu).max(0.0).round() as u32;
|
||||
(x, y, w, h)
|
||||
}
|
||||
|
||||
fn node_description(node: &Node) -> String {
|
||||
let name = node.metadata.name.trim();
|
||||
let description = node.metadata.description.trim();
|
||||
match (name.is_empty(), description.is_empty()) {
|
||||
(true, true) => "未命名 UI 图片元素".to_string(),
|
||||
(false, true) => name.to_string(),
|
||||
(true, false) => description.to_string(),
|
||||
(false, false) => format!("{name}:{description}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_todo_nodes(
|
||||
node: &Node,
|
||||
parent: &crate::ui_editor::layout::dimension::UIRect,
|
||||
ppu: f32,
|
||||
output: &mut Vec<SeparationNode>,
|
||||
) {
|
||||
let mut children = Vec::new();
|
||||
let rect = node.layout.transform.resolve(parent);
|
||||
for child in &node.children {
|
||||
collect_todo_nodes(child, &rect, ppu, &mut children);
|
||||
}
|
||||
if is_unbound_image(node) {
|
||||
let (x, y, w, h) = node_pixel_rect(node, parent, ppu);
|
||||
output.push(SeparationNode {
|
||||
id: node.id.clone(),
|
||||
global_pos_x_px: x,
|
||||
global_pos_y_px: y,
|
||||
width_px: w,
|
||||
height_px: h,
|
||||
note: SeparationNote {
|
||||
description: node_description(node),
|
||||
text_note: String::new(),
|
||||
},
|
||||
children,
|
||||
rework_count: 0,
|
||||
});
|
||||
} else {
|
||||
output.extend(children);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn construct_separation_state(state: &State) -> SeparationState {
|
||||
let unprocessed_trees = state
|
||||
.ui_trees
|
||||
.iter()
|
||||
.filter_map(|tree| {
|
||||
let image = state.ui_design_images.get(&tree.src_ui_design)?;
|
||||
let ppu = image.pixels_per_unit.get();
|
||||
let size = image.pixel_size / ppu;
|
||||
let root_rect =
|
||||
crate::ui_editor::layout::dimension::UIRect::new(nalgebra::Point2::origin(), size);
|
||||
let mut children = Vec::new();
|
||||
collect_todo_nodes(&tree.root, &root_rect, ppu, &mut children);
|
||||
(!children.is_empty()).then(|| SeparationTree {
|
||||
src_ui_design: tree.src_ui_design.clone(),
|
||||
root: SeparationNode {
|
||||
id: tree.root.id.clone(),
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: image.pixel_size.x.max(0.0).round() as u32,
|
||||
height_px: image.pixel_size.y.max(0.0).round() as u32,
|
||||
note: SeparationNote::default(),
|
||||
children,
|
||||
rework_count: 0,
|
||||
},
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
SeparationState {
|
||||
schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(),
|
||||
unprocessed_trees,
|
||||
bound: Vec::new(),
|
||||
problematic_nodes: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn next_leaf_batch(tree: &SeparationTree) -> Vec<&SeparationNode> {
|
||||
fn leaves<'a>(node: &'a SeparationNode, output: &mut Vec<&'a SeparationNode>) {
|
||||
if node.children.is_empty() {
|
||||
output.push(node);
|
||||
} else {
|
||||
for child in &node.children {
|
||||
leaves(child, output);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
leaves(&tree.root, &mut output);
|
||||
output
|
||||
}
|
||||
|
||||
pub fn validate_binding_response(
|
||||
response: &BindingResp,
|
||||
batch: &[&SeparationNode],
|
||||
) -> Result<(), String> {
|
||||
let expected = batch
|
||||
.iter()
|
||||
.map(|node| node.id.clone())
|
||||
.collect::<std::collections::HashSet<_>>();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for decision in &response.decisions {
|
||||
let node_id = match decision {
|
||||
BindingDecision::Ok { to_node, .. } | BindingDecision::NeedRework { to_node, .. } => {
|
||||
to_node
|
||||
}
|
||||
};
|
||||
if !expected.contains(node_id) {
|
||||
return Err(format!("视觉绑定返回了未知节点:{}", node_id.as_str()));
|
||||
}
|
||||
if !seen.insert(node_id.clone()) {
|
||||
return Err(format!("视觉绑定重复返回节点:{}", node_id.as_str()));
|
||||
}
|
||||
if let BindingDecision::NeedRework {
|
||||
problem_description,
|
||||
..
|
||||
} = decision
|
||||
{
|
||||
if problem_description.trim().is_empty() {
|
||||
return Err("NeedRework 必须包含问题描述".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
if seen.len() != expected.len() {
|
||||
return Err("视觉绑定未覆盖当前 batch 的全部节点".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,420 @@
|
||||
use super::model::*;
|
||||
use super::prompt::{gen_binding_prompt, gen_extract_prompt};
|
||||
use crate::config::{build_game_creator_llm_client_from_llm_config, load_game_creator_app_config};
|
||||
use crate::platform_session::current_platform_session;
|
||||
use crate::ui_editor::commands::separation::*;
|
||||
use crate::ui_editor::commands::utils::{
|
||||
parse_limited_llm_tool_arguments, read_ui_reference_image_data_url, request_ui_editor_llm,
|
||||
request_with_feedback, strict_json_schema,
|
||||
};
|
||||
use crate::ui_editor::state::State;
|
||||
use crate::ui_editor::utils::NodeId;
|
||||
use base64::Engine as _;
|
||||
use image::ImageFormat;
|
||||
use platform_llm::{
|
||||
LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
pub fn apply_batch_patch(
|
||||
state: &mut SeparationState,
|
||||
tree_index: usize,
|
||||
decisions: &[BindingDecision],
|
||||
cut_paths: &std::collections::HashMap<NodeId, String>,
|
||||
) -> Result<(), String> {
|
||||
let tree = state
|
||||
.unprocessed_trees
|
||||
.get_mut(tree_index)
|
||||
.ok_or_else(|| "separation tree 索引无效".to_string())?;
|
||||
let batch = next_leaf_batch(tree);
|
||||
validate_binding_response(
|
||||
&BindingResp {
|
||||
decisions: decisions.to_vec(),
|
||||
},
|
||||
&batch,
|
||||
)?;
|
||||
let rework_counts = batch
|
||||
.iter()
|
||||
.map(|node| (node.id.clone(), node.rework_count))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
let mut ids = std::collections::HashSet::new();
|
||||
for decision in decisions {
|
||||
match decision {
|
||||
BindingDecision::Ok { to_node, .. } => {
|
||||
let path = cut_paths
|
||||
.get(to_node)
|
||||
.ok_or_else(|| format!("缺少节点 {} 的 cut 图片", to_node.as_str()))?;
|
||||
state.bound.push(BoundNode {
|
||||
node_id: to_node.clone(),
|
||||
cut_image_path: path.clone(),
|
||||
});
|
||||
ids.insert(to_node.clone());
|
||||
}
|
||||
BindingDecision::NeedRework {
|
||||
to_node,
|
||||
problem_description,
|
||||
} => {
|
||||
let count = rework_counts.get(to_node).copied().unwrap_or(0) + 1;
|
||||
if count >= MAX_REWORK_COUNT {
|
||||
state.problematic_nodes.push(ProblematicNode {
|
||||
node_id: to_node.clone(),
|
||||
problem_description: problem_description.clone(),
|
||||
rework_count: count,
|
||||
});
|
||||
ids.insert(to_node.clone());
|
||||
} else {
|
||||
increment_rework_count(&mut tree.root, to_node, count);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
remove_ids(&mut tree.root, &ids);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mark_batch_problematic(
|
||||
state: &mut SeparationState,
|
||||
tree_index: usize,
|
||||
batch: &[&SeparationNode],
|
||||
error: String,
|
||||
) {
|
||||
let ids = batch
|
||||
.iter()
|
||||
.map(|node| node.id.clone())
|
||||
.collect::<std::collections::HashSet<_>>();
|
||||
for node in batch {
|
||||
state.problematic_nodes.push(ProblematicNode {
|
||||
node_id: node.id.clone(),
|
||||
problem_description: error.clone(),
|
||||
rework_count: MAX_REWORK_COUNT,
|
||||
});
|
||||
}
|
||||
if let Some(tree) = state.unprocessed_trees.get_mut(tree_index) {
|
||||
remove_ids(&mut tree.root, &ids);
|
||||
}
|
||||
}
|
||||
|
||||
fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) {
|
||||
if node.id == *id {
|
||||
node.rework_count = count;
|
||||
return;
|
||||
}
|
||||
for child in &mut node.children {
|
||||
increment_rework_count(child, id, count);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RawEditResponse {
|
||||
data: Vec<RawEditItem>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct RawEditItem {
|
||||
b64_json: String,
|
||||
}
|
||||
|
||||
async fn raw_image_edit(
|
||||
session: &crate::platform_session::PlatformSessionSnapshot,
|
||||
image_data_url: &str,
|
||||
prompt: &str,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<String, String> {
|
||||
let (mime, data) = image_data_url
|
||||
.split_once(",")
|
||||
.ok_or_else(|| "界面图 data URL 无效".to_string())?;
|
||||
let mime = mime
|
||||
.strip_prefix("data:")
|
||||
.and_then(|v| v.strip_suffix(";base64"))
|
||||
.unwrap_or("image/png");
|
||||
let client = crate::http_client::agc_main_site_client_builder()
|
||||
.build()
|
||||
.map_err(|e| format!("创建图片编辑客户端失败:{e}"))?;
|
||||
let url = format!(
|
||||
"{}/api/raw/v1/images/edit",
|
||||
session.api_base_url.trim_end_matches('/')
|
||||
);
|
||||
let body = serde_json::json!({
|
||||
"image": {"data": data, "mimeType": mime},
|
||||
"prompt": prompt,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"output_format": "png",
|
||||
"background": "transparent"
|
||||
});
|
||||
let response = crate::http_client::with_agc_main_site_marker(
|
||||
client
|
||||
.post(url)
|
||||
.bearer_auth(&session.access_token)
|
||||
.json(&body),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("图片分离请求失败:{e}"))?;
|
||||
if !response.status().is_success() {
|
||||
return Err(format!("图片分离请求失败(HTTP {})", response.status()));
|
||||
}
|
||||
let payload = response
|
||||
.json::<RawEditResponse>()
|
||||
.await
|
||||
.map_err(|e| format!("解析图片分离响应失败:{e}"))?;
|
||||
payload
|
||||
.data
|
||||
.into_iter()
|
||||
.next()
|
||||
.map(|item| format!("data:image/png;base64,{}", item.b64_json))
|
||||
.ok_or_else(|| "图片分离响应没有图像".to_string())
|
||||
}
|
||||
|
||||
fn build_marked_image(
|
||||
source_url: &str,
|
||||
nodes: &[&SeparationNode],
|
||||
target: &Path,
|
||||
) -> Result<String, String> {
|
||||
let encoded = source_url
|
||||
.split_once(',')
|
||||
.map(|(_, d)| d)
|
||||
.ok_or_else(|| "源图 data URL 无效".to_string())?;
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(encoded)
|
||||
.map_err(|e| format!("解码源图失败:{e}"))?;
|
||||
let mut image = image::load_from_memory(&bytes)
|
||||
.map_err(|e| format!("读取源图失败:{e}"))?
|
||||
.to_rgba8();
|
||||
let width = image.width();
|
||||
let height = image.height();
|
||||
for node in nodes {
|
||||
let x0 = node.global_pos_x_px.min(width.saturating_sub(1));
|
||||
let y0 = node.global_pos_y_px.min(height.saturating_sub(1));
|
||||
let x1 = node
|
||||
.global_pos_x_px
|
||||
.saturating_add(node.width_px)
|
||||
.min(width)
|
||||
.saturating_sub(1);
|
||||
let y1 = node
|
||||
.global_pos_y_px
|
||||
.saturating_add(node.height_px)
|
||||
.min(height)
|
||||
.saturating_sub(1);
|
||||
if x0 >= x1 || y0 >= y1 {
|
||||
continue;
|
||||
}
|
||||
for x in x0..=x1 {
|
||||
image.put_pixel(x, y0, image::Rgba([0, 255, 0, 255]));
|
||||
image.put_pixel(x, y1, image::Rgba([0, 255, 0, 255]));
|
||||
}
|
||||
for y in y0..=y1 {
|
||||
image.put_pixel(x0, y, image::Rgba([0, 255, 0, 255]));
|
||||
image.put_pixel(x1, y, image::Rgba([0, 255, 0, 255]));
|
||||
}
|
||||
for y in y0..=y1 {
|
||||
for x in x0..=x1 {
|
||||
if x > x0 && x < x1 && y > y0 && y < y1 {
|
||||
image.put_pixel(x, y, image::Rgba([180, 0, 180, 120]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
image::DynamicImage::ImageRgba8(image.clone())
|
||||
.save_with_format(target, image::ImageFormat::Png)
|
||||
.map_err(|e| format!("写入标记图失败:{e}"))?;
|
||||
let mut png = Vec::new();
|
||||
image::DynamicImage::ImageRgba8(image)
|
||||
.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
|
||||
.map_err(|e| format!("编码标记图失败:{e}"))?;
|
||||
Ok(format!(
|
||||
"data:image/png;base64,{}",
|
||||
base64::engine::general_purpose::STANDARD.encode(png)
|
||||
))
|
||||
}
|
||||
|
||||
async fn visual_binding(
|
||||
source_url: String,
|
||||
processed_url: String,
|
||||
nodes: &[&SeparationNode],
|
||||
) -> Result<BindingResp, String> {
|
||||
let llm_config = load_game_creator_app_config()
|
||||
.map_err(|e| e.to_string())?
|
||||
.llm;
|
||||
let client = build_game_creator_llm_client_from_llm_config(&llm_config, "llm")
|
||||
.map_err(|e| e.to_string())?;
|
||||
let schema = strict_json_schema::<BindingResp>()?;
|
||||
let tool = LlmFunctionTool::new(
|
||||
"bind_ui_elements",
|
||||
"确认处理图中的区域对应哪些 UI 节点",
|
||||
schema,
|
||||
)
|
||||
.with_strict(true);
|
||||
let base_prompt = gen_binding_prompt(nodes.to_vec());
|
||||
request_with_feedback(
|
||||
2,
|
||||
|feedback| {
|
||||
let prompt = feedback.map_or_else(
|
||||
|| base_prompt.clone(),
|
||||
|error| format!("{base_prompt}\n上一次输出错误:{error}\n请修正并完整返回。"),
|
||||
);
|
||||
let source_url = source_url.clone();
|
||||
let processed_url = processed_url.clone();
|
||||
let tool = tool.clone();
|
||||
let client = client.clone();
|
||||
let llm_config = llm_config.clone();
|
||||
async move {
|
||||
let request = LlmRunRequest::new(vec![
|
||||
LlmMessage::system("你是 UI 图片视觉绑定器。只根据图像判断区域,不做 OCR。"),
|
||||
LlmMessage::user_multimodal(vec![
|
||||
LlmMessageContentPart::InputText { text: prompt },
|
||||
LlmMessageContentPart::InputImage {
|
||||
image_url: source_url,
|
||||
},
|
||||
LlmMessageContentPart::InputImage {
|
||||
image_url: processed_url,
|
||||
},
|
||||
]),
|
||||
])
|
||||
.with_function_tools(vec![tool.clone()])
|
||||
.with_tool_choice(LlmToolChoice::Required);
|
||||
request_ui_editor_llm(&client, &llm_config, request)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|response| {
|
||||
response
|
||||
.tool_calls
|
||||
.into_iter()
|
||||
.find(|call| call.name == "bind_ui_elements")
|
||||
.map(|call| call.arguments)
|
||||
.ok_or_else(|| "视觉绑定模型未返回工具调用".to_string())
|
||||
})
|
||||
.and_then(|arguments| parse_limited_llm_tool_arguments(&arguments))
|
||||
.and_then(|args| {
|
||||
serde_json::from_value::<BindingResp>(args)
|
||||
.map_err(|e| format!("视觉绑定结果无效:{e}"))
|
||||
})
|
||||
.and_then(|parsed| Ok(parsed))
|
||||
}
|
||||
},
|
||||
|value: &BindingResp| validate_binding_response(value, nodes),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn separate_ui_impl(
|
||||
project_path: String,
|
||||
asset_id: String,
|
||||
state: State,
|
||||
) -> Result<SeparationDTO, String> {
|
||||
let session = current_platform_session().ok_or_else(|| "请先登录平台账号".to_string())?;
|
||||
let root = Path::new(project_path.trim());
|
||||
let sidecar = separation_sidecar_dir(root, &asset_id)?;
|
||||
fs::create_dir_all(&sidecar).map_err(|e| format!("创建 separation sidecar 失败:{e}"))?;
|
||||
let state_path = sidecar.join("state.json");
|
||||
let mut separation = if state_path.exists() {
|
||||
read_separation_state(&state_path)?
|
||||
} else {
|
||||
construct_separation_state(&state)
|
||||
};
|
||||
for (tree_index, tree) in separation.unprocessed_trees.clone().iter().enumerate() {
|
||||
let image = state
|
||||
.ui_design_images
|
||||
.get(&tree.src_ui_design)
|
||||
.ok_or_else(|| "缺少源界面图".to_string())?;
|
||||
let source_path = crate::project::resolve_local_project_path(root, &image.path)?;
|
||||
let source_url = read_ui_reference_image_data_url(source_path).await?;
|
||||
write_separation_state(&state_path, &separation)?;
|
||||
loop {
|
||||
let Some(current_tree) = separation.unprocessed_trees.get(tree_index) else {
|
||||
break;
|
||||
};
|
||||
let batch_nodes = next_leaf_batch(current_tree)
|
||||
.into_iter()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
if batch_nodes.is_empty() {
|
||||
break;
|
||||
}
|
||||
let batch = batch_nodes.iter().collect::<Vec<_>>();
|
||||
let prompt =
|
||||
gen_extract_prompt(batch.iter().map(|n| n.note.clone()).collect::<Vec<_>>());
|
||||
let marker_path = sidecar.join(format!("marked-{}.png", separation.bound.len()));
|
||||
let marked_url = match build_marked_image(&source_url, &batch, &marker_path) {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
mark_batch_problematic(&mut separation, tree_index, &batch, error);
|
||||
write_separation_state(&state_path, &separation)?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let processed_url = match raw_image_edit(
|
||||
&session,
|
||||
&marked_url,
|
||||
&prompt,
|
||||
image.pixel_size.x as u32,
|
||||
image.pixel_size.y as u32,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
mark_batch_problematic(&mut separation, tree_index, &batch, error);
|
||||
write_separation_state(&state_path, &separation)?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let processed_path = sidecar.join(format!("processed-{}.png", separation.bound.len()));
|
||||
let processed_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(
|
||||
processed_url
|
||||
.split_once(',')
|
||||
.map(|(_, d)| d)
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
fs::write(&processed_path, processed_bytes).map_err(|e| e.to_string())?;
|
||||
let binding = match visual_binding(source_url.clone(), processed_url, &batch).await {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
mark_batch_problematic(&mut separation, tree_index, &batch, error);
|
||||
write_separation_state(&state_path, &separation)?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let mut cut_paths = std::collections::HashMap::new();
|
||||
for decision in &binding.decisions {
|
||||
if let BindingDecision::Ok {
|
||||
to_node,
|
||||
separated_image_area,
|
||||
} = decision
|
||||
{
|
||||
let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str()));
|
||||
cut_processed_image(&processed_path, separated_image_area, &cut_path)?;
|
||||
cut_paths.insert(to_node.clone(), cut_path.to_string_lossy().to_string());
|
||||
}
|
||||
}
|
||||
apply_batch_patch(&mut separation, tree_index, &binding.decisions, &cut_paths)?;
|
||||
write_separation_state(&state_path, &separation)?;
|
||||
}
|
||||
}
|
||||
let _ = fs::remove_file(state_path);
|
||||
Ok(separation_dto(&separation))
|
||||
}
|
||||
|
||||
fn cut_processed_image(source: &Path, area: &BindingArea, target: &Path) -> Result<(), String> {
|
||||
let image = image::open(source).map_err(|e| format!("读取处理图失败:{e}"))?;
|
||||
area.validate_in(image.width(), image.height())?;
|
||||
let cropped = image.crop_imm(
|
||||
area.global_pos_x_px,
|
||||
area.global_pos_y_px,
|
||||
area.width_px,
|
||||
area.height_px,
|
||||
);
|
||||
cropped
|
||||
.save_with_format(target, ImageFormat::Png)
|
||||
.map_err(|e| format!("写入 cut 图片失败:{e}"))
|
||||
}
|
||||
|
||||
fn remove_ids(node: &mut SeparationNode, ids: &std::collections::HashSet<NodeId>) {
|
||||
node.children.retain(|child| !ids.contains(&child.id));
|
||||
for child in &mut node.children {
|
||||
remove_ids(child, ids);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ use base64::Engine as _;
|
||||
use platform_llm::{LlmClient, LlmError, LlmRunRequest, LlmRunResponse};
|
||||
use schemars::JsonSchema;
|
||||
use std::fs::File;
|
||||
use std::future::Future;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
@@ -25,6 +26,33 @@ pub(crate) async fn request_ui_editor_llm(
|
||||
request_game_creator_llm_text(client, llm, request).await
|
||||
}
|
||||
|
||||
/// 结构化 LLM 请求的小型 repair harness:第一次请求或校验失败后,
|
||||
/// 将错误反馈给模型并只额外重试一次。网络/模型调用本身的错误也会
|
||||
/// 进入第二次请求的反馈文本;调用方负责在第二次失败后决定业务状态。
|
||||
pub(crate) async fn request_with_feedback<T, Request, Fut, Validate>(
|
||||
more_turn: usize,
|
||||
request: Request,
|
||||
validate: Validate,
|
||||
) -> Result<T, String>
|
||||
where
|
||||
Request: Fn(Option<String>) -> Fut,
|
||||
Fut: Future<Output = Result<T, String>>,
|
||||
Validate: Fn(&T) -> Result<(), String>,
|
||||
{
|
||||
let mut feedback = None;
|
||||
for attempt in 0..=more_turn {
|
||||
let result = request(feedback.clone())
|
||||
.await
|
||||
.and_then(|value| validate(&value).map(|_| value));
|
||||
match result {
|
||||
Ok(value) => return Ok(value),
|
||||
Err(error) if attempt < more_turn => feedback = Some(error),
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
unreachable!("repair harness always returns within requested turns")
|
||||
}
|
||||
|
||||
pub(crate) fn parse_limited_llm_tool_arguments(
|
||||
arguments: &str,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
@@ -144,6 +172,27 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feedback_harness_zero_more_turn_calls_once_without_feedback() {
|
||||
let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let seen = calls.clone();
|
||||
let result = request_with_feedback(
|
||||
0,
|
||||
move |feedback| {
|
||||
let seen = seen.clone();
|
||||
async move {
|
||||
seen.lock().unwrap().push(feedback);
|
||||
Ok::<_, String>(serde_json::json!({"ok": true}))
|
||||
}
|
||||
},
|
||||
|_| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("single turn should succeed");
|
||||
assert_eq!(result, serde_json::json!({"ok": true}));
|
||||
assert_eq!(calls.lock().unwrap().as_slice(), &[None]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reference_image_rejects_file_over_five_mib_before_reading() {
|
||||
let directory = tempfile::tempdir().expect("reference image fixture");
|
||||
|
||||
Reference in New Issue
Block a user