重构UI编辑器素材切分工作流模块
拆分提取与绑定提示词模块 拆分图片编辑、视觉绑定、裁切和批次 patch 实现 同步技术方案实现组织说明并更新提示词测试
This commit is contained in:
@@ -254,11 +254,32 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn next_extract_prompt_contains_previous_rework_notes() {
|
||||
let note = SeparationNote {
|
||||
description: "图标".to_string(),
|
||||
rework_notes: vec!["不要带父背景".to_string()],
|
||||
let node = SeparationNode {
|
||||
id: NodeId::new("image").unwrap(),
|
||||
kind: SeparationNodeKind::ImageTarget,
|
||||
global_pos_x_px: 0,
|
||||
global_pos_y_px: 0,
|
||||
width_px: 1,
|
||||
height_px: 1,
|
||||
note: SeparationNote {
|
||||
description: "图标".to_string(),
|
||||
rework_notes: vec!["不要带父背景".to_string()],
|
||||
},
|
||||
children: vec![],
|
||||
rework_count: 0,
|
||||
};
|
||||
let prompt = super::prompt::gen_extract_prompt(vec![note]);
|
||||
let tree = SeparationTree {
|
||||
src_ui_design: UIDesignImageId::new("page").unwrap(),
|
||||
root: node.clone(),
|
||||
root_extractable: true,
|
||||
};
|
||||
let separation = SeparationState {
|
||||
schema_version: SEPARATION_STATE_SCHEMA_VERSION.to_string(),
|
||||
trees: vec![tree.clone()],
|
||||
bound: vec![],
|
||||
problematic_nodes: vec![],
|
||||
};
|
||||
let prompt = super::prompt::gen_extract_prompt(&separation, &tree, &[&node]);
|
||||
assert!(prompt.contains("previous rework notes:\n- 不要带父背景"));
|
||||
}
|
||||
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
use crate::ui_editor::commands::separation::{SeparationNode, SeparationNote};
|
||||
|
||||
const SHARED_SEPARATION_REQ: &str = r#"
|
||||
|
||||
MUST hard edges; preserve no glow/blur beyond the exact visible shape.
|
||||
NEVER keep its parent's background with it.
|
||||
NEVER include any text unless requested.
|
||||
|
||||
UI elements that needs to extract has been marked with GREEN line frames box with crossline inside. (only for mark purpose, NEVER wrap a frame in your extraction).
|
||||
On some UI elements, there is some PURPLE filled area, they were removed UI elements, reconstruct the background under where they were.
|
||||
|
||||
MUST extract exactly these marked UI elements area.
|
||||
"#;
|
||||
pub(super) fn gen_extract_prompt(separation_notes: Vec<SeparationNote>) -> String {
|
||||
let extract_system_prompt = format!(
|
||||
r#"
|
||||
This is a UI design image, not a normal photo/illustration. Extract it strictly as UI elements/layers, not as a generic foreground/background extraction.
|
||||
Treat distinct UI element as its own layer with hard, clean, pixel-accurate edges and full transparency outside the element.
|
||||
|
||||
MUST keep each element at its original position on a transparent canvas.
|
||||
{SHARED_SEPARATION_REQ}
|
||||
here are UI elements to extract:
|
||||
|
||||
"#
|
||||
);
|
||||
let mut result = extract_system_prompt;
|
||||
result.reserve(512);
|
||||
for elem in separation_notes {
|
||||
result.push_str(&elem.as_prompt());
|
||||
result.push('\n');
|
||||
}
|
||||
result
|
||||
}
|
||||
pub(super) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String {
|
||||
let binding_system_prompt = format!(
|
||||
r#"
|
||||
You will be given a src UI design image and a processed image, where some ui elements are separated.
|
||||
You need to recognize and review the separation using the given tool.
|
||||
field notes:
|
||||
* extracted_area MUST be the recognized area from the processed image, INSTEAD OF from the src image.
|
||||
The processed image is the only authoritative image for extracted_area.
|
||||
Return the pixel bounding box of the extracted element as it appears in the processed image.
|
||||
Do not copy, infer, or reuse the source node rectangle.
|
||||
The src image is only for identifying which semantic UI element belongs to to_node.
|
||||
|
||||
Here were the separation requirements:
|
||||
```
|
||||
{SHARED_SEPARATION_REQ}
|
||||
```
|
||||
And you should also review if the extracted's successfully meet the src image:
|
||||
* shape
|
||||
* color
|
||||
* style
|
||||
* edge process
|
||||
...
|
||||
|
||||
if not, use `NeedRework` data structure in the tool to indicate the (it should be from which) node id and advice.
|
||||
your advice (less than 20 words) will be used to improve the separation in the next time.
|
||||
|
||||
these node need handle:
|
||||
"#
|
||||
);
|
||||
let mut result = binding_system_prompt;
|
||||
result.reserve(512);
|
||||
for elem in nodes {
|
||||
result.push_str(&elem.as_prompt());
|
||||
result.push('\n');
|
||||
}
|
||||
result
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
use crate::ui_editor::commands::separation::SeparationNode;
|
||||
|
||||
pub(crate) fn gen_binding_prompt(nodes: Vec<&SeparationNode>) -> String {
|
||||
let binding_system_prompt = r#"
|
||||
You will be given a src UI design image and a processed image, where some ui elements are separated.
|
||||
You need to recognize and review the separation using the given tool.
|
||||
field notes:
|
||||
* extracted_area MUST be the recognized area from the processed image, INSTEAD OF from the src image.
|
||||
The processed image is the only authoritative image for extracted_area.
|
||||
Return the pixel bounding box of the extracted element as it appears in the processed image.
|
||||
Do not copy, infer, or reuse the source node rectangle.
|
||||
The src image is only for identifying which semantic UI element belongs to to_node.
|
||||
|
||||
Here were the separation requirements:
|
||||
Preserve hard edges and the exact visible shape.
|
||||
The processed image is a transparent atlas containing the requested image layers.
|
||||
Do not use the source node rectangle as the extracted area.
|
||||
And you should also review if the extracted's successfully meet the src image:
|
||||
* shape
|
||||
* color
|
||||
* style
|
||||
* edge process
|
||||
...
|
||||
|
||||
if not, use `NeedRework` data structure in the tool to indicate the (it should be from which) node id and advice.
|
||||
your advice (less than 20 words) will be used to improve the separation in the next time.
|
||||
|
||||
these node need handle:
|
||||
"#
|
||||
.to_string();
|
||||
let mut result = binding_system_prompt;
|
||||
result.reserve(512);
|
||||
for elem in nodes {
|
||||
result.push_str(&elem.as_prompt());
|
||||
result.push('\n');
|
||||
}
|
||||
result
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
use crate::ui_editor::commands::separation::{SeparationNode, SeparationNodeKind};
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::ui_editor::commands::separation::model::{SeparationState, SeparationTree};
|
||||
use crate::ui_editor::utils::NodeId;
|
||||
|
||||
pub(crate) fn gen_extract_prompt(
|
||||
state: &SeparationState,
|
||||
tree: &SeparationTree,
|
||||
batch: &[&SeparationNode],
|
||||
) -> String {
|
||||
let mut result = r#"
|
||||
This is a UI design image, not a normal photo/illustration. Extract it strictly as UI elements/layers, not as a generic foreground/background extraction.
|
||||
Treat distinct UI element as its own layer with hard, clean, pixel-accurate edges and full transparency outside the element.
|
||||
Generate one transparent atlas at the requested canvas size. You may move or scale output layers so they do not cover one another.
|
||||
Preserve hard edges and the exact visible shape. Never split a scene/background into multiple scene layers.
|
||||
Ordinary text is editable UI text: remove it from its parent image/background and do not generate a text raster layer.
|
||||
Extract only the image nodes marked OUTPUT_THIS_TURN. Reconstruct every child/text layer that is listed under a parent but is not an output target.
|
||||
|
||||
UI layer tree:
|
||||
"#
|
||||
.to_string();
|
||||
result.reserve(2048);
|
||||
let target_ids = batch
|
||||
.iter()
|
||||
.map(|node| node.id.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
let terminal_ids = state
|
||||
.bound
|
||||
.iter()
|
||||
.map(|node| node.node_id.clone())
|
||||
.chain(
|
||||
state
|
||||
.problematic_nodes
|
||||
.iter()
|
||||
.map(|node| node.node_id.clone()),
|
||||
)
|
||||
.collect::<HashSet<_>>();
|
||||
let mut lines = Vec::new();
|
||||
let mut index = 1usize;
|
||||
append_context_lines(
|
||||
&tree.root,
|
||||
0,
|
||||
true,
|
||||
&target_ids,
|
||||
&terminal_ids,
|
||||
&mut index,
|
||||
&mut lines,
|
||||
);
|
||||
result.push_str(&lines.join("\n"));
|
||||
result
|
||||
}
|
||||
|
||||
fn append_context_lines(
|
||||
node: &SeparationNode,
|
||||
depth: usize,
|
||||
is_root: bool,
|
||||
target_ids: &HashSet<NodeId>,
|
||||
terminal_ids: &HashSet<NodeId>,
|
||||
index: &mut usize,
|
||||
lines: &mut Vec<String>,
|
||||
) {
|
||||
let status = match node.kind {
|
||||
SeparationNodeKind::TextRemovalOnly => "REMOVE_ONLY",
|
||||
SeparationNodeKind::PureContainer => "CONTEXT_ONLY",
|
||||
SeparationNodeKind::ImageTarget => {
|
||||
if target_ids.contains(&node.id) {
|
||||
"OUTPUT_THIS_TURN"
|
||||
} else if terminal_ids.contains(&node.id) {
|
||||
"DONE"
|
||||
} else {
|
||||
"CONTEXT_ONLY"
|
||||
}
|
||||
}
|
||||
};
|
||||
let role = if is_root { "root" } else { "child" };
|
||||
lines.push(format!(
|
||||
"{}. [{}] depth={} role={} rect=({}, {}, {}, {}) {}",
|
||||
*index,
|
||||
status,
|
||||
depth,
|
||||
role,
|
||||
node.global_pos_x_px,
|
||||
node.global_pos_y_px,
|
||||
node.width_px,
|
||||
node.height_px,
|
||||
node.note.as_prompt()
|
||||
));
|
||||
*index += 1;
|
||||
for child in &node.children {
|
||||
append_context_lines(
|
||||
child,
|
||||
depth + 1,
|
||||
false,
|
||||
target_ids,
|
||||
terminal_ids,
|
||||
index,
|
||||
lines,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
mod binding;
|
||||
mod extract;
|
||||
|
||||
pub(super) use binding::gen_binding_prompt;
|
||||
pub(super) use extract::gen_extract_prompt;
|
||||
File diff suppressed because it is too large
Load Diff
+130
@@ -0,0 +1,130 @@
|
||||
use crate::config::{build_game_creator_llm_client_from_llm_config, load_game_creator_app_config};
|
||||
use crate::ui_editor::commands::separation::prompt::gen_binding_prompt;
|
||||
use crate::ui_editor::commands::separation::{
|
||||
validate_binding_response, BindingResp, SeparationNode,
|
||||
};
|
||||
use crate::ui_editor::commands::utils::{
|
||||
parse_limited_llm_tool_arguments, request_ui_editor_llm, run_with_repair_history,
|
||||
strict_json_schema,
|
||||
};
|
||||
use platform_llm::{
|
||||
LlmFunctionTool, LlmMessage, LlmMessageContentPart, LlmRunRequest, LlmToolChoice,
|
||||
};
|
||||
use std::time::Instant;
|
||||
|
||||
pub(super) async fn visual_binding(
|
||||
source_url: String,
|
||||
processed_url: String,
|
||||
nodes: &[&SeparationNode],
|
||||
) -> Result<BindingResp, String> {
|
||||
let started = Instant::now();
|
||||
let result = visual_binding_inner(source_url, processed_url, nodes).await;
|
||||
app_log!(
|
||||
"ui_separation.visual_binding.timing outcome={} elapsed_ms={} nodes={}",
|
||||
if result.is_ok() { "ok" } else { "error" },
|
||||
started.elapsed().as_millis(),
|
||||
nodes.len()
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
async fn visual_binding_inner(
|
||||
source_url: String,
|
||||
processed_url: String,
|
||||
nodes: &[&SeparationNode],
|
||||
) -> Result<BindingResp, String> {
|
||||
app_log!(
|
||||
"ui_separation.visual_binding.start nodes={} source_url_chars={} processed_url_chars={}",
|
||||
nodes.len(),
|
||||
source_url.chars().count(),
|
||||
processed_url.chars().count()
|
||||
);
|
||||
let llm_config = load_game_creator_app_config()
|
||||
.map_err(|e| {
|
||||
app_log!("ui_separation.error stage=visual_binding reason=load_config error={e}");
|
||||
e.to_string()
|
||||
})?
|
||||
.llm;
|
||||
let client =
|
||||
build_game_creator_llm_client_from_llm_config(&llm_config, "llm").map_err(|e| {
|
||||
app_log!("ui_separation.error stage=visual_binding reason=build_client error={e}");
|
||||
e.to_string()
|
||||
})?;
|
||||
let schema = strict_json_schema::<BindingResp>().map_err(|error| {
|
||||
app_log!("ui_separation.error stage=visual_binding reason=build_schema error={error}");
|
||||
error
|
||||
})?;
|
||||
let tool = LlmFunctionTool::new(
|
||||
"bind_ui_elements",
|
||||
"确认处理图中的区域对应哪些 UI 节点",
|
||||
schema,
|
||||
)
|
||||
.with_strict(true);
|
||||
let initial_history = vec![
|
||||
LlmMessage::system(gen_binding_prompt(nodes.to_vec())),
|
||||
LlmMessage::user_multimodal(vec![
|
||||
LlmMessageContentPart::InputText {
|
||||
text: "processed image:".to_string(),
|
||||
},
|
||||
LlmMessageContentPart::InputImage {
|
||||
image_url: processed_url.clone(),
|
||||
},
|
||||
LlmMessageContentPart::InputText {
|
||||
text: "src image:".to_string(),
|
||||
},
|
||||
LlmMessageContentPart::InputImage {
|
||||
image_url: source_url.clone(),
|
||||
},
|
||||
]),
|
||||
];
|
||||
let result = run_with_repair_history(
|
||||
2,
|
||||
initial_history,
|
||||
|history| {
|
||||
let tool = tool.clone();
|
||||
let client = client.clone();
|
||||
let llm_config = llm_config.clone();
|
||||
async move {
|
||||
let request = LlmRunRequest::new(history)
|
||||
.with_function_tools(vec![tool.clone()])
|
||||
.with_tool_choice(LlmToolChoice::Required);
|
||||
let request_started = Instant::now();
|
||||
let response = request_ui_editor_llm(&client, &llm_config, request).await;
|
||||
app_log!(
|
||||
"ui_separation.llm.timing outcome={} elapsed_ms={}",
|
||||
if response.is_ok() { "ok" } else { "error" },
|
||||
request_started.elapsed().as_millis()
|
||||
);
|
||||
response
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|response| {
|
||||
response
|
||||
.tool_calls
|
||||
.into_iter()
|
||||
.find(|call| call.name == "bind_ui_elements")
|
||||
.map(|call| call.arguments)
|
||||
.ok_or_else(|| "视觉绑定模型未返回工具调用".to_string())
|
||||
})
|
||||
.and_then(|arguments| parse_limited_llm_tool_arguments(&arguments))
|
||||
.and_then(|args| {
|
||||
serde_json::from_value::<BindingResp>(args)
|
||||
.map_err(|e| format!("视觉绑定结果无效:{e}"))
|
||||
})
|
||||
}
|
||||
},
|
||||
|value: &BindingResp| validate_binding_response(value, nodes),
|
||||
)
|
||||
.await;
|
||||
match &result {
|
||||
Ok(value) => app_log!(
|
||||
"ui_separation.visual_binding.completed nodes={} decisions={}",
|
||||
nodes.len(),
|
||||
value.decisions.len()
|
||||
),
|
||||
Err(error) => app_log!(
|
||||
"ui_separation.error stage=visual_binding reason=failed nodes={} error={error}",
|
||||
nodes.len()
|
||||
),
|
||||
}
|
||||
result
|
||||
}
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
use crate::ui_editor::commands::separation::area::normalize_binding_area;
|
||||
use crate::ui_editor::commands::separation::model::BindingArea;
|
||||
use image::ImageFormat;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Instant;
|
||||
|
||||
pub(super) async fn cut_processed_image(
|
||||
source: PathBuf,
|
||||
area: BindingArea,
|
||||
target: PathBuf,
|
||||
) -> Result<(), String> {
|
||||
let started = Instant::now();
|
||||
let result = cut_processed_image_inner(source, area, target).await;
|
||||
app_log!(
|
||||
"ui_separation.cut_image.timing outcome={} elapsed_ms={}",
|
||||
if result.is_ok() { "ok" } else { "error" },
|
||||
started.elapsed().as_millis()
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
async fn cut_processed_image_inner(
|
||||
source: PathBuf,
|
||||
area: BindingArea,
|
||||
target: PathBuf,
|
||||
) -> Result<(), String> {
|
||||
app_log!(
|
||||
"ui_separation.cut_image.start source_file={} target_file={} area=({}, {}, {}, {})",
|
||||
source
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("<unknown>"),
|
||||
target
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("<unknown>"),
|
||||
area.global_pos_x_px,
|
||||
area.global_pos_y_px,
|
||||
area.width_px,
|
||||
area.height_px
|
||||
);
|
||||
tokio::task::spawn_blocking(move || cut_processed_image_blocking(&source, &area, &target))
|
||||
.await
|
||||
.map_err(|error| format!("裁切处理图任务失败:{error}"))?
|
||||
}
|
||||
|
||||
fn cut_processed_image_blocking(
|
||||
source: &Path,
|
||||
area: &BindingArea,
|
||||
target: &Path,
|
||||
) -> Result<(), String> {
|
||||
let image = image::open(source)
|
||||
.map_err(|e| format!("读取处理图失败:{e}"))?
|
||||
.to_rgba8();
|
||||
let normalized = normalize_binding_area(&image, *area)?;
|
||||
let original_area = *area;
|
||||
let normalized_area = normalized.area;
|
||||
app_log!(
|
||||
"ui_separation.cut_image.normalized changed={} clamped={} transparent={} original_area=({}, {}, {}, {}) normalized_area=({}, {}, {}, {})",
|
||||
normalized.changed,
|
||||
normalized.clamped,
|
||||
normalized.transparent,
|
||||
original_area.global_pos_x_px,
|
||||
original_area.global_pos_y_px,
|
||||
original_area.width_px,
|
||||
original_area.height_px,
|
||||
normalized_area.global_pos_x_px,
|
||||
normalized_area.global_pos_y_px,
|
||||
normalized_area.width_px,
|
||||
normalized_area.height_px
|
||||
);
|
||||
let cropped = image::imageops::crop_imm(
|
||||
&image,
|
||||
normalized_area.global_pos_x_px,
|
||||
normalized_area.global_pos_y_px,
|
||||
normalized_area.width_px,
|
||||
normalized_area.height_px,
|
||||
)
|
||||
.to_image();
|
||||
cropped
|
||||
.save_with_format(target, ImageFormat::Png)
|
||||
.map_err(|e| format!("写入 cut 图片失败:{e}"))?;
|
||||
app_log!(
|
||||
"ui_separation.cut_image.completed target_file={} width={} height={}",
|
||||
target
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("<unknown>"),
|
||||
normalized_area.width_px,
|
||||
normalized_area.height_px
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
use crate::platform_session::PlatformSessionSnapshot;
|
||||
use base64::Engine as _;
|
||||
use serde::Deserialize;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RawEditResponse {
|
||||
data: Vec<RawEditItem>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RawEditItem {
|
||||
b64_json: String,
|
||||
}
|
||||
|
||||
pub(super) async fn raw_image_edit(
|
||||
session: &PlatformSessionSnapshot,
|
||||
image_data_url: &str,
|
||||
prompt: &str,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<String, String> {
|
||||
let started = Instant::now();
|
||||
let result = raw_image_edit_inner(session, image_data_url, prompt, width, height).await;
|
||||
app_log!(
|
||||
"ui_separation.image_edit.timing outcome={} elapsed_ms={} width={} height={}",
|
||||
if result.is_ok() { "ok" } else { "error" },
|
||||
started.elapsed().as_millis(),
|
||||
width,
|
||||
height
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
async fn raw_image_edit_inner(
|
||||
session: &PlatformSessionSnapshot,
|
||||
image_data_url: &str,
|
||||
prompt: &str,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<String, String> {
|
||||
app_log!(
|
||||
"ui_separation.image_edit.start width={} height={} prompt_chars={}",
|
||||
width,
|
||||
height,
|
||||
prompt.chars().count()
|
||||
);
|
||||
let (mime, data) = image_data_url
|
||||
.split_once(',')
|
||||
.ok_or_else(|| "界面图 data URL 无效".to_string())?;
|
||||
let mime = mime
|
||||
.strip_prefix("data:")
|
||||
.and_then(|value| value.strip_suffix(";base64"))
|
||||
.unwrap_or("image/png");
|
||||
if !mime.eq_ignore_ascii_case("image/png") {
|
||||
return Err("图片分离请求只支持 PNG 源图".to_string());
|
||||
}
|
||||
let image_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(data.trim())
|
||||
.map_err(|error| format!("解码源图失败:{error}"))?;
|
||||
if image_bytes.is_empty() {
|
||||
return Err("源图不能为空".to_string());
|
||||
}
|
||||
let client = crate::http_client::agc_main_site_client_builder()
|
||||
.build()
|
||||
.map_err(|error| format!("创建图片编辑客户端失败:{error}"))?;
|
||||
let url = format!(
|
||||
"{}/api/raw/v1/images/edit",
|
||||
session.api_base_url.trim_end_matches('/')
|
||||
);
|
||||
let image_part = reqwest::multipart::Part::bytes(image_bytes)
|
||||
.file_name("image.png")
|
||||
.mime_str("image/png")
|
||||
.map_err(|error| format!("构造图片编辑文件部件失败:{error}"))?;
|
||||
let body = reqwest::multipart::Form::new()
|
||||
.part("image", image_part)
|
||||
.text("prompt", prompt.to_string())
|
||||
.text("width", width.to_string())
|
||||
.text("height", height.to_string())
|
||||
.text("output_format", "png")
|
||||
.text("background", "transparent");
|
||||
let response = crate::http_client::with_agc_main_site_marker(
|
||||
client
|
||||
.post(url)
|
||||
.bearer_auth(&session.access_token)
|
||||
.multipart(body),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| {
|
||||
app_log!("ui_separation.error stage=image_edit reason=send error={error}");
|
||||
format!("图片分离请求失败:{error}")
|
||||
})?;
|
||||
if !response.status().is_success() {
|
||||
app_log!(
|
||||
"ui_separation.error stage=image_edit reason=http_status status={}",
|
||||
response.status()
|
||||
);
|
||||
return Err(format!("图片分离请求失败(HTTP {})", response.status()));
|
||||
}
|
||||
let payload = response.json::<RawEditResponse>().await.map_err(|error| {
|
||||
app_log!("ui_separation.error stage=image_edit reason=parse_response error={error}");
|
||||
format!("解析图片分离响应失败:{error}")
|
||||
})?;
|
||||
let result = payload
|
||||
.data
|
||||
.into_iter()
|
||||
.next()
|
||||
.map(|item| format!("data:image/png;base64,{}", item.b64_json))
|
||||
.ok_or_else(|| "图片分离响应没有图像".to_string());
|
||||
match &result {
|
||||
Ok(value) => app_log!(
|
||||
"ui_separation.image_edit.completed data_url_chars={}",
|
||||
value.chars().count()
|
||||
),
|
||||
Err(error) => {
|
||||
app_log!("ui_separation.error stage=image_edit reason=empty_result error={error}")
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(super) async fn write_processed_image(
|
||||
processed_url: String,
|
||||
target: PathBuf,
|
||||
) -> Result<(), String> {
|
||||
let started = Instant::now();
|
||||
let result = write_processed_image_inner(processed_url, target).await;
|
||||
app_log!(
|
||||
"ui_separation.processed_image.write.timing outcome={} elapsed_ms={}",
|
||||
if result.is_ok() { "ok" } else { "error" },
|
||||
started.elapsed().as_millis()
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
async fn write_processed_image_inner(processed_url: String, target: PathBuf) -> Result<(), String> {
|
||||
app_log!(
|
||||
"ui_separation.processed_image.write.start target_file={} data_url_chars={}",
|
||||
target
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("<unknown>"),
|
||||
processed_url.chars().count()
|
||||
);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let encoded = processed_url
|
||||
.split_once(',')
|
||||
.map(|(_, data)| data)
|
||||
.ok_or_else(|| "处理图 data URL 无效".to_string())?;
|
||||
let processed_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(encoded)
|
||||
.map_err(|error| format!("解析处理图失败:{error}"))?;
|
||||
let byte_len = processed_bytes.len();
|
||||
fs::write(&target, processed_bytes)
|
||||
.map_err(|error| format!("写入处理图失败:{}: {error}", target.display()))
|
||||
.map(|_| {
|
||||
app_log!(
|
||||
"ui_separation.processed_image.write.completed target_file={} bytes={}",
|
||||
target
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("<unknown>"),
|
||||
byte_len
|
||||
);
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|error| format!("写入处理图任务失败:{error}"))?
|
||||
}
|
||||
+222
@@ -0,0 +1,222 @@
|
||||
mod binding;
|
||||
mod cut;
|
||||
mod image_edit;
|
||||
mod patch;
|
||||
|
||||
pub use patch::apply_batch_patch;
|
||||
|
||||
use super::model::*;
|
||||
use super::persistence::{
|
||||
project_relative_path, read_separation_state, separation_dto, separation_sidecar_dir,
|
||||
separation_state_path, write_separation_state,
|
||||
};
|
||||
use super::prompt::gen_extract_prompt;
|
||||
use super::tree::next_image_batch;
|
||||
use crate::platform_session::current_platform_session;
|
||||
use crate::ui_editor::commands::utils::read_ui_reference_image_data_url;
|
||||
use crate::ui_editor::state::State;
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
pub(crate) async fn separate_ui_impl(
|
||||
project_path: String,
|
||||
asset_id: String,
|
||||
state: State,
|
||||
) -> Result<SeparationDTO, String> {
|
||||
app_log!(
|
||||
"ui_separation.start asset_id={} ui_trees={} ui_images={} sprites={}",
|
||||
asset_id,
|
||||
state.ui_trees.len(),
|
||||
state.ui_design_images.len(),
|
||||
state.sprite_assets.len()
|
||||
);
|
||||
let session = current_platform_session().ok_or_else(|| "请先登录平台账号".to_string())?;
|
||||
let root = Path::new(project_path.trim());
|
||||
let sidecar = separation_sidecar_dir(root, &asset_id).map_err(|error| {
|
||||
app_log!(
|
||||
"ui_separation.error stage=sidecar_dir asset_id={} error={error}",
|
||||
asset_id
|
||||
);
|
||||
error
|
||||
})?;
|
||||
fs::create_dir_all(&sidecar).map_err(|error| {
|
||||
app_log!(
|
||||
"ui_separation.error stage=sidecar_create asset_id={} error={error}",
|
||||
asset_id
|
||||
);
|
||||
format!("创建 separation sidecar 失败:{error}")
|
||||
})?;
|
||||
let state_path = separation_state_path(root, &asset_id)?;
|
||||
let restored = state_path.exists();
|
||||
let mut separation = if restored {
|
||||
app_log!("ui_separation.state_restore.start asset_id={asset_id}");
|
||||
read_separation_state(&state_path).map_err(|error| {
|
||||
app_log!(
|
||||
"ui_separation.error stage=state_restore asset_id={} error={error}",
|
||||
asset_id
|
||||
);
|
||||
error
|
||||
})?
|
||||
} else {
|
||||
app_log!("ui_separation.state_construct.start asset_id={asset_id}");
|
||||
super::tree::construct_separation_state(&state)
|
||||
};
|
||||
app_log!(
|
||||
"ui_separation.state_ready asset_id={} restored={} trees={} bound={} problematic={}",
|
||||
asset_id,
|
||||
restored,
|
||||
separation.trees.len(),
|
||||
separation.bound.len(),
|
||||
separation.problematic_nodes.len()
|
||||
);
|
||||
|
||||
for tree_index in 0..separation.trees.len() {
|
||||
let tree = &separation.trees[tree_index];
|
||||
let image_id = tree.src_ui_design.clone();
|
||||
let image = state
|
||||
.ui_design_images
|
||||
.get(&image_id)
|
||||
.ok_or_else(|| "缺少源界面图".to_string())?;
|
||||
let source_path = crate::project::resolve_local_project_path(root, &image.path)?;
|
||||
let source_url = read_ui_reference_image_data_url(source_path)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
app_log!(
|
||||
"ui_separation.error stage=read_source tree_index={} image_id={} error={error}",
|
||||
tree_index,
|
||||
image_id.as_str()
|
||||
);
|
||||
error
|
||||
})?;
|
||||
app_log!(
|
||||
"ui_separation.tree_start tree_index={} image_id={} width={} height={}",
|
||||
tree_index,
|
||||
image_id.as_str(),
|
||||
image.pixel_size.x.round() as u32,
|
||||
image.pixel_size.y.round() as u32
|
||||
);
|
||||
write_separation_state(&state_path, &separation).map_err(|error| {
|
||||
app_log!(
|
||||
"ui_separation.error stage=state_checkpoint tree_index={} error={error}",
|
||||
tree_index
|
||||
);
|
||||
error
|
||||
})?;
|
||||
let mut batch_index = 0usize;
|
||||
loop {
|
||||
let batch_started = Instant::now();
|
||||
let Some(current_tree) = separation.trees.get(tree_index) else {
|
||||
break;
|
||||
};
|
||||
let batch_nodes = next_image_batch(&separation, current_tree)
|
||||
.into_iter()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
if batch_nodes.is_empty() {
|
||||
app_log!(
|
||||
"ui_separation.tree_completed tree_index={} image_id={} bound={} problematic={}",
|
||||
tree_index, image_id.as_str(), separation.bound.len(), separation.problematic_nodes.len()
|
||||
);
|
||||
break;
|
||||
}
|
||||
let batch = batch_nodes.iter().collect::<Vec<_>>();
|
||||
let prompt = gen_extract_prompt(&separation, current_tree, &batch);
|
||||
app_log!(
|
||||
"ui_separation.batch_start tree_index={} batch_index={} nodes={} prompt_chars={} rework_total={}",
|
||||
tree_index, batch_index, batch.len(), prompt.chars().count(),
|
||||
batch.iter().map(|node| node.rework_count).sum::<u32>()
|
||||
);
|
||||
let processed_url = match image_edit::raw_image_edit(
|
||||
&session,
|
||||
&source_url,
|
||||
&prompt,
|
||||
image.pixel_size.x as u32,
|
||||
image.pixel_size.y as u32,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
app_log!("ui_separation.error stage=image_edit tree_index={} batch_index={} error={error}", tree_index, batch_index);
|
||||
write_separation_state(&state_path, &separation)?;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let processed_path = sidecar.join(format!("processed-{}.png", separation.bound.len()));
|
||||
if let Err(error) =
|
||||
image_edit::write_processed_image(processed_url.clone(), processed_path.clone())
|
||||
.await
|
||||
{
|
||||
app_log!("ui_separation.error stage=processed_image_write tree_index={} batch_index={} error={error}", tree_index, batch_index);
|
||||
write_separation_state(&state_path, &separation)?;
|
||||
return Err(error);
|
||||
}
|
||||
let binding = match binding::visual_binding(source_url.clone(), processed_url, &batch)
|
||||
.await
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
app_log!("ui_separation.error stage=visual_binding tree_index={} batch_index={} error={error}", tree_index, batch_index);
|
||||
write_separation_state(&state_path, &separation)?;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
app_log!(
|
||||
"ui_separation.binding_decisions tree_index={} batch_index={} decisions={}",
|
||||
tree_index,
|
||||
batch_index,
|
||||
binding.decisions.len()
|
||||
);
|
||||
let mut cut_paths = HashMap::new();
|
||||
let mut cut_error = None;
|
||||
for decision in &binding.decisions {
|
||||
if let BindingDecision::Ok {
|
||||
to_node,
|
||||
extracted_area,
|
||||
} = decision
|
||||
{
|
||||
let cut_path = sidecar.join(format!("cut-{}.png", to_node.as_str()));
|
||||
match cut::cut_processed_image(
|
||||
processed_path.clone(),
|
||||
*extracted_area,
|
||||
cut_path.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
cut_paths
|
||||
.insert(to_node.clone(), project_relative_path(root, &cut_path)?);
|
||||
}
|
||||
Err(error) => {
|
||||
app_log!("ui_separation.error stage=cut_image tree_index={} batch_index={} node_id={} error={error}", tree_index, batch_index, to_node.as_str());
|
||||
cut_error =
|
||||
Some(format!("节点 {} 的分离区域无效:{error}", to_node.as_str()));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(error) = cut_error {
|
||||
app_log!("ui_separation.error stage=cut_batch tree_index={} batch_index={} error={error}", tree_index, batch_index);
|
||||
write_separation_state(&state_path, &separation)?;
|
||||
return Err(error);
|
||||
}
|
||||
patch::apply_batch_patch(&mut separation, tree_index, &binding.decisions, &cut_paths)?;
|
||||
write_separation_state(&state_path, &separation)?;
|
||||
app_log!(
|
||||
"ui_separation.batch_completed tree_index={} batch_index={} cuts={} bound={} problematic={} elapsed_ms={}",
|
||||
tree_index, batch_index, cut_paths.len(), separation.bound.len(), separation.problematic_nodes.len(), batch_started.elapsed().as_millis()
|
||||
);
|
||||
batch_index += 1;
|
||||
}
|
||||
}
|
||||
app_log!(
|
||||
"ui_separation.completed asset_id={} bound_nodes={} problematic_nodes={}",
|
||||
asset_id,
|
||||
separation.bound.len(),
|
||||
separation.problematic_nodes.len()
|
||||
);
|
||||
Ok(separation_dto(&separation))
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
use crate::ui_editor::commands::separation::{
|
||||
next_image_batch, validate_binding_response, BindingDecision, BindingResp, BoundNode,
|
||||
ProblematicNode, SeparationNode, SeparationState, MAX_REWORK_COUNT,
|
||||
};
|
||||
use crate::ui_editor::utils::NodeId;
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub fn apply_batch_patch(
|
||||
state: &mut SeparationState,
|
||||
tree_index: usize,
|
||||
decisions: &[BindingDecision],
|
||||
cut_paths: &HashMap<NodeId, String>,
|
||||
) -> Result<(), String> {
|
||||
app_log!(
|
||||
"ui_separation.batch_patch.start tree_index={} decisions={} cut_paths={}",
|
||||
tree_index,
|
||||
decisions.len(),
|
||||
cut_paths.len()
|
||||
);
|
||||
let batch_nodes = {
|
||||
let tree = state
|
||||
.trees
|
||||
.get(tree_index)
|
||||
.ok_or_else(|| "separation tree 索引无效".to_string())?;
|
||||
next_image_batch(state, tree)
|
||||
.into_iter()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let batch = batch_nodes.iter().collect::<Vec<_>>();
|
||||
validate_binding_response(
|
||||
&BindingResp {
|
||||
decisions: decisions.to_vec(),
|
||||
},
|
||||
&batch,
|
||||
)?;
|
||||
let rework_counts = batch_nodes
|
||||
.iter()
|
||||
.map(|node| (node.id.clone(), node.rework_count))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let tree = state.trees.get_mut(tree_index).expect("tree index checked");
|
||||
for decision in decisions {
|
||||
match decision {
|
||||
BindingDecision::Ok { to_node, .. } => {
|
||||
let path = cut_paths
|
||||
.get(to_node)
|
||||
.ok_or_else(|| format!("缺少节点 {} 的 cut 图片", to_node.as_str()))?;
|
||||
state.bound.push(BoundNode {
|
||||
node_id: to_node.clone(),
|
||||
cut_image_path: path.clone(),
|
||||
});
|
||||
}
|
||||
BindingDecision::NeedRework {
|
||||
to_node,
|
||||
advice: problem_description,
|
||||
} => {
|
||||
append_rework_note(&mut tree.root, to_node, problem_description);
|
||||
let count = rework_counts.get(to_node).copied().unwrap_or(0) + 1;
|
||||
increment_rework_count(&mut tree.root, to_node, count);
|
||||
if count >= MAX_REWORK_COUNT {
|
||||
state.problematic_nodes.push(ProblematicNode {
|
||||
node_id: to_node.clone(),
|
||||
problem_description: problem_description.clone(),
|
||||
rework_count: count,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
app_log!(
|
||||
"ui_separation.batch_patch.completed tree_index={} bound={} problematic={} pending_root_children={}",
|
||||
tree_index,
|
||||
state.bound.len(),
|
||||
state.problematic_nodes.len(),
|
||||
tree.root.children.len()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn append_rework_note(node: &mut SeparationNode, id: &NodeId, note: &str) -> bool {
|
||||
if node.id == *id {
|
||||
node.note.rework_notes.push(note.to_string());
|
||||
return true;
|
||||
}
|
||||
node.children
|
||||
.iter_mut()
|
||||
.any(|child| append_rework_note(child, id, note))
|
||||
}
|
||||
|
||||
fn increment_rework_count(node: &mut SeparationNode, id: &NodeId, count: u32) {
|
||||
if node.id == *id {
|
||||
node.rework_count = count;
|
||||
return;
|
||||
}
|
||||
for child in &mut node.children {
|
||||
increment_rework_count(child, id, count);
|
||||
}
|
||||
}
|
||||
@@ -102,6 +102,12 @@ Rust 在接收并校验工具结果后,才把这两个枚举变体映射为正
|
||||
- 图片编辑调用当前 Raw GPT Image 2 multipart 合同:`image`(PNG 文件)、`prompt`、`width`、`height`、`output_format=png`、`background=transparent`;不再发送旧 JSON/base64 请求体。
|
||||
- 后端 raw operation 的持久状态与扣费后崩溃恢复窗口,遵循 Raw GPT Image 2 方案中的独立 TODO。
|
||||
|
||||
## 实现组织
|
||||
|
||||
- separation prompt 按职责拆分为 `prompt/extract.rs` 与 `prompt/binding.rs`,由 `prompt/mod.rs` 统一导出。
|
||||
- separation workflow 按执行边界拆分为 `workflow/image_edit.rs`(Raw image-edit 与处理图写入)、`workflow/binding.rs`(视觉绑定)、`workflow/cut.rs`(像素归一化与裁切)、`workflow/patch.rs`(批次状态 patch);`workflow/mod.rs` 仅负责批次编排与 sidecar 检查点。
|
||||
- 上述拆分只调整 Rust 模块边界,不改变批次选择、重试、sidecar 持久化、绑定校验或错误恢复语义。
|
||||
|
||||
## TODO
|
||||
|
||||
- 正在执行 batch 的持久化和恢复。
|
||||
|
||||
Reference in New Issue
Block a user