WIP: UI编辑器自动分图层切图标 #304

Draft
k88936 wants to merge 100 commits from feat/ui-editor-auto-seperation into master
5 changed files with 270 additions and 132 deletions
Showing only changes of commit 0400103a19 - Show all commits
@@ -14,28 +14,20 @@ POST /api/raw/v1/images/edit
## 请求合同
请求使用 `application/json`。图片字段只使用原始图片的 base64 数据和 MIME 类型,不接受 object key、URL、Data URL 或 Blob URL
请求使用 `multipart/form-data`,不再接受 JSON/base64 入站格式。图片直接作为文件字段上传,避免 base64 膨胀和入站解码;服务端仍在扣费前完成 PNG 完整解码与资源限制校验
```json
{
"image": {
"data": "<base64>",
"mimeType": "image/png"
},
"mask": {
"data": "<base64>",
"mimeType": "image/png"
},
"prompt": "修改图片",
"quality": "auto",
"background": "auto",
"output_format": "png",
"width": 1536,
"height": 1024
}
```text
image: <PNG 文件,必填>
mask: <PNG 文件,可选>
prompt: 修改图片
quality: auto
background: auto
output_format: png
width: 1536
height: 1024
```
`image` 是必填的单图结构 `{ data, mimeType }``mask` 可选并使用相同结构。`image``mask``mimeType` 必须 `image/png`,base64 解码后必须是可完整解码的有效 PNG 文件,提供 mask 时其宽高必须与 image 完全一致;空数据、非 PNG 字节、MIME 不匹配或尺寸不一致均在扣费前返回 400。服务端不把输入格式另建成请求参数。`prompt` 必填,UTF-8 原始字节长度不得超过 `4 KiB`;超限在扣费前返回 400。`quality``background``output_format` 采用 GPT Image 模型支持的值。
`image``mask` 必须 `image/png` 文件字段;服务端不信任客户端文件名,转发时使用固定文件名。空文件、非 PNG 字节、MIME 不匹配或 mask 与 image 尺寸不一致均在扣费前返回 400。`prompt` 必填,UTF-8 原始字节长度不得超过 `4 KiB`;超限在扣费前返回 400。`quality``background``output_format` 采用 GPT Image 模型支持的值。字段不能重复,未知字段拒绝;缺失的必填字段拒绝。
`width``height` 使用严格输出尺寸规则,均在扣费前校验:
@@ -46,7 +38,7 @@ POST /api/raw/v1/images/edit
校验通过后按整数尺寸发送给 provider,不静默 clamp 或改写调用者尺寸。
Raw 路由的 JSON body limit 为 `64 MiB`为 base64 编码膨胀和可选 mask 留出空间;同时必须在 base64 解码后拒绝空 PNG,并保留图片格式校验,避免仅依赖 HTTP body limit。PNG 解码使用与输出合同一致的资源上限:宽高各不超过 `3840`,解码分配不超过 `8294400 × 4` 字节;base64 与 PNG 解码在阻塞线程执行,不占用 Tokio 异步 worker
Raw 路由的 multipart body limit 为 `64 MiB`覆盖图片和文本字段;文件字段由 multipart 解析器直接收集为字节,随后在阻塞线程中完成 PNG 解码。PNG 解码使用与输出合同一致的资源上限:宽高各不超过 `3840`,解码分配不超过 `8294400 × 4` 字节;不再执行 base64 入站解码
服务端发送给 `platform-image` 时固定注入:
@@ -75,7 +67,7 @@ n = 1
## 预检查与计费事务
所有请求、JSON、base64、图片结构和 provider 参数检查必须在扣费前完成。预检查失败直接返回 4xx,不产生钱包流水,也不调用 provider。
所有 multipart 字段、图片结构和 provider 参数检查必须在扣费前完成。预检查失败直接返回 4xx,不产生钱包流水,也不调用 provider。
检查通过后,api-server 进入现有资产操作计费边界,通过 SpacetimeDB 钱包事务 procedure 原子完成:
@@ -91,13 +83,13 @@ raw 操作使用独立的 operation / ledger 命名空间,例如 `raw-image-ed
## Provider 边界
`platform-image` 保留 VectorEngine 协议细节。raw handler 只负责:认证、JSON DTO、base64 解码、预检查、计费编排和响应映射。provider 请求仍由 `platform-image` 统一构造,并携带 `model``n``quality``background``output_format`、尺寸及图片参考字节。
`platform-image` 保留 VectorEngine 协议细节。raw handler 只负责:认证、multipart 字段解析、PNG 预检查、计费编排和响应映射。provider 请求仍由 `platform-image` 统一构造,并携带 `model``n``quality``background``output_format`、尺寸及图片参考字节。
provider 响应只提取并透传 `data[].b64_json` 字符串,不在服务端解码图片 base64,也不读取或回传 provider 的 `output_format`(该字段只是请求参数回显)。发送、响应读取、上游状态、响应解析和缺图失败必须生成 `PlatformImageFailureAudit`,由 api-server 写入现有外部 API 失败审计链;成功结果同时写入统一的 `external_generation_run` 追踪事件。raw handler 只将上游 `b64_json` 原样写入 `data[].b64_json`
## 代码拆分
- `server-rs/crates/api-server/src/raw_image.rs`:独立路由 handler、请求/响应 DTO、base64 输入校验、预检查和 raw billing 编排。
- `server-rs/crates/api-server/src/raw_image.rs`:独立路由 handler、multipart 字段解析、请求/响应 DTO、PNG 输入校验、预检查和 raw billing 编排。
- `server-rs/crates/platform-image/src/vector_engine/raw_edit.rs`:raw 编辑选项、严格尺寸校验、独立 provider 请求映射和 `b64_json` 响应透传;不复用现有 editor 图片编辑 client 或其 multipart transport。
- `server-rs/crates/api-server/src/modules/raw.rs`:只注册 `/api/raw/v1/images/edit` 并挂载 Bearer middleware。
+24
View File
@@ -470,6 +470,7 @@ dependencies = [
"matchit",
"memchr",
"mime",
"multer",
"percent-encoding",
"pin-project-lite",
"serde_core",
@@ -2924,6 +2925,23 @@ dependencies = [
"pxfm",
]
[[package]]
name = "multer"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
dependencies = [
"bytes",
"encoding_rs",
"futures-util",
"http",
"httparse",
"memchr",
"mime",
"spin",
"version_check",
]
[[package]]
name = "naga"
version = "27.0.3"
@@ -5629,6 +5647,12 @@ dependencies = [
"tokio-tungstenite 0.27.0",
]
[[package]]
name = "spin"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
[[package]]
name = "sse-stream"
version = "0.2.5"
+1 -1
View File
@@ -7,7 +7,7 @@ license.workspace = true
[dependencies]
aes = { workspace = true }
async-stream = { workspace = true }
axum = { workspace = true, features = ["ws"] }
axum = { workspace = true, features = ["ws", "multipart"] }
base64 = { workspace = true }
cbc = { workspace = true }
bytes = { workspace = true }
+221 -104
View File
@@ -1,15 +1,14 @@
use axum::{
Json,
extract::{Extension, State},
extract::{Extension, Multipart, State},
http::StatusCode,
};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use image::{GenericImageView, ImageFormat, ImageReader};
use platform_image::{
RAW_IMAGE_MAX_EDGE, RAW_IMAGE_MAX_PIXELS, RawImageEditOptions, ReferenceImage,
create_vector_engine_raw_image_edit, validate_raw_image_edit_dimensions,
};
use serde::{Deserialize, Serialize};
use serde::Serialize;
use serde_json::json;
use std::io::Cursor;
@@ -29,16 +28,15 @@ use crate::{
};
use time::OffsetDateTime;
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(crate) struct RawImageData {
pub(crate) data: String,
#[derive(Debug)]
struct RawImageData {
pub(crate) bytes: Vec<u8>,
pub(crate) mime_type: String,
pub(crate) file_name: String,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub(crate) struct RawImageEditRequest {
#[derive(Debug)]
struct RawImageEditRequest {
pub(crate) image: RawImageData,
pub(crate) mask: Option<RawImageData>,
pub(crate) prompt: String,
@@ -65,8 +63,9 @@ pub(crate) async fn edit_raw_image(
State(state): State<AppState>,
Extension(request_context): Extension<RequestContext>,
Extension(authenticated): Extension<AuthenticatedAccessToken>,
Json(payload): Json<RawImageEditRequest>,
multipart: Multipart,
) -> Result<Json<RawImageEditResponse>, AppError> {
let payload = parse_multipart_request(multipart).await?;
let prepared = tokio::task::spawn_blocking(move || prepare_request(payload))
.await
.map_err(|error| {
@@ -97,7 +96,7 @@ pub(crate) async fn edit_raw_image(
let generated = match create_vector_engine_raw_image_edit(
&provider_settings,
prepared.prompt.as_str(),
&prepared.image,
prepared.image,
prepared.options,
"raw_image_edit",
)
@@ -153,6 +152,114 @@ struct PreparedRawImageEdit {
height: u32,
}
async fn parse_multipart_request(
mut multipart: Multipart,
) -> Result<RawImageEditRequest, AppError> {
let mut image = None;
let mut mask = None;
let mut prompt = None;
let mut quality = None;
let mut background = None;
let mut output_format = None;
let mut width = None;
let mut height = None;
while let Some(field) = multipart
.next_field()
.await
.map_err(|error| bad_request(format!("multipart 字段读取失败:{error}")))?
{
let name = field
.name()
.ok_or_else(|| bad_request("multipart 字段缺少名称"))?
.to_string();
match name.as_str() {
"image" => {
if image.is_some() {
return Err(bad_request("image 字段不能重复"));
}
image = Some(read_multipart_image(field, "image").await?);
}
"mask" => {
if mask.is_some() {
return Err(bad_request("mask 字段不能重复"));
}
mask = Some(read_multipart_image(field, "mask").await?);
}
"prompt" => set_text_field(&mut prompt, field, "prompt").await?,
"quality" => set_text_field(&mut quality, field, "quality").await?,
"background" => set_text_field(&mut background, field, "background").await?,
"output_format" => set_text_field(&mut output_format, field, "output_format").await?,
"width" => set_text_field(&mut width, field, "width").await?,
"height" => set_text_field(&mut height, field, "height").await?,
_ => return Err(bad_request(format!("不支持的 multipart 字段:{name}"))),
}
}
let image = image.ok_or_else(|| bad_request("image 字段不能为空"))?;
let prompt = prompt.ok_or_else(|| bad_request("prompt 字段不能为空"))?;
let width = parse_multipart_u32(width, "width")?;
let height = parse_multipart_u32(height, "height")?;
Ok(RawImageEditRequest {
image,
mask,
prompt,
quality,
background,
output_format,
width,
height,
})
}
async fn read_multipart_image(
field: axum::extract::multipart::Field<'_>,
name: &str,
) -> Result<RawImageData, AppError> {
let mime_type = field.content_type().unwrap_or_default().to_string();
if !mime_type.eq_ignore_ascii_case("image/png") {
return Err(bad_request(format!("{name} 必须为 image/png")));
}
let bytes = field
.bytes()
.await
.map_err(|error| bad_request(format!("{name} 文件读取失败:{error}")))?;
if bytes.is_empty() {
return Err(bad_request(format!("{name} 文件不能为空")));
}
Ok(RawImageData {
bytes: bytes.to_vec(),
mime_type: "image/png".to_string(),
file_name: format!("{name}.png"),
})
}
async fn set_text_field(
target: &mut Option<String>,
field: axum::extract::multipart::Field<'_>,
name: &str,
) -> Result<(), AppError> {
if target.is_some() {
return Err(bad_request(format!("{name} 字段不能重复")));
}
*target = Some(
field
.text()
.await
.map_err(|error| bad_request(format!("{name} 字段读取失败:{error}")))?,
);
Ok(())
}
fn parse_multipart_u32(value: Option<String>, field: &str) -> Result<u32, AppError> {
let value = value.ok_or_else(|| bad_request(format!("{field} 字段不能为空")))?;
value
.trim()
.parse::<u32>()
.map_err(|_| bad_request(format!("{field} 必须为有效整数")))
}
fn prepare_request(payload: RawImageEditRequest) -> Result<PreparedRawImageEdit, AppError> {
if payload.prompt.trim().is_empty() {
return Err(bad_request("prompt 不能为空"));
@@ -232,24 +339,24 @@ fn validate_optional_value<const N: usize>(
fn decode_image(value: RawImageData, field: &str) -> Result<(ReferenceImage, u32, u32), AppError> {
let mime_type = value.mime_type.trim().to_string();
if !mime_type.eq_ignore_ascii_case("image/png") {
return Err(bad_request(format!("{field}.mimeType 必须为 image/png")));
return Err(bad_request(format!(
"{field} Content-Type 必须为 image/png"
)));
}
let bytes = BASE64_STANDARD
.decode(value.data.trim())
.map_err(|_| bad_request(format!("{field}.data 必须是有效 base64")))?;
let bytes = value.bytes;
if bytes.is_empty() {
return Err(bad_request(format!("{field}.data 不能为空")));
return Err(bad_request(format!("{field} 文件不能为空")));
}
let mut reader = ImageReader::new(Cursor::new(bytes.as_slice()))
.with_guessed_format()
.map_err(|_| bad_request(format!("{field}.data 必须是有效 PNG 文件")))?;
.map_err(|_| bad_request(format!("{field} 文件必须是有效 PNG 文件")))?;
let mut limits = image::Limits::default();
limits.max_image_width = Some(RAW_IMAGE_MAX_EDGE);
limits.max_image_height = Some(RAW_IMAGE_MAX_EDGE);
limits.max_alloc = Some(RAW_IMAGE_MAX_PIXELS.saturating_mul(4));
reader.limits(limits);
if reader.format() != Some(ImageFormat::Png) {
return Err(bad_request(format!("{field}.data 必须是有效 PNG 文件")));
return Err(bad_request(format!("{field} 文件必须是有效 PNG 文件")));
}
let decoded = reader
.decode()
@@ -258,8 +365,8 @@ fn decode_image(value: RawImageData, field: &str) -> Result<(ReferenceImage, u32
Ok((
ReferenceImage {
bytes,
file_name: format!("{field}.png"),
mime_type: "image/png".to_string(),
file_name: value.file_name,
mime_type,
},
width,
height,
@@ -269,9 +376,9 @@ fn decode_image(value: RawImageData, field: &str) -> Result<(ReferenceImage, u32
fn map_decode_image_error(field: &str, error: image::ImageError) -> AppError {
let message = match error {
image::ImageError::Limits(_) => {
format!("{field}.data 超出 PNG 尺寸或解码资源上限(单边不超过 {RAW_IMAGE_MAX_EDGE}px")
format!("{field} 文件超出 PNG 尺寸或解码资源上限(单边不超过 {RAW_IMAGE_MAX_EDGE}px")
}
_ => format!("{field}.data 必须是有效 PNG 文件"),
_ => format!("{field} 文件必须是有效 PNG 文件"),
};
bad_request(message)
}
@@ -302,37 +409,84 @@ fn bad_request(message: impl Into<String>) -> AppError {
#[cfg(test)]
mod tests {
use super::*;
use axum::{body::Body, extract::FromRequest, http::Request};
use image::{ImageFormat, Rgba, RgbaImage};
use std::io::Cursor;
fn encoded_png(width: u32, height: u32) -> String {
fn png_bytes(width: u32, height: u32) -> Vec<u8> {
let image = RgbaImage::from_pixel(width, height, Rgba([255, 0, 0, 255]));
let mut bytes = Vec::new();
image
.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png)
.expect("test PNG should encode");
BASE64_STANDARD.encode(bytes)
bytes
}
#[test]
fn request_uses_one_image_object_and_rejects_images_array() {
let payload = serde_json::json!({
"image": {"data": encoded_png(1, 1), "mimeType": "image/png"},
"prompt": "edit",
"width": 1024,
"height": 1024
});
let parsed: RawImageEditRequest = serde_json::from_value(payload).expect("image object");
let prepared = prepare_request(parsed).expect("request should prepare");
assert!(prepared.image.bytes.starts_with(b"\x89PNG\r\n\x1a\n"));
fn request(image: Vec<u8>, mask: Option<Vec<u8>>) -> RawImageEditRequest {
RawImageEditRequest {
image: RawImageData {
bytes: image,
mime_type: "image/png".to_string(),
file_name: "image.png".to_string(),
},
mask: mask.map(|bytes| RawImageData {
bytes,
mime_type: "image/png".to_string(),
file_name: "mask.png".to_string(),
}),
prompt: "edit".to_string(),
width: 1024,
height: 1024,
quality: None,
background: None,
output_format: None,
}
}
let array_payload = serde_json::json!({
"images": [{"data": "aGVsbG8=", "mimeType": "image/png"}],
"prompt": "edit",
"width": 1024,
"height": 1024
});
assert!(serde_json::from_value::<RawImageEditRequest>(array_payload).is_err());
fn multipart_body(boundary: &str, image: &[u8]) -> Vec<u8> {
let mut body = Vec::new();
let add_text = |body: &mut Vec<u8>, name: &str, value: &str| {
body.extend_from_slice(format!(
"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n"
).as_bytes());
};
add_text(&mut body, "prompt", "edit");
add_text(&mut body, "width", "1024");
add_text(&mut body, "height", "1024");
body.extend_from_slice(
format!(
"--{boundary}\r\nContent-Disposition: form-data; name=\"image\"; filename=\"ignored.png\"\r\nContent-Type: image/png\r\n\r\n"
)
.as_bytes(),
);
body.extend_from_slice(image);
body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes());
body
}
#[tokio::test]
async fn multipart_parser_accepts_binary_image_and_text_fields() {
let boundary = "raw-test-boundary";
let image = png_bytes(1, 1);
let body = multipart_body(boundary, &image);
let request = Request::builder()
.header(
"content-type",
format!("multipart/form-data; boundary={boundary}"),
)
.body(Body::from(body))
.expect("multipart request");
let multipart = Multipart::from_request(request, &())
.await
.expect("multipart");
let parsed = parse_multipart_request(multipart)
.await
.expect("multipart fields should parse");
assert_eq!(parsed.prompt, "edit");
assert_eq!(parsed.width, 1024);
assert_eq!(parsed.height, 1024);
assert!(parsed.image.bytes.starts_with(b"\x89PNG\r\n\x1a\n"));
}
#[test]
@@ -349,24 +503,6 @@ mod tests {
);
}
#[test]
fn invalid_base64_uses_generic_client_message() {
let payload = serde_json::json!({
"image": {"data": "not base64!", "mimeType": "image/png"},
"prompt": "edit",
"width": 1024,
"height": 1024
});
let parsed: RawImageEditRequest = serde_json::from_value(payload).expect("request");
let error = match prepare_request(parsed) {
Ok(_) => panic!("invalid base64 should fail"),
Err(error) => error,
};
let rendered = format!("{error:?}");
assert!(rendered.contains("image.data 必须是有效 base64"));
assert!(!rendered.contains("InvalidByte"));
}
#[test]
fn dimensions_follow_strict_raw_image_contract() {
assert!(validate_raw_image_edit_dimensions(1024, 1024).is_ok());
@@ -380,38 +516,32 @@ mod tests {
#[test]
fn input_requires_decodable_png_and_png_mime() {
let valid = serde_json::json!({
"image": {"data": encoded_png(1, 1), "mimeType": "IMAGE/PNG"},
"prompt": "edit",
"width": 1024,
"height": 1024
});
assert!(prepare_request(serde_json::from_value(valid).expect("valid request")).is_ok());
assert!(prepare_request(request(png_bytes(1, 1), None)).is_ok());
for (data, mime_type) in [("aGVsbG8=", "image/png"), ("aGVsbG8=", "image/jpeg")] {
let payload = serde_json::json!({
"image": {"data": data, "mimeType": mime_type},
"prompt": "edit",
"width": 1024,
"height": 1024
});
assert!(prepare_request(serde_json::from_value(payload).expect("request")).is_err());
}
let invalid_bytes = RawImageEditRequest {
image: RawImageData {
bytes: b"hello".to_vec(),
mime_type: "image/png".to_string(),
file_name: "image.png".to_string(),
},
..request(png_bytes(1, 1), None)
};
assert!(prepare_request(invalid_bytes).is_err());
let invalid_mime = RawImageEditRequest {
image: RawImageData {
bytes: png_bytes(1, 1),
mime_type: "image/jpeg".to_string(),
file_name: "image.jpg".to_string(),
},
..request(png_bytes(1, 1), None)
};
assert!(prepare_request(invalid_mime).is_err());
}
#[test]
fn oversized_valid_png_reports_resource_limit() {
let payload = serde_json::json!({
"image": {
"data": encoded_png(RAW_IMAGE_MAX_EDGE + 1, 1),
"mimeType": "image/png"
},
"prompt": "edit",
"width": 1024,
"height": 1024
});
let parsed: RawImageEditRequest = serde_json::from_value(payload).expect("request");
let error = match prepare_request(parsed) {
let error = match prepare_request(request(png_bytes(RAW_IMAGE_MAX_EDGE + 1, 1), None)) {
Ok(_) => panic!("oversized PNG should fail"),
Err(error) => error,
};
@@ -421,15 +551,7 @@ mod tests {
#[test]
fn mask_must_match_source_image_dimensions() {
let payload = serde_json::json!({
"image": {"data": encoded_png(2, 1), "mimeType": "image/png"},
"mask": {"data": encoded_png(1, 1), "mimeType": "image/png"},
"prompt": "edit",
"width": 1024,
"height": 1024
});
let parsed: RawImageEditRequest = serde_json::from_value(payload).expect("request");
let error = match prepare_request(parsed) {
let error = match prepare_request(request(png_bytes(2, 1), Some(png_bytes(1, 1)))) {
Ok(_) => panic!("mismatched mask should fail before billing"),
Err(error) => error,
};
@@ -439,13 +561,8 @@ mod tests {
#[test]
fn prompt_uses_raw_utf8_byte_limit() {
let payload = serde_json::json!({
"image": {"data": encoded_png(1, 1), "mimeType": "image/png"},
"prompt": "a".repeat(RAW_IMAGE_MAX_PROMPT_BYTES + 1),
"width": 1024,
"height": 1024
});
let parsed: RawImageEditRequest = serde_json::from_value(payload).expect("request");
let mut parsed = request(png_bytes(1, 1), None);
parsed.prompt = "a".repeat(RAW_IMAGE_MAX_PROMPT_BYTES + 1);
let error = match prepare_request(parsed) {
Ok(_) => panic!("oversized prompt should fail before image decode and billing"),
Err(error) => error,
@@ -104,7 +104,7 @@ pub fn validate_raw_image_edit_dimensions(
pub async fn create_vector_engine_raw_image_edit(
settings: &VectorEngineImageSettings,
prompt: &str,
image: &ReferenceImage,
image: ReferenceImage,
options: RawImageEditOptions,
failure_context: &str,
) -> Result<RawImageEditResult, PlatformImageError> {
@@ -126,6 +126,11 @@ pub async fn create_vector_engine_raw_image_edit(
reference_image_count,
));
};
let ReferenceImage {
bytes: image_bytes,
file_name: image_file_name,
mime_type: image_mime_type,
} = image;
let mut form = Form::new()
.text("model", GPT_IMAGE_2_MODEL.to_string())
.text("n", "1".to_string())
@@ -133,9 +138,9 @@ pub async fn create_vector_engine_raw_image_edit(
.text("size", format!("{}x{}", options.width, options.height))
.part(
"image",
Part::bytes(image.bytes.clone())
.file_name(image.file_name.clone())
.mime_str(image.mime_type.as_str())
Part::bytes(image_bytes)
.file_name(image_file_name)
.mime_str(image_mime_type.as_str())
.map_err(|error| invalid_request(failure_context, error.to_string()))?,
);
if let Some(value) = options.quality {