GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into codex/wooden-fish-template

Browse Source
This commit is contained in:
五香丸子
2026-05-22 04:00:52 +08:00
parent 5f1128540e ea9d3aafbf
commit 0faabb27c7
121 changed files with 10876 additions and 3477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

251
server-rs/crates/api-server/src/openai_image_generation.rs
View File

@@ -15,7 +15,7 @@ use crate::{
};
pub(crate) const GPT_IMAGE_2_MODEL: &str = "gpt-image-2";
pub(crate) const VECTOR_ENGINE_GPT_IMAGE_2_MODEL: &str = "gpt-image-2-all";
pub(crate) const VECTOR_ENGINE_GPT_IMAGE_2_MODEL: &str = GPT_IMAGE_2_MODEL;
const VECTOR_ENGINE_PROVIDER: &str = "vector-engine";
#[derive(Clone)]
@@ -62,7 +62,7 @@ pub(crate) struct OpenAiReferenceImage {
pub file_name: String,
}
// 中文注释：RPG、方洞等图片资产统一走 VectorEngine GPT-image-2-all，避免把密钥或供应商协议暴露到前端。
// 中文注释：RPG、方洞等图片资产统一走后端 VectorEngine GPT-image-2，避免把密钥或供应商协议暴露到前端。
pub(crate) fn require_openai_image_settings(
state: &AppState,
) -> Result<OpenAiImageSettings, AppError> {
@@ -106,7 +106,7 @@ pub(crate) fn build_openai_image_http_client(
) -> Result<reqwest::Client, AppError> {
reqwest::Client::builder()
.timeout(Duration::from_millis(settings.request_timeout_ms))
// 中文注释：同一客户端也会承载 `/v1/images/edits` multipart 图生图请求，强制 HTTP/1.1 可避开部分网关对 HTTP/2 multipart 流的兼容问题。
// 中文注释：参考图会走 multipart edits；强制 HTTP/1.1 可避开部分网关对长耗时上传流的兼容问题。
.http1_only()
.build()
.map_err(|error| {
@@ -127,6 +127,22 @@ pub(crate) async fn create_openai_image_generation(
reference_images: &[String],
failure_context: &str,
) -> Result<OpenAiGeneratedImages, AppError> {
if !reference_images.is_empty() {
let resolved_references =
resolve_openai_reference_images(http_client, reference_images, failure_context).await?;
return create_openai_image_edit_with_references(
http_client,
settings,
prompt,
negative_prompt,
size,
candidate_count,
resolved_references.as_slice(),
failure_context,
)
.await;
}
let request_url = vector_engine_images_generation_url(settings);
let normalized_size = normalize_image_size(size);
let request_body = build_openai_image_request_body(
@@ -386,6 +402,7 @@ pub(crate) async fn create_openai_image_edit(
prompt,
negative_prompt,
size,
1,
std::slice::from_ref(reference_image),
failure_context,
)
@@ -398,6 +415,7 @@ pub(crate) async fn create_openai_image_edit_with_references(
prompt: &str,
negative_prompt: Option<&str>,
size: &str,
candidate_count: u32,
reference_images: &[OpenAiReferenceImage],
failure_context: &str,
) -> Result<OpenAiGeneratedImages, AppError> {
@@ -405,12 +423,11 @@ pub(crate) async fn create_openai_image_edit_with_references(
return Err(
AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({
"provider": VECTOR_ENGINE_PROVIDER,
"message": format!("{failure_context}：缺少参考图"),
"message": format!("{failure_context}：缺少参考图，图片编辑需要至少一张参考图。"),
})),
);
}
let task_id = format!("vector-engine-edit-{}", current_utc_micros());
let request_url = vector_engine_images_edit_url(settings);
let normalized_size = normalize_image_size(size);
@@ -420,9 +437,10 @@ pub(crate) async fn create_openai_image_edit_with_references(
"prompt",
build_prompt_with_negative(prompt, negative_prompt),
)
.text("n", "1")
.text("n", candidate_count.clamp(1, 4).to_string())
.text("size", normalized_size.clone());
for reference_image in reference_images {
for reference_image in reference_images.iter().take(5) {
let image_part = reqwest::multipart::Part::bytes(reference_image.bytes.clone())
.file_name(reference_image.file_name.clone())
.mime_str(reference_image.mime_type.as_str())
@@ -434,8 +452,8 @@ pub(crate) async fn create_openai_image_edit_with_references(
form = form.part("image", image_part);
}
let reference_image_count = reference_images.iter().take(5).count();
let started_at = std::time::Instant::now();
let reference_image_count = reference_images.len();
let response = match http_client
.post(request_url.as_str())
.header(
@@ -578,43 +596,51 @@ pub(crate) async fn create_openai_image_edit_with_references(
return Err(error);
}
};
let task_id = extract_generation_id(&response_json.payload)
.unwrap_or_else(|| format!("vector-engine-edit-{}", current_utc_micros()));
let actual_prompt = find_first_string_by_key(&response_json.payload, "revised_prompt")
.or_else(|| find_first_string_by_key(&response_json.payload, "actual_prompt"));
let image_urls = extract_image_urls(&response_json.payload);
if !image_urls.is_empty() {
let download_started_at = std::time::Instant::now();
let mut generated =
match download_images_from_urls(http_client, task_id, image_urls, 1).await {
Ok(generated) => generated,
Err(error) => {
record_openai_image_failure_if_configured(
settings,
build_openai_image_failure_audit_draft(
request_url.as_str(),
failure_context,
"image_download",
Some(response_status.as_u16()),
Some(app_error_status_class(error.status_code())),
false,
false,
error.body_text().as_str(),
None,
None,
Some(download_started_at.elapsed().as_millis() as u64),
Some(prompt.chars().count()),
Some(reference_image_count),
),
)
.await;
return Err(error);
}
};
let mut generated = match download_images_from_urls(
http_client,
task_id,
image_urls,
candidate_count,
)
.await
{
Ok(generated) => generated,
Err(error) => {
record_openai_image_failure_if_configured(
settings,
build_openai_image_failure_audit_draft(
request_url.as_str(),
failure_context,
"image_download",
Some(response_status.as_u16()),
Some(app_error_status_class(error.status_code())),
false,
false,
error.body_text().as_str(),
None,
None,
Some(download_started_at.elapsed().as_millis() as u64),
Some(prompt.chars().count()),
Some(reference_image_count),
),
)
.await;
return Err(error);
}
};
generated.actual_prompt = actual_prompt;
return Ok(generated);
}
let b64_images = extract_b64_images(&response_json.payload);
if !b64_images.is_empty() {
let mut generated = images_from_base64(task_id, b64_images, 1);
let mut generated = images_from_base64(task_id, b64_images, candidate_count);
generated.actual_prompt = actual_prompt;
return Ok(generated);
}
@@ -641,7 +667,7 @@ pub(crate) async fn create_openai_image_edit_with_references(
Err(
AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({
"provider": VECTOR_ENGINE_PROVIDER,
"message": format!("{failure_context}：VectorEngine 未返回编辑图片"),
"message": format!("{failure_context}：VectorEngine 未返回图片"),
})),
)
}
@@ -651,12 +677,12 @@ pub(crate) fn build_openai_image_request_body(
negative_prompt: Option<&str>,
size: &str,
candidate_count: u32,
reference_images: &[String],
_reference_images: &[String],
) -> Value {
let mut body = Map::from_iter([
let body = Map::from_iter([
(
"model".to_string(),
Value::String(VECTOR_ENGINE_GPT_IMAGE_2_MODEL.to_string()),
Value::String(GPT_IMAGE_2_MODEL.to_string()),
),
(
"prompt".to_string(),
@@ -669,10 +695,6 @@ pub(crate) fn build_openai_image_request_body(
),
]);
if !reference_images.is_empty() {
body.insert("image".to_string(), json!(reference_images));
}
Value::Object(body)
}
@@ -784,6 +806,100 @@ pub(crate) async fn download_remote_image(
})
}
async fn resolve_openai_reference_images(
http_client: &reqwest::Client,
reference_images: &[String],
failure_context: &str,
) -> Result<Vec<OpenAiReferenceImage>, AppError> {
let mut resolved = Vec::new();
for (index, source) in reference_images.iter().take(5).enumerate() {
let source = source.trim();
if source.is_empty() {
continue;
}
if let Some(reference_image) = parse_openai_reference_image_data_url(source, index)? {
resolved.push(reference_image);
continue;
}
if source.starts_with("http://") || source.starts_with("https://") {
let downloaded = download_remote_image(http_client, source)
.await
.map_err(|error| {
map_openai_image_request_error(format!(
"{failure_context}：下载参考图失败：{}",
error.body_text()
))
})?;
resolved.push(OpenAiReferenceImage {
bytes: downloaded.bytes,
mime_type: downloaded.mime_type.clone(),
file_name: format!(
"reference-{index}.{}",
mime_to_extension(downloaded.mime_type.as_str())
),
});
continue;
}
return Err(
AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({
"provider": VECTOR_ENGINE_PROVIDER,
"message": format!("{failure_context}：参考图必须是图片 Data URL 或 HTTP(S) URL。"),
})),
);
}
if resolved.is_empty() {
return Err(
AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({
"provider": VECTOR_ENGINE_PROVIDER,
"message": format!("{failure_context}：图片编辑需要至少一张参考图。"),
})),
);
}
Ok(resolved)
}
fn parse_openai_reference_image_data_url(
source: &str,
index: usize,
) -> Result<Option<OpenAiReferenceImage>, AppError> {
let Some(body) = source.strip_prefix("data:") else {
return Ok(None);
};
let Some((mime_type, data)) = body.split_once(";base64,") else {
return Err(
AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({
"provider": VECTOR_ENGINE_PROVIDER,
"message": "参考图 Data URL 必须是 base64 图片。",
})),
);
};
if !mime_type.starts_with("image/") {
return Err(
AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({
"provider": VECTOR_ENGINE_PROVIDER,
"message": "参考图 Data URL 必须是图片类型。",
})),
);
}
let bytes = BASE64_STANDARD.decode(data.trim()).map_err(|error| {
AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({
"provider": VECTOR_ENGINE_PROVIDER,
"message": format!("参考图 Data URL 解码失败：{error}"),
}))
})?;
let mime_type = normalize_downloaded_image_mime_type(mime_type);
Ok(Some(OpenAiReferenceImage {
bytes,
file_name: format!(
"reference-{index}.{}",
mime_to_extension(mime_type.as_str())
),
mime_type,
}))
}
fn parse_json_payload(
raw_text: &str,
failure_context: &str,
@@ -1095,7 +1211,7 @@ mod tests {
use super::*;
#[test]
fn gpt_image_2_request_uses_vector_engine_contract() {
fn gpt_image_2_generation_request_uses_create_model_without_reference_images() {
let body = build_openai_image_request_body(
"雾海神殿",
Some("文字，水印"),
@@ -1104,16 +1220,41 @@ mod tests {
&["data:image/png;base64,abcd".to_string()],
);
assert_eq!(body["model"], VECTOR_ENGINE_GPT_IMAGE_2_MODEL);
assert_eq!(body["model"], GPT_IMAGE_2_MODEL);
assert_eq!(body["size"], "1536x1024");
assert_eq!(body["n"], 2);
assert!(body.get("official_fallback").is_none());
assert_eq!(body["image"][0], "data:image/png;base64,abcd");
assert!(body.get("image").is_none());
assert!(body["prompt"].as_str().unwrap_or_default().contains("避免"));
}
#[test]
fn vector_engine_edit_url_uses_images_edits_endpoint() {
fn vector_engine_generation_url_normalizes_base_url() {
let root_settings = OpenAiImageSettings {
base_url: "https://vector.example".to_string(),
api_key: "test-key".to_string(),
request_timeout_ms: 1_000_000,
external_api_audit_state: None,
};
let v1_settings = OpenAiImageSettings {
base_url: "https://vector.example/v1".to_string(),
api_key: "test-key".to_string(),
request_timeout_ms: 1_000_000,
external_api_audit_state: None,
};
assert_eq!(
vector_engine_images_generation_url(&root_settings),
"https://vector.example/v1/images/generations"
);
assert_eq!(
vector_engine_images_generation_url(&v1_settings),
"https://vector.example/v1/images/generations"
);
}
#[test]
fn vector_engine_edit_url_normalizes_base_url() {
let root_settings = OpenAiImageSettings {
base_url: "https://vector.example".to_string(),
api_key: "test-key".to_string(),
@@ -1153,6 +1294,7 @@ mod tests {
"提示词",
None,
"1:1",
1,
&[],
"测试图片编辑失败",
)
@@ -1163,6 +1305,21 @@ mod tests {
assert!(error.body_text().contains("缺少参考图"));
}
#[test]
fn reference_data_url_resolves_to_edit_image_part() {
let source = format!(
"data:image/png;base64,{}",
BASE64_STANDARD.encode(b"pngbytes")
);
let image = parse_openai_reference_image_data_url(source.as_str(), 2)
.expect("data url should parse")
.expect("data url should resolve image");
assert_eq!(image.bytes, b"pngbytes");
assert_eq!(image.mime_type, "image/png");
assert_eq!(image.file_name, "reference-2.png");
}
#[test]
fn b64_json_response_decodes_png_image() {
let images = images_from_base64(