Files
Genarrative/server-rs/crates/api-server/src/openai_image_generation.rs
T
kdletters ae375a8e5f 合并编辑器素材库能力
合并 codex/editor-asset-library 到宣发素材演示分支

保留素材库、生成器、音频、UI设计图和持久化相关能力

补回宣发素材入口与 5 泥点计费契约
2026-06-18 22:36:54 +08:00

742 lines
24 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use axum::http::StatusCode;
use platform_image::{
DownloadedImage, GeneratedImages, PlatformImageError, PlatformImageStatusHint, ReferenceImage,
VECTOR_ENGINE_PROVIDER, VectorEngineImageSettings, build_vector_engine_image_http_client,
create_vector_engine_image_edit, create_vector_engine_image_edit_with_references,
create_vector_engine_image_edit_with_references_and_model,
create_vector_engine_image_generation, create_vector_engine_image_generation_with_model,
create_vector_engine_nanobanana_generate_content,
};
#[cfg(test)]
use platform_image::{
build_vector_engine_image_request_body, vector_engine_images_edit_url,
vector_engine_images_generation_url,
};
use serde_json::{Value, json};
use time::OffsetDateTime;
use crate::{
external_api_audit::{
ExternalApiFailureDraft, build_external_api_failure_draft_from_platform_image_audit,
record_external_api_failure,
},
http_error::AppError,
request_context::RequestContext,
state::AppState,
tracking::record_external_generation_run_after_success,
};
pub(crate) use platform_image::GPT_IMAGE_2_MODEL;
#[cfg(test)]
use platform_image::VECTOR_ENGINE_GPT_IMAGE_2_MODEL;
pub(crate) type OpenAiGeneratedImages = GeneratedImages;
pub(crate) type DownloadedOpenAiImage = DownloadedImage;
pub(crate) type OpenAiReferenceImage = ReferenceImage;
#[derive(Clone)]
pub(crate) struct OpenAiImageSettings {
pub base_url: String,
pub api_key: String,
pub request_timeout_ms: u64,
pub external_api_audit_state: Option<AppState>,
pub external_api_audit_user_id: Option<String>,
pub external_api_audit_profile_id: Option<String>,
pub external_api_audit_request_id: Option<String>,
}
impl std::fmt::Debug for OpenAiImageSettings {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("OpenAiImageSettings")
.field("base_url", &self.base_url)
.field("api_key", &"<redacted>")
.field("request_timeout_ms", &self.request_timeout_ms)
.field(
"external_api_audit_enabled",
&self.external_api_audit_state.is_some(),
)
.field(
"external_api_audit_user_id",
&self.external_api_audit_user_id,
)
.field(
"external_api_audit_profile_id",
&self.external_api_audit_profile_id,
)
.field(
"external_api_audit_request_id",
&self.external_api_audit_request_id,
)
.finish()
}
}
// 中文注释:api-server 只负责配置、审计和 HTTP envelopeVectorEngine 协议细节统一由 platform-image provider 承接。
pub(crate) fn require_openai_image_settings(
state: &AppState,
) -> Result<OpenAiImageSettings, AppError> {
let base_url = state
.config
.vector_engine_base_url
.trim()
.trim_end_matches('/');
if base_url.is_empty() {
return Err(
AppError::from_status(StatusCode::SERVICE_UNAVAILABLE).with_details(json!({
"provider": VECTOR_ENGINE_PROVIDER,
"reason": "VECTOR_ENGINE_BASE_URL 未配置",
})),
);
}
let api_key = state
.config
.vector_engine_api_key
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.ok_or_else(|| {
AppError::from_status(StatusCode::SERVICE_UNAVAILABLE).with_details(json!({
"provider": VECTOR_ENGINE_PROVIDER,
"reason": "VECTOR_ENGINE_API_KEY 未配置",
}))
})?;
Ok(OpenAiImageSettings {
base_url: base_url.to_string(),
api_key: api_key.to_string(),
request_timeout_ms: state.config.vector_engine_image_request_timeout_ms.max(1),
external_api_audit_state: Some(state.clone()),
external_api_audit_user_id: None,
external_api_audit_profile_id: None,
external_api_audit_request_id: None,
})
}
pub(crate) fn build_openai_image_http_client(
settings: &OpenAiImageSettings,
) -> Result<reqwest::Client, AppError> {
build_vector_engine_image_http_client(&settings.provider_settings())
.map_err(map_platform_image_error)
}
pub(crate) async fn create_openai_image_generation(
http_client: &reqwest::Client,
settings: &OpenAiImageSettings,
prompt: &str,
negative_prompt: Option<&str>,
size: &str,
candidate_count: u32,
reference_images: &[String],
failure_context: &str,
) -> Result<OpenAiGeneratedImages, AppError> {
let started_at_micros = current_utc_micros();
let request_payload = json!({
"size": size,
"candidateCount": candidate_count,
"promptChars": prompt.chars().count(),
"negativePromptChars": negative_prompt.map(str::chars).map(Iterator::count),
"referenceImageCount": reference_images.len(),
});
let result = create_vector_engine_image_generation(
http_client,
&settings.provider_settings(),
prompt,
negative_prompt,
size,
candidate_count,
reference_images,
failure_context,
)
.await;
map_platform_image_result(
settings,
result,
"image_generation",
failure_context,
request_payload,
started_at_micros,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn create_openai_image_generation_with_model(
http_client: &reqwest::Client,
settings: &OpenAiImageSettings,
model: &str,
prompt: &str,
negative_prompt: Option<&str>,
size: &str,
candidate_count: u32,
reference_images: &[String],
failure_context: &str,
) -> Result<OpenAiGeneratedImages, AppError> {
let started_at_micros = current_utc_micros();
let request_payload = json!({
"model": model,
"size": size,
"candidateCount": candidate_count,
"promptChars": prompt.chars().count(),
"negativePromptChars": negative_prompt.map(str::chars).map(Iterator::count),
"referenceImageCount": reference_images.len(),
});
let result = create_vector_engine_image_generation_with_model(
http_client,
&settings.provider_settings(),
model,
prompt,
negative_prompt,
size,
candidate_count,
reference_images,
failure_context,
)
.await;
map_platform_image_result(
settings,
result,
"image_generation",
failure_context,
request_payload,
started_at_micros,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn create_openai_nanobanana_generate_content(
http_client: &reqwest::Client,
settings: &OpenAiImageSettings,
model: &str,
prompt: &str,
negative_prompt: Option<&str>,
aspect_ratio: &str,
image_size: &str,
reference_images: &[OpenAiReferenceImage],
failure_context: &str,
) -> Result<OpenAiGeneratedImages, AppError> {
let started_at_micros = current_utc_micros();
let request_payload = json!({
"model": model,
"aspectRatio": aspect_ratio,
"imageSize": image_size,
"promptChars": prompt.chars().count(),
"negativePromptChars": negative_prompt.map(str::chars).map(Iterator::count),
"referenceImageCount": reference_images.len(),
});
let result = create_vector_engine_nanobanana_generate_content(
http_client,
&settings.provider_settings(),
model,
prompt,
negative_prompt,
aspect_ratio,
image_size,
reference_images,
failure_context,
)
.await;
map_platform_image_result(
settings,
result,
"nanobanana_generate_content",
failure_context,
request_payload,
started_at_micros,
)
.await
}
pub(crate) async fn create_openai_image_edit(
http_client: &reqwest::Client,
settings: &OpenAiImageSettings,
prompt: &str,
negative_prompt: Option<&str>,
size: &str,
reference_image: &OpenAiReferenceImage,
failure_context: &str,
) -> Result<OpenAiGeneratedImages, AppError> {
let started_at_micros = current_utc_micros();
let request_payload = json!({
"size": size,
"promptChars": prompt.chars().count(),
"negativePromptChars": negative_prompt.map(str::chars).map(Iterator::count),
"referenceImageCount": 1,
});
let result = create_vector_engine_image_edit(
http_client,
&settings.provider_settings(),
prompt,
negative_prompt,
size,
reference_image,
failure_context,
)
.await;
map_platform_image_result(
settings,
result,
"image_edit",
failure_context,
request_payload,
started_at_micros,
)
.await
}
pub(crate) async fn create_openai_image_edit_with_references(
http_client: &reqwest::Client,
settings: &OpenAiImageSettings,
prompt: &str,
negative_prompt: Option<&str>,
size: &str,
candidate_count: u32,
reference_images: &[OpenAiReferenceImage],
failure_context: &str,
) -> Result<OpenAiGeneratedImages, AppError> {
let started_at_micros = current_utc_micros();
let request_payload = json!({
"size": size,
"candidateCount": candidate_count,
"promptChars": prompt.chars().count(),
"negativePromptChars": negative_prompt.map(str::chars).map(Iterator::count),
"referenceImageCount": reference_images.len(),
});
let result = create_vector_engine_image_edit_with_references(
http_client,
&settings.provider_settings(),
prompt,
negative_prompt,
size,
candidate_count,
reference_images,
failure_context,
)
.await;
map_platform_image_result(
settings,
result,
"image_edit_with_references",
failure_context,
request_payload,
started_at_micros,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn create_openai_image_edit_with_references_and_model(
http_client: &reqwest::Client,
settings: &OpenAiImageSettings,
model: &str,
prompt: &str,
negative_prompt: Option<&str>,
size: &str,
candidate_count: u32,
reference_images: &[OpenAiReferenceImage],
failure_context: &str,
) -> Result<OpenAiGeneratedImages, AppError> {
let started_at_micros = current_utc_micros();
let request_payload = json!({
"model": model,
"size": size,
"promptChars": prompt.chars().count(),
"negativePromptChars": negative_prompt.map(str::chars).map(Iterator::count),
"referenceImageCount": reference_images.len(),
});
let result = create_vector_engine_image_edit_with_references_and_model(
http_client,
&settings.provider_settings(),
model,
prompt,
negative_prompt,
size,
candidate_count,
reference_images,
failure_context,
)
.await;
map_platform_image_result(
settings,
result,
"image_edit_with_references",
failure_context,
request_payload,
started_at_micros,
)
.await
}
#[cfg(test)]
pub(crate) fn build_openai_image_request_body(
prompt: &str,
negative_prompt: Option<&str>,
size: &str,
candidate_count: u32,
reference_images: &[String],
) -> Value {
build_vector_engine_image_request_body(
prompt,
negative_prompt,
size,
candidate_count,
reference_images,
)
}
impl OpenAiImageSettings {
pub(crate) fn with_external_api_audit_actor(
mut self,
user_id: Option<String>,
profile_id: Option<String>,
) -> Self {
self.external_api_audit_user_id = user_id;
self.external_api_audit_profile_id = profile_id;
self
}
pub(crate) fn with_external_api_audit_context(
mut self,
request_context: &RequestContext,
user_id: Option<String>,
profile_id: Option<String>,
) -> Self {
self.external_api_audit_user_id = user_id;
self.external_api_audit_profile_id = profile_id;
self.external_api_audit_request_id = Some(request_context.request_id().to_string());
self
}
fn provider_settings(&self) -> VectorEngineImageSettings {
VectorEngineImageSettings {
base_url: self.base_url.clone(),
api_key: self.api_key.clone(),
request_timeout_ms: self.request_timeout_ms.max(1),
}
}
}
async fn map_platform_image_result(
settings: &OpenAiImageSettings,
result: Result<OpenAiGeneratedImages, PlatformImageError>,
operation: &'static str,
failure_context: &str,
request_payload: Value,
started_at_micros: i64,
) -> Result<OpenAiGeneratedImages, AppError> {
match result {
Ok(value) => {
if let Some(state) = settings.external_api_audit_state.as_ref() {
record_external_generation_run_after_success(
state,
VECTOR_ENGINE_PROVIDER,
operation,
failure_context,
request_payload,
started_at_micros,
true,
None,
Some(value.task_id.clone()),
Some(json!({
"imageCount": value.images.len(),
"actualPromptChars": value.actual_prompt.as_ref().map(|prompt| prompt.chars().count()),
})),
)
.await;
}
Ok(value)
}
Err(error) => {
if let Some(state) = settings.external_api_audit_state.as_ref() {
record_external_generation_run_after_success(
state,
VECTOR_ENGINE_PROVIDER,
operation,
failure_context,
request_payload,
started_at_micros,
false,
Some(error.message().to_string()),
None,
None,
)
.await;
}
record_openai_image_failure_if_configured(settings, &error).await;
Err(map_platform_image_error(error))
}
}
}
pub(crate) async fn record_openai_image_failure_if_configured(
settings: &OpenAiImageSettings,
error: &PlatformImageError,
) {
let Some(state) = settings.external_api_audit_state.as_ref() else {
return;
};
let Some(draft) = build_openai_image_failure_audit_draft(error) else {
return;
};
let draft = draft
.with_user_id(settings.external_api_audit_user_id.clone())
.with_profile_id(settings.external_api_audit_profile_id.clone())
.with_request_id(settings.external_api_audit_request_id.clone());
record_external_api_failure(state, draft).await;
}
pub(crate) fn build_openai_image_failure_audit_draft(
error: &PlatformImageError,
) -> Option<ExternalApiFailureDraft> {
error
.audit()
.map(build_external_api_failure_draft_from_platform_image_audit)
}
pub(crate) fn map_platform_image_error(error: PlatformImageError) -> AppError {
let status = match error.status_hint() {
PlatformImageStatusHint::BadRequest => StatusCode::BAD_REQUEST,
PlatformImageStatusHint::ServiceUnavailable => StatusCode::SERVICE_UNAVAILABLE,
PlatformImageStatusHint::BadGateway => StatusCode::BAD_GATEWAY,
PlatformImageStatusHint::GatewayTimeout => StatusCode::GATEWAY_TIMEOUT,
};
let mut details = json!({
"provider": error.provider(),
"message": error.message(),
});
match &error {
PlatformImageError::InvalidConfig { .. } | PlatformImageError::InvalidRequest { .. } => {}
PlatformImageError::Request {
endpoint,
timeout,
connect,
request,
body,
status_code,
source,
..
} => {
details["endpoint"] = json!(endpoint);
details["timeout"] = json!(timeout);
details["connect"] = json!(connect);
details["request"] = json!(request);
details["body"] = json!(body);
details["status"] = json!(status_code);
details["source"] = json!(source);
}
PlatformImageError::Upstream {
upstream_status,
raw_excerpt,
..
} => {
details["upstreamStatus"] = json!(upstream_status);
details["rawExcerpt"] = json!(raw_excerpt);
}
PlatformImageError::ResponseParse { raw_excerpt, .. } => {
details["rawExcerpt"] = json!(raw_excerpt);
}
PlatformImageError::MissingImage { .. } => {}
}
if let Some(audit) = error.audit() {
details["endpoint"] = json!(audit.endpoint);
details["failureStage"] = json!(audit.failure_stage);
details["statusClass"] = json!(audit.status_class);
details["retryable"] = json!(audit.retryable);
details["timeout"] = json!(audit.timeout);
details["latencyMs"] = json!(audit.latency_ms);
details["promptChars"] = json!(audit.prompt_chars);
details["referenceImageCount"] = json!(audit.reference_image_count);
details["imageModel"] = json!(audit.image_model);
details["rawExcerpt"] = json!(audit.raw_excerpt);
details["errorSource"] = json!(audit.error_source);
}
AppError::from_status(status).with_details(details)
}
#[cfg(test)]
fn vector_engine_images_generation_url_for_test(settings: &OpenAiImageSettings) -> String {
vector_engine_images_generation_url(&settings.provider_settings())
}
#[cfg(test)]
fn vector_engine_images_edit_url_for_test(settings: &OpenAiImageSettings) -> String {
vector_engine_images_edit_url(&settings.provider_settings())
}
#[cfg(test)]
mod tests {
use super::*;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
#[test]
fn gpt_image_2_generation_request_uses_create_model_without_reference_images() {
let body = build_openai_image_request_body(
"雾海神殿",
Some("文字,水印"),
"16:9",
2,
&["data:image/png;base64,abcd".to_string()],
);
assert_eq!(body["model"], GPT_IMAGE_2_MODEL);
assert_eq!(body["size"], "1536x1024");
assert_eq!(body["n"], 2);
assert!(body.get("official_fallback").is_none());
assert!(body.get("image").is_none());
assert!(body["prompt"].as_str().unwrap_or_default().contains("避免"));
}
#[test]
fn vector_engine_generation_url_normalizes_base_url() {
let root_settings = OpenAiImageSettings {
base_url: "https://vector.example".to_string(),
api_key: "test-key".to_string(),
request_timeout_ms: 1_000_000,
external_api_audit_state: None,
external_api_audit_user_id: None,
external_api_audit_profile_id: None,
external_api_audit_request_id: None,
};
let v1_settings = OpenAiImageSettings {
base_url: "https://vector.example/v1".to_string(),
api_key: "test-key".to_string(),
request_timeout_ms: 1_000_000,
external_api_audit_state: None,
external_api_audit_user_id: None,
external_api_audit_profile_id: None,
external_api_audit_request_id: None,
};
assert_eq!(
vector_engine_images_generation_url_for_test(&root_settings),
"https://vector.example/v1/images/generations"
);
assert_eq!(
vector_engine_images_generation_url_for_test(&v1_settings),
"https://vector.example/v1/images/generations"
);
}
#[test]
fn vector_engine_edit_url_normalizes_base_url() {
let root_settings = OpenAiImageSettings {
base_url: "https://vector.example".to_string(),
api_key: "test-key".to_string(),
request_timeout_ms: 1_000_000,
external_api_audit_state: None,
external_api_audit_user_id: None,
external_api_audit_profile_id: None,
external_api_audit_request_id: None,
};
let v1_settings = OpenAiImageSettings {
base_url: "https://vector.example/v1".to_string(),
api_key: "test-key".to_string(),
request_timeout_ms: 1_000_000,
external_api_audit_state: None,
external_api_audit_user_id: None,
external_api_audit_profile_id: None,
external_api_audit_request_id: None,
};
assert_eq!(
vector_engine_images_edit_url_for_test(&root_settings),
"https://vector.example/v1/images/edits"
);
assert_eq!(
vector_engine_images_edit_url_for_test(&v1_settings),
"https://vector.example/v1/images/edits"
);
}
#[tokio::test]
async fn vector_engine_multi_reference_edit_rejects_empty_references() {
let settings = OpenAiImageSettings {
base_url: "https://vector.example".to_string(),
api_key: "test-key".to_string(),
request_timeout_ms: 1_000_000,
external_api_audit_state: None,
external_api_audit_user_id: None,
external_api_audit_profile_id: None,
external_api_audit_request_id: None,
};
let http_client = reqwest::Client::new();
let result = create_openai_image_edit_with_references(
&http_client,
&settings,
"提示词",
None,
"1:1",
1,
&[],
"测试图片编辑失败",
)
.await;
let error = result.expect_err("empty references should be rejected locally");
assert_eq!(error.status_code(), StatusCode::BAD_REQUEST);
assert!(error.body_text().contains("缺少参考图"));
}
#[test]
fn reference_data_url_stays_provider_owned() {
let source = format!(
"data:image/png;base64,{}",
BASE64_STANDARD.encode(b"pngbytes")
);
let body = build_openai_image_request_body("提示词", None, "1:1", 1, &[source]);
assert!(body.get("image").is_none());
}
#[test]
fn vector_engine_upstream_failure_builds_tracking_ready_audit_event() {
let audit = platform_image::PlatformImageFailureAudit {
provider: VECTOR_ENGINE_PROVIDER,
endpoint: "https://vector.example/v1/images/generations".to_string(),
operation: "拼图 UI 背景图生成失败".to_string(),
failure_stage: "upstream_status",
status_code: Some(429),
status_class: None,
timeout: false,
retryable: true,
error_message: "上游限流".to_string(),
error_source: None,
raw_excerpt: Some("{\"error\":\"rate limited\"}".to_string()),
latency_ms: Some(321),
prompt_chars: Some(42),
reference_image_count: Some(1),
image_model: Some(VECTOR_ENGINE_GPT_IMAGE_2_MODEL),
};
let tracking = crate::external_api_audit::build_external_api_failure_tracking_draft(
&build_external_api_failure_draft_from_platform_image_audit(&audit),
);
assert_eq!(
tracking.event_key,
crate::external_api_audit::EXTERNAL_API_FAILURE_EVENT_KEY
);
assert_eq!(tracking.scope_id, VECTOR_ENGINE_PROVIDER);
assert_eq!(tracking.metadata["provider"], VECTOR_ENGINE_PROVIDER);
assert_eq!(tracking.metadata["statusCode"], 429);
assert_eq!(tracking.metadata["statusClass"], "4xx");
assert_eq!(tracking.metadata["failureStage"], "upstream_status");
assert_eq!(tracking.metadata["retryable"], true);
assert_eq!(tracking.metadata["promptChars"], 42);
assert_eq!(tracking.metadata["referenceImageCount"], 1);
assert_eq!(
tracking.metadata["imageModel"],
VECTOR_ENGINE_GPT_IMAGE_2_MODEL
);
}
}
fn current_utc_micros() -> i64 {
(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000) as i64
}