GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

接入画板生成视频功能

Browse Source 
新增画板底部生成视频入口、Lovart 风格面板、视频图层渲染与元数据展示。

接入 /api/editor/videos/generations 契约与后端 Ark/VectorEngine 视频任务链路。

统一编辑器生成类泥点配置，并补充 UI 设计图、参考图与生成面板结构测试。

更新编辑器技术方案、生成类面板方案和 Hermes 共享决策/踩坑记录。
This commit is contained in:
高物
2026-06-17 20:47:27 +08:00
parent d1cd300695
commit b2fd5574db
39 changed files with 3390 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
server-rs/crates/platform-image/src/lib.rs
View File

@@ -7,8 +7,11 @@ pub use vector_engine::{
PlatformImageFailureAudit, PlatformImageStatusHint, ReferenceImage,
VECTOR_ENGINE_GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER, VectorEngineImageSettings,
build_vector_engine_image_http_client, build_vector_engine_image_request_body,
create_vector_engine_image_edit, create_vector_engine_image_edit_with_references,
build_vector_engine_nanobanana_generate_content_request_body, create_vector_engine_image_edit,
create_vector_engine_image_edit_with_references,
create_vector_engine_image_edit_with_references_and_model,
create_vector_engine_image_generation, create_vector_engine_image_generation_with_model,
download_remote_image, vector_engine_images_edit_url, vector_engine_images_generation_url,
create_vector_engine_nanobanana_generate_content, download_remote_image,
vector_engine_images_edit_url, vector_engine_images_generation_url,
vector_engine_nanobanana_generate_content_url,
};

143
server-rs/crates/platform-image/src/vector_engine/client.rs
View File

@@ -14,9 +14,10 @@ use super::{
image_source::resolve_reference_images,
request::{
build_vector_engine_image_edit_request_log_params,
build_vector_engine_image_request_body_with_model, normalize_image_size,
build_vector_engine_image_request_body_with_model,
build_vector_engine_nanobanana_generate_content_request_body, normalize_image_size,
normalize_vector_engine_image_model, vector_engine_images_edit_url,
vector_engine_images_generation_url,
vector_engine_images_generation_url, vector_engine_nanobanana_generate_content_url,
},
response::handle_vector_engine_response,
types::{GeneratedImages, ReferenceImage, VectorEngineImageSettings},
@@ -181,6 +182,144 @@ pub async fn create_vector_engine_image_generation_with_model(
.await
}
#[allow(clippy::too_many_arguments)]
pub async fn create_vector_engine_nanobanana_generate_content(
http_client: &reqwest::Client,
settings: &VectorEngineImageSettings,
model: &str,
prompt: &str,
negative_prompt: Option<&str>,
aspect_ratio: &str,
image_size: &str,
reference_images: &[ReferenceImage],
failure_context: &str,
) -> Result<GeneratedImages, PlatformImageError> {
let model = normalize_vector_engine_image_model(model);
let request_url = vector_engine_nanobanana_generate_content_url(settings, model);
let request_body = build_vector_engine_nanobanana_generate_content_request_body(
prompt,
negative_prompt,
aspect_ratio,
image_size,
reference_images,
);
let reference_image_count = reference_images.iter().take(14).count();
let reference_image_bytes_total: usize = reference_images
.iter()
.take(14)
.map(|image| image.bytes.len())
.sum();
let request_params = serde_json::json!({
"model": model,
"promptChars": prompt.trim().chars().count(),
"negativePromptChars": negative_prompt
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::chars)
.map(Iterator::count)
.unwrap_or_default(),
"aspectRatio": aspect_ratio,
"imageSize": image_size,
"referenceImageCount": reference_image_count,
"referenceImageBytesTotal": reference_image_bytes_total,
});
let started_at = std::time::Instant::now();
let mut attempt = 1;
let response = loop {
match send_vector_engine_json_request_with_curl(
request_url.as_str(),
settings.api_key.as_str(),
&request_body,
settings.request_timeout_ms,
)
.await
{
Ok(response) => {
if should_retry_vector_engine_upstream_status(response.status, attempt) {
retry_vector_engine_upstream_status_after_delay(
"nanobanana_generate_content",
request_url.as_str(),
attempt,
response.status,
response.body.as_str(),
started_at.elapsed().as_millis() as u64,
Some(prompt.chars().count()),
Some(reference_image_count),
Some(&request_params),
)
.await;
attempt += 1;
continue;
}
break response;
}
Err(error) => {
if should_retry_vector_engine_curl_send_error(&error, attempt) {
retry_vector_engine_send_after_delay(
"nanobanana_generate_content",
request_url.as_str(),
"request_send",
attempt,
error.is_timeout(),
error.is_connect() || error.is_transient_transport(),
true,
false,
error.to_string().as_str(),
started_at.elapsed().as_millis() as u64,
Some(prompt.chars().count()),
Some(reference_image_count),
Some(&request_params),
)
.await;
attempt += 1;
continue;
}
return Err(map_curl_error(
format!("{failure_context}：创建 nanobanana2 图片生成任务失败").as_str(),
request_url.as_str(),
"request_send",
error,
started_at.elapsed().as_millis() as u64,
Some(prompt.chars().count()),
Some(reference_image_count),
Some(&request_params),
));
}
}
};
let response_status = response.status;
tracing::info!(
provider = VECTOR_ENGINE_PROVIDER,
endpoint = %request_url,
status = response_status,
image_model = model,
prompt_chars = prompt.chars().count(),
aspect_ratio,
image_size,
reference_image_count,
reference_image_bytes_total,
request_params = %request_params,
attempt,
elapsed_ms = started_at.elapsed().as_millis() as u64,
failure_context,
"VectorEngine nanobanana2 图片生成 HTTP 返回"
);
let response_text = response.body;
handle_vector_engine_response(
http_client,
request_url.as_str(),
response_status,
response_text.as_str(),
failure_context,
started_at.elapsed().as_millis() as u64,
Some(prompt.chars().count()),
Some(reference_image_count),
1,
"vector-engine-nanobanana",
)
.await
}
pub async fn create_vector_engine_image_edit(
http_client: &reqwest::Client,
settings: &VectorEngineImageSettings,

5
server-rs/crates/platform-image/src/vector_engine/mod.rs
View File

@@ -16,13 +16,16 @@ pub use client::{
create_vector_engine_image_edit, create_vector_engine_image_edit_with_references,
create_vector_engine_image_edit_with_references_and_model,
create_vector_engine_image_generation, create_vector_engine_image_generation_with_model,
create_vector_engine_nanobanana_generate_content,
};
pub use constants::{GPT_IMAGE_2_MODEL, VECTOR_ENGINE_GPT_IMAGE_2_MODEL, VECTOR_ENGINE_PROVIDER};
pub use error::{PlatformImageError, PlatformImageStatusHint};
pub use image_source::download_remote_image;
pub use request::{
build_vector_engine_image_request_body, build_vector_engine_image_request_body_with_model,
normalize_image_size, vector_engine_images_edit_url, vector_engine_images_generation_url,
build_vector_engine_nanobanana_generate_content_request_body, normalize_image_size,
vector_engine_images_edit_url, vector_engine_images_generation_url,
vector_engine_nanobanana_generate_content_url,
};
pub use transport::build_vector_engine_image_http_client;
pub use types::{DownloadedImage, GeneratedImages, ReferenceImage, VectorEngineImageSettings};

39
server-rs/crates/platform-image/src/vector_engine/payload.rs
View File

@@ -88,9 +88,48 @@ pub(super) fn extract_image_urls(payload: &Value) -> Vec<String> {
pub(super) fn extract_b64_images(payload: &Value) -> Vec<String> {
let mut values = Vec::new();
collect_strings_by_key(payload, "b64_json", &mut values);
collect_inline_image_data(payload, &mut values);
values
}
fn collect_inline_image_data(value: &Value, results: &mut Vec<String>) {
match value {
Value::Array(entries) => {
for entry in entries {
collect_inline_image_data(entry, results);
}
}
Value::Object(object) => {
for key in ["inlineData", "inline_data"] {
if let Some(Value::Object(inline_data)) = object.get(key) {
let mime_type = inline_data
.get("mimeType")
.or_else(|| inline_data.get("mime_type"))
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or("image/png")
.to_ascii_lowercase();
if !mime_type.is_empty() && !mime_type.starts_with("image/") {
continue;
}
if let Some(data) = inline_data
.get("data")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
{
results.push(data.to_string());
}
}
}
for nested_value in object.values() {
collect_inline_image_data(nested_value, results);
}
}
_ => {}
}
}
pub(super) fn parse_api_error_message(raw_text: &str, fallback_message: &str) -> String {
if raw_text.trim().is_empty() {
return fallback_message.to_string();

81
server-rs/crates/platform-image/src/vector_engine/request.rs
View File

@@ -32,10 +32,7 @@ pub fn build_vector_engine_image_request_body_with_model(
) -> Value {
let model = normalize_vector_engine_image_model(model);
let body = Map::from_iter([
(
"model".to_string(),
Value::String(model.to_string()),
),
("model".to_string(), Value::String(model.to_string())),
(
"prompt".to_string(),
Value::String(build_prompt_with_negative(prompt, negative_prompt)),
@@ -50,6 +47,42 @@ pub fn build_vector_engine_image_request_body_with_model(
Value::Object(body)
}
pub fn build_vector_engine_nanobanana_generate_content_request_body(
prompt: &str,
negative_prompt: Option<&str>,
aspect_ratio: &str,
image_size: &str,
reference_images: &[ReferenceImage],
) -> Value {
let prompt = build_prompt_with_negative(prompt, negative_prompt);
let mut parts = vec![json!({ "text": prompt })];
for reference_image in reference_images.iter().take(14) {
parts.push(json!({
"inline_data": {
"mime_type": reference_image.mime_type,
"data": base64::Engine::encode(
&base64::engine::general_purpose::STANDARD,
reference_image.bytes.as_slice()
),
},
}));
}
json!({
"contents": [{
"role": "user",
"parts": parts,
}],
"generationConfig": {
"responseModalities": ["IMAGE"],
"imageConfig": {
"aspectRatio": normalize_nanobanana_aspect_ratio(aspect_ratio),
"imageSize": normalize_nanobanana_image_size(image_size),
},
},
})
}
pub fn normalize_vector_engine_image_model(model: &str) -> &str {
match model.trim() {
"" => GPT_IMAGE_2_MODEL,
@@ -71,6 +104,31 @@ pub fn normalize_image_size(size: &str) -> String {
.to_string()
}
fn normalize_nanobanana_aspect_ratio(aspect_ratio: &str) -> &str {
match aspect_ratio.trim() {
"2:3" => "2:3",
"3:2" => "3:2",
"3:4" => "3:4",
"4:3" => "4:3",
"4:5" => "4:5",
"5:4" => "5:4",
"9:16" => "9:16",
"16:9" => "16:9",
"21:9" => "21:9",
_ => "1:1",
}
}
fn normalize_nanobanana_image_size(image_size: &str) -> &str {
match image_size.trim() {
// 中文注释：nanobanana / Gemini 3.1 的 0.5K 在 VectorEngine 文档中要求传 512。
"512" | "0.5K" => "512",
"2K" => "2K",
"4K" => "4K",
_ => "1K",
}
}
pub fn vector_engine_images_generation_url(settings: &VectorEngineImageSettings) -> String {
if settings.base_url.ends_with("/v1") {
format!("{}/images/generations", settings.base_url)
@@ -87,6 +145,21 @@ pub fn vector_engine_images_edit_url(settings: &VectorEngineImageSettings) -> St
}
}
pub fn vector_engine_nanobanana_generate_content_url(
settings: &VectorEngineImageSettings,
model: &str,
) -> String {
let base_url = settings
.base_url
.trim_end_matches("/v1")
.trim_end_matches('/');
format!(
"{}/v1beta/models/{}:generateContent",
base_url,
normalize_vector_engine_image_model(model)
)
}
pub(crate) fn build_vector_engine_image_edit_request_log_params(
model: &str,
prompt: &str,

60
server-rs/crates/platform-image/src/vector_engine/tests.rs
View File

@@ -66,8 +66,10 @@ mod tests {
assert_eq!(reference.mime_type, "image/png");
assert_eq!(reference.bytes, b"\x89PNG\r\n\x1A\nrest");
let image = decode_generated_image_base64(BASE64_STANDARD.encode(b"\x89PNG\r\n\x1A\nrest").as_str())
.expect("base64 image should decode");
let image = decode_generated_image_base64(
BASE64_STANDARD.encode(b"\x89PNG\r\n\x1A\nrest").as_str(),
)
.expect("base64 image should decode");
assert_eq!(image.extension, "png");
assert_eq!(image.mime_type, "image/png");
assert_eq!(image.bytes, b"\x89PNG\r\n\x1A\nrest");
@@ -121,10 +123,22 @@ mod tests {
audit: Some(audit.clone()),
};
assert_eq!(invalid_config.status_hint(), PlatformImageStatusHint::ServiceUnavailable);
assert_eq!(invalid_request.status_hint(), PlatformImageStatusHint::BadRequest);
assert_eq!(request_error.status_hint(), PlatformImageStatusHint::GatewayTimeout);
assert_eq!(upstream_timeout.status_hint(), PlatformImageStatusHint::GatewayTimeout);
assert_eq!(
invalid_config.status_hint(),
PlatformImageStatusHint::ServiceUnavailable
);
assert_eq!(
invalid_request.status_hint(),
PlatformImageStatusHint::BadRequest
);
assert_eq!(
request_error.status_hint(),
PlatformImageStatusHint::GatewayTimeout
);
assert_eq!(
upstream_timeout.status_hint(),
PlatformImageStatusHint::GatewayTimeout
);
assert_eq!(
PlatformImageError::MissingImage {
provider: VECTOR_ENGINE_PROVIDER,
@@ -137,7 +151,10 @@ mod tests {
let audit_ref = upstream_timeout.audit().expect("audit should be preserved");
assert_eq!(audit_ref.provider, VECTOR_ENGINE_PROVIDER);
assert_eq!(audit_ref.endpoint, "https://vector.example/v1/images/generations");
assert_eq!(
audit_ref.endpoint,
"https://vector.example/v1/images/generations"
);
assert_eq!(audit_ref.status_code, Some(504));
assert_eq!(audit_ref.status_class, Some("5xx"));
assert!(audit_ref.timeout);
@@ -158,7 +175,27 @@ mod tests {
{"url": "https://example.com/b.png"}
],
"nested": {
"b64_json": ["YWJj", "ZGVm"]
"b64_json": ["YWJj", "ZGVm"],
"parts": [
{
"inlineData": {
"mimeType": "image/png",
"data": "aW1hZ2UtMQ=="
}
},
{
"inline_data": {
"mime_type": "image/jpeg",
"data": "aW1hZ2UtMg=="
}
},
{
"inlineData": {
"mimeType": "text/plain",
"data": "bm90LWltYWdl"
}
}
]
}
});
@@ -171,7 +208,12 @@ mod tests {
);
assert_eq!(
extract_b64_images(&payload),
vec!["YWJj".to_string(), "ZGVm".to_string()]
vec![
"YWJj".to_string(),
"ZGVm".to_string(),
"aW1hZ2UtMQ==".to_string(),
"aW1hZ2UtMg==".to_string(),
]
);
}
}

127
server-rs/crates/platform-image/tests/vector_engine.rs
View File

@@ -1,9 +1,11 @@
use platform_image::vector_engine::{
GPT_IMAGE_2_MODEL, ReferenceImage, VECTOR_ENGINE_PROVIDER, VectorEngineImageSettings,
build_vector_engine_image_http_client, build_vector_engine_image_request_body,
build_vector_engine_image_request_body_with_model, create_vector_engine_image_edit,
create_vector_engine_image_generation,
build_vector_engine_image_request_body_with_model,
build_vector_engine_nanobanana_generate_content_request_body, create_vector_engine_image_edit,
create_vector_engine_image_generation, create_vector_engine_nanobanana_generate_content,
vector_engine_images_edit_url, vector_engine_images_generation_url,
vector_engine_nanobanana_generate_content_url,
};
use std::{
sync::{
@@ -69,6 +71,60 @@ fn vector_engine_request_body_can_use_nanobanana2_model() {
assert_eq!(body["n"], 1);
}
#[test]
fn vector_engine_request_body_can_use_nanobanana2_half_k() {
let body = build_vector_engine_image_request_body_with_model(
"gemini-3.1-flash-image-preview",
"生成图标 spritesheet",
None,
"512",
1,
&[],
);
assert_eq!(body["model"], "gemini-3.1-flash-image-preview");
assert_eq!(body["size"], "512");
}
#[test]
fn nanobanana_generate_content_body_carries_aspect_ratio_and_image_size() {
let body = build_vector_engine_nanobanana_generate_content_request_body(
"生成角色图",
Some("文字、水印"),
"2:3",
"512",
&[],
);
assert_eq!(body["contents"][0]["role"], "user");
assert_eq!(
body["contents"][0]["parts"][0]["text"],
"生成角色图\n避免：文字、水印"
);
assert_eq!(body["generationConfig"]["responseModalities"][0], "IMAGE");
assert_eq!(
body["generationConfig"]["imageConfig"]["aspectRatio"],
"2:3"
);
assert_eq!(body["generationConfig"]["imageConfig"]["imageSize"], "512");
assert!(body.get("model").is_none());
assert!(body.get("n").is_none());
}
#[test]
fn nanobanana_generate_content_url_uses_model_path() {
let settings = VectorEngineImageSettings {
base_url: "https://vector.example/v1".to_string(),
api_key: "test-key".to_string(),
request_timeout_ms: 1_000,
};
assert_eq!(
vector_engine_nanobanana_generate_content_url(&settings, "gemini-3.1-flash-image-preview"),
"https://vector.example/v1beta/models/gemini-3.1-flash-image-preview:generateContent"
);
}
#[tokio::test]
async fn vector_engine_image_edit_retries_send_timeout_once_and_succeeds() {
let listener = TcpListener::bind("127.0.0.1:0")
@@ -136,6 +192,73 @@ async fn vector_engine_image_edit_retries_send_timeout_once_and_succeeds() {
server.abort();
}
#[tokio::test]
async fn nanobanana_generate_content_posts_native_body_and_reads_inline_data() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("mock server should bind");
let server_addr = listener
.local_addr()
.expect("mock server address should be readable");
let server = tokio::spawn(async move {
let Ok((mut stream, _)) = listener.accept().await else {
return;
};
let mut request = Vec::new();
let mut buffer = [0_u8; 4096];
loop {
let Ok(read) = stream.read(&mut buffer).await else {
return;
};
if read == 0 {
return;
}
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
let request_text = String::from_utf8_lossy(request.as_slice());
assert!(
request_text.contains("/v1beta/models/gemini-3.1-flash-image-preview:generateContent")
);
assert!(request_text.contains("\"aspectRatio\":\"2:3\""));
assert!(request_text.contains("\"imageSize\":\"512\""));
let body = r#"{"candidates":[{"content":{"parts":[{"inlineData":{"mimeType":"image/png","data":"iVBORw0KGgpyZXN0"}}]}}]}"#;
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}",
body.len(),
body
);
let _ = stream.write_all(response.as_bytes()).await;
});
let settings = VectorEngineImageSettings {
base_url: format!("http://{}", server_addr),
api_key: "test-key".to_string(),
request_timeout_ms: 1_000,
};
let client = build_vector_engine_image_http_client(&settings).expect("client should build");
let generated = create_vector_engine_nanobanana_generate_content(
&client,
&settings,
"gemini-3.1-flash-image-preview",
"生成角色图",
Some("文字、水印"),
"2:3",
"512",
&[],
"测试 nanobanana",
)
.await
.expect("nanobanana response should parse");
assert_eq!(generated.images.len(), 1);
assert_eq!(generated.images[0].mime_type, "image/png");
server.abort();
}
#[tokio::test]
async fn vector_engine_image_generation_retries_upstream_502_once_and_succeeds() {
let listener = TcpListener::bind("127.0.0.1:0")