GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1

Browse Source
This commit is contained in:
高物
2026-05-08 20:48:29 +08:00
parent abf1f1ebea
commit 94975e4735
82 changed files with 7786 additions and 1012 deletions
Download Patch File Download Diff File Expand all files Collapse all files

272
server-rs/crates/api-server/src/puzzle.rs
View File

@@ -13,12 +13,13 @@ use axum::{
},
};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use image::ImageFormat;
use module_assets::{
AssetObjectAccessPolicy, AssetObjectFieldError, build_asset_entity_binding_input,
build_asset_object_upsert_input, generate_asset_binding_id, generate_asset_object_id,
};
use module_puzzle::{PuzzleGeneratedImageCandidate, PuzzleRuntimeLevelStatus};
use platform_llm::{LlmMessage, LlmTextRequest};
use platform_llm::{LlmMessage, LlmMessageContentPart, LlmTextRequest};
use platform_oss::{
LegacyAssetPrefix, OssHeadObjectRequest, OssObjectAccess, OssPutObjectRequest,
OssSignedGetObjectUrlRequest,
@@ -78,7 +79,7 @@ use crate::{
},
auth::AuthenticatedAccessToken,
http_error::AppError,
llm_model_routing::CREATION_TEMPLATE_LLM_MODEL,
llm_model_routing::{CREATION_TEMPLATE_LLM_MODEL, PUZZLE_LEVEL_NAME_VISION_LLM_MODEL},
platform_errors::map_oss_error,
prompt::puzzle::{
draft::{
@@ -88,6 +89,7 @@ use crate::{
image::{PUZZLE_DEFAULT_NEGATIVE_PROMPT, build_puzzle_image_prompt},
level_name::{
PUZZLE_FIRST_LEVEL_NAME_SYSTEM_PROMPT, build_puzzle_first_level_name_user_prompt,
build_puzzle_first_level_name_vision_user_text,
},
tags::{PUZZLE_TAG_GENERATION_SYSTEM_PROMPT, build_puzzle_tag_generation_user_prompt},
},
@@ -112,6 +114,7 @@ const PUZZLE_ENTITY_KIND: &str = "puzzle_work";
const PUZZLE_GENERATED_IMAGE_SIZE: &str = "1024*1024";
const PUZZLE_APIMART_GENERATED_IMAGE_SIZE: &str = "1:1";
const PUZZLE_APIMART_GEMINI_RESOLUTION: &str = "1K";
const PUZZLE_LEVEL_NAME_VISION_IMAGE_MAX_SIDE: u32 = 768;
pub async fn create_puzzle_agent_session(
State(state): State<AppState>,
@@ -204,7 +207,8 @@ pub async fn generate_puzzle_onboarding_work(
PUZZLE_AGENT_API_BASE_PROVIDER,
map_puzzle_generation_endpoint_error(error),
)
})?;
})?
.into_records();
let selected = candidates.first().cloned().ok_or_else(|| {
puzzle_error_response(
&request_context,
@@ -864,8 +868,9 @@ pub async fn execute_puzzle_agent_action(
if let Some(levels_json) = levels_json.as_ref() {
draft.levels = parse_puzzle_level_records_from_module_json(levels_json)?;
}
let target_level =
let mut target_level =
select_puzzle_level_for_api(&draft, target_level_id.as_deref())?;
let fallback_level_name = target_level.level_name.clone();
let prompt = resolve_puzzle_level_image_prompt(
payload.prompt_text.as_deref(),
&target_level.picture_description,
@@ -886,10 +891,32 @@ pub async fn execute_puzzle_agent_action(
)
.await
.map_err(map_puzzle_generation_endpoint_error)?;
if candidates.is_empty() {
return Err(AppError::from_status(StatusCode::BAD_GATEWAY).with_details(
json!({
"provider": PUZZLE_AGENT_API_BASE_PROVIDER,
"message": "拼图候选图生成结果为空",
}),
));
}
if let Some(refined_level_name) = generate_puzzle_first_level_name_from_image(
&state,
target_level.picture_description.as_str(),
&candidates[0].downloaded_image,
)
.await
{
target_level.level_name = refined_level_name;
}
let generated_level_name = target_level.level_name.clone();
let levels_json_with_generated_name =
Some(serialize_puzzle_level_records_for_module(
&build_puzzle_levels_with_primary_name(&draft, &target_level),
)?);
let candidates_json = serde_json::to_string(
&candidates
.iter()
.map(to_puzzle_generated_image_candidate)
.map(|candidate| to_puzzle_generated_image_candidate(&candidate.record))
.collect::<Vec<_>>(),
)
.map_err(|error| {
@@ -904,7 +931,7 @@ pub async fn execute_puzzle_agent_action(
session_id: session.session_id.clone(),
owner_user_id: owner_user_id.clone(),
level_id: Some(target_level.level_id.clone()),
levels_json,
levels_json: levels_json_with_generated_name,
candidates_json,
saved_at_micros: now,
})
@@ -925,9 +952,15 @@ pub async fn execute_puzzle_agent_action(
let fallback_session =
replace_puzzle_session_draft_snapshot(session, draft, now);
Ok(apply_generated_puzzle_candidates_to_session_snapshot(
fallback_session,
apply_generated_puzzle_first_level_name_to_session_snapshot(
fallback_session,
target_level.level_id.as_str(),
generated_level_name.as_str(),
fallback_level_name.as_str(),
now,
),
target_level.level_id.as_str(),
candidates,
candidates.into_records(),
now,
))
}
@@ -2830,6 +2863,91 @@ async fn generate_puzzle_first_level_name(state: &AppState, picture_description:
build_fallback_puzzle_first_level_name(picture_description)
}
async fn generate_puzzle_first_level_name_from_image(
state: &AppState,
picture_description: &str,
image: &PuzzleDownloadedImage,
) -> Option<String> {
let Some(llm_client) = state.creative_agent_gpt5_client() else {
return None;
};
let Some(image_data_url) = build_puzzle_level_name_image_data_url(image) else {
tracing::warn!(
provider = PUZZLE_AGENT_API_BASE_PROVIDER,
picture_chars = picture_description.chars().count(),
"拼图首关名图片输入压缩失败，保留文本关卡名"
);
return None;
};
let user_text = build_puzzle_first_level_name_vision_user_text(picture_description);
let response = llm_client
.request_text(
LlmTextRequest::new(vec![
LlmMessage::system(PUZZLE_FIRST_LEVEL_NAME_SYSTEM_PROMPT),
LlmMessage::user_multimodal(vec![
LlmMessageContentPart::InputText { text: user_text },
LlmMessageContentPart::InputImage {
image_url: image_data_url,
},
]),
])
.with_model(PUZZLE_LEVEL_NAME_VISION_LLM_MODEL)
.with_max_tokens(80),
)
.await;
match response {
Ok(response) => {
parse_puzzle_first_level_name_from_text(response.content.as_str()).or_else(|| {
tracing::warn!(
provider = PUZZLE_AGENT_API_BASE_PROVIDER,
model = PUZZLE_LEVEL_NAME_VISION_LLM_MODEL,
picture_chars = picture_description.chars().count(),
"拼图首关名视觉模型返回非法，保留文本关卡名"
);
None
})
}
Err(error) => {
tracing::warn!(
provider = PUZZLE_AGENT_API_BASE_PROVIDER,
model = PUZZLE_LEVEL_NAME_VISION_LLM_MODEL,
picture_chars = picture_description.chars().count(),
error = %error,
"拼图首关名视觉生成失败，保留文本关卡名"
);
None
}
}
}
fn build_puzzle_level_name_image_data_url(image: &PuzzleDownloadedImage) -> Option<String> {
let bytes = resize_puzzle_level_name_image_bytes(image.bytes.as_slice())
.unwrap_or_else(|| image.bytes.clone());
let mime_type = if bytes.starts_with(b"\x89PNG\r\n\x1A\n") {
"image/png"
} else {
image.mime_type.as_str()
};
Some(format!(
"data:{};base64,{}",
normalize_puzzle_downloaded_image_mime_type(mime_type),
BASE64_STANDARD.encode(bytes)
))
}
fn resize_puzzle_level_name_image_bytes(bytes: &[u8]) -> Option<Vec<u8>> {
let image = image::load_from_memory(bytes).ok()?;
let resized = image.resize(
PUZZLE_LEVEL_NAME_VISION_IMAGE_MAX_SIDE,
PUZZLE_LEVEL_NAME_VISION_IMAGE_MAX_SIDE,
image::imageops::FilterType::Triangle,
);
let mut cursor = std::io::Cursor::new(Vec::new());
resized.write_to(&mut cursor, ImageFormat::Png).ok()?;
Some(cursor.into_inner())
}
fn parse_puzzle_first_level_name_from_text(text: &str) -> Option<String> {
let trimmed = text.trim();
let json_text = if let Some(start) = trimmed.find('{')
@@ -2985,9 +3103,6 @@ async fn compile_puzzle_draft_with_initial_cover(
let generated_level_name =
generate_puzzle_first_level_name(state, &target_level.picture_description).await;
target_level.level_name = generated_level_name.clone();
let levels_json_with_generated_name = Some(serialize_puzzle_level_records_for_module(
&build_puzzle_levels_with_primary_name(&draft, &target_level),
)?);
let image_prompt = resolve_puzzle_draft_cover_prompt(
prompt_text,
&target_level.picture_description,
@@ -3008,19 +3123,32 @@ async fn compile_puzzle_draft_with_initial_cover(
.await?;
let selected_candidate_id = candidates
.iter()
.find(|candidate| candidate.selected)
.find(|candidate| candidate.record.selected)
.or_else(|| candidates.first())
.map(|candidate| candidate.candidate_id.clone())
.map(|candidate| candidate.record.candidate_id.clone())
.ok_or_else(|| {
AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({
"provider": PUZZLE_AGENT_API_BASE_PROVIDER,
"message": "拼图候选图生成结果为空",
}))
})?;
if let Some(refined_level_name) = generate_puzzle_first_level_name_from_image(
state,
target_level.picture_description.as_str(),
&candidates[0].downloaded_image,
)
.await
{
target_level.level_name = refined_level_name;
}
let generated_level_name = target_level.level_name.clone();
let levels_json_with_generated_name = Some(serialize_puzzle_level_records_for_module(
&build_puzzle_levels_with_primary_name(&draft, &target_level),
)?);
let candidates_json = serde_json::to_string(
&candidates
.iter()
.map(to_puzzle_generated_image_candidate)
.map(|candidate| to_puzzle_generated_image_candidate(&candidate.record))
.collect::<Vec<_>>(),
)
.map_err(|error| {
@@ -3061,7 +3189,7 @@ async fn compile_puzzle_draft_with_initial_cover(
now,
),
target_level.level_id.as_str(),
candidates.clone(),
candidates.into_records(),
now,
);
Ok((session, true))
@@ -3138,9 +3266,6 @@ async fn compile_puzzle_draft_with_uploaded_cover(
let generated_level_name =
generate_puzzle_first_level_name(state, &target_level.picture_description).await;
target_level.level_name = generated_level_name.clone();
let levels_json_with_generated_name = Some(serialize_puzzle_level_records_for_module(
&build_puzzle_levels_with_primary_name(&draft, &target_level),
)?);
let image_prompt = resolve_puzzle_draft_cover_prompt(
prompt_text,
&target_level.picture_description,
@@ -3152,6 +3277,24 @@ async fn compile_puzzle_draft_with_uploaded_cover(
compiled_session.session_id,
target_level.candidates.len() + 1
);
let uploaded_downloaded_image = PuzzleDownloadedImage {
extension: puzzle_mime_to_extension(uploaded_image.mime_type.as_str()).to_string(),
mime_type: normalize_puzzle_downloaded_image_mime_type(uploaded_image.mime_type.as_str()),
bytes: uploaded_image.bytes,
};
if let Some(refined_level_name) = generate_puzzle_first_level_name_from_image(
state,
target_level.picture_description.as_str(),
&uploaded_downloaded_image,
)
.await
{
target_level.level_name = refined_level_name;
}
let generated_level_name = target_level.level_name.clone();
let levels_json_with_generated_name = Some(serialize_puzzle_level_records_for_module(
&build_puzzle_levels_with_primary_name(&draft, &target_level),
)?);
let persisted_upload = persist_puzzle_generated_asset(
state,
owner_user_id.as_str(),
@@ -3159,13 +3302,7 @@ async fn compile_puzzle_draft_with_uploaded_cover(
&target_level.level_name,
candidate_id.as_str(),
"uploaded-direct",
PuzzleDownloadedImage {
extension: puzzle_mime_to_extension(uploaded_image.mime_type.as_str()).to_string(),
mime_type: normalize_puzzle_downloaded_image_mime_type(
uploaded_image.mime_type.as_str(),
),
bytes: uploaded_image.bytes,
},
uploaded_downloaded_image,
current_utc_micros(),
)
.await?;
@@ -3865,7 +4002,7 @@ async fn generate_puzzle_image_candidates(
image_model: Option<&str>,
candidate_count: u32,
candidate_start_index: usize,
) -> Result<Vec<PuzzleGeneratedImageCandidateRecord>, AppError> {
) -> Result<Vec<GeneratedPuzzleImageCandidate>, AppError> {
let count = candidate_count.clamp(1, 1);
let resolved_model = resolve_puzzle_image_model(image_model);
let actual_prompt = build_puzzle_image_prompt(level_name, prompt);
@@ -3914,6 +4051,7 @@ async fn generate_puzzle_image_candidates(
"{session_id}-candidate-{}",
candidate_start_index + index + 1
);
let downloaded_image = image.clone();
let asset = persist_puzzle_generated_asset(
state,
owner_user_id,
@@ -3926,30 +4064,22 @@ async fn generate_puzzle_image_candidates(
)
.await
.map_err(map_puzzle_generation_endpoint_error)?;
items.push(PuzzleGeneratedImageCandidateResponse {
candidate_id,
image_src: asset.image_src,
asset_id: asset.asset_id,
prompt: prompt.to_string(),
actual_prompt: Some(actual_prompt.clone()),
source_type: resolved_model.candidate_source_type().to_string(),
// 单图生成结果总是直接成为当前正式图。
selected: index == 0,
items.push(GeneratedPuzzleImageCandidate {
record: PuzzleGeneratedImageCandidateRecord {
candidate_id,
image_src: asset.image_src,
asset_id: asset.asset_id,
prompt: prompt.to_string(),
actual_prompt: Some(actual_prompt.clone()),
source_type: resolved_model.candidate_source_type().to_string(),
// 单图生成结果总是直接成为当前正式图。
selected: index == 0,
},
downloaded_image,
});
}
Ok(items
.into_iter()
.map(|candidate| PuzzleGeneratedImageCandidateRecord {
candidate_id: candidate.candidate_id,
image_src: candidate.image_src,
asset_id: candidate.asset_id,
prompt: candidate.prompt,
actual_prompt: candidate.actual_prompt,
source_type: candidate.source_type,
selected: candidate.selected,
})
.collect())
Ok(items)
}
#[cfg(test)]
@@ -3977,6 +4107,7 @@ mod tests {
assert_eq!(body["size"], PUZZLE_APIMART_GENERATED_IMAGE_SIZE);
assert_eq!(body["resolution"], PUZZLE_APIMART_GEMINI_RESOLUTION);
assert_eq!(body["n"], 1);
assert_eq!(body["official_fallback"], true);
assert_eq!(body["image_urls"][0], "data:image/png;base64,abcd");
assert!(
body["prompt"]
@@ -4014,6 +4145,7 @@ mod tests {
prompt_text: None,
reference_image_src: None,
image_model: Some(PUZZLE_IMAGE_MODEL_GPT_IMAGE_2.to_string()),
ai_redraw: None,
candidate_count: Some(1),
candidate_id: None,
level_id: Some("puzzle-level-1".to_string()),
@@ -4073,6 +4205,26 @@ mod tests {
);
}
#[test]
fn puzzle_level_name_image_data_url_downsizes_generated_image() {
let image = image::DynamicImage::ImageRgb8(image::RgbImage::new(4, 4));
let mut cursor = std::io::Cursor::new(Vec::new());
image
.write_to(&mut cursor, ImageFormat::Png)
.expect("test image should encode");
let downloaded = PuzzleDownloadedImage {
extension: "png".to_string(),
mime_type: "image/png".to_string(),
bytes: cursor.into_inner(),
};
let data_url = build_puzzle_level_name_image_data_url(&downloaded)
.expect("data url should be generated");
assert!(data_url.starts_with("data:image/png;base64,"));
assert!(data_url.len() > "data:image/png;base64,".len());
}
#[test]
fn puzzle_first_level_name_snapshot_defaults_work_title() {
let levels_json = serde_json::to_string(&vec![json!({
@@ -4091,6 +4243,7 @@ mod tests {
prompt_text: None,
reference_image_src: None,
image_model: Some(PUZZLE_IMAGE_MODEL_GPT_IMAGE_2.to_string()),
ai_redraw: None,
candidate_count: Some(1),
candidate_id: None,
level_id: Some("puzzle-level-1".to_string()),
@@ -4181,6 +4334,30 @@ struct PuzzleGeneratedImages {
images: Vec<PuzzleDownloadedImage>,
}
struct GeneratedPuzzleImageCandidate {
record: PuzzleGeneratedImageCandidateRecord,
downloaded_image: PuzzleDownloadedImage,
}
impl GeneratedPuzzleImageCandidate {
fn into_record(self) -> PuzzleGeneratedImageCandidateRecord {
self.record
}
}
trait GeneratedPuzzleImageCandidatesExt {
fn into_records(self) -> Vec<PuzzleGeneratedImageCandidateRecord>;
}
impl GeneratedPuzzleImageCandidatesExt for Vec<GeneratedPuzzleImageCandidate> {
fn into_records(self) -> Vec<PuzzleGeneratedImageCandidateRecord> {
self.into_iter()
.map(GeneratedPuzzleImageCandidate::into_record)
.collect()
}
}
#[derive(Clone)]
struct PuzzleDownloadedImage {
extension: String,
mime_type: String,
@@ -4361,6 +4538,7 @@ fn build_puzzle_apimart_image_request_body(
Value::String(build_puzzle_apimart_prompt(prompt, negative_prompt)),
),
("n".to_string(), json!(candidate_count.clamp(1, 1))),
("official_fallback".to_string(), Value::Bool(true)),
("size".to_string(), Value::String(size.to_string())),
]);
body.insert(