Files
Genarrative/server-rs/crates/api-server/src/match3d.rs

610 lines
20 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use std::{
collections::BTreeMap,
convert::Infallible,
future::Future,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use axum::{
Json,
extract::{Extension, Path, State, rejection::JsonRejection},
http::{HeaderName, StatusCode, header},
response::{
IntoResponse, Response,
sse::{Event, Sse},
},
};
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use futures_util::{StreamExt, stream::FuturesUnordered};
use image::{GenericImageView, ImageFormat};
use module_match3d::{
MATCH3D_MESSAGE_ID_PREFIX, MATCH3D_PROFILE_ID_PREFIX, MATCH3D_RUN_ID_PREFIX,
MATCH3D_SESSION_ID_PREFIX,
};
use platform_llm::{LlmMessage, LlmTextRequest};
use platform_oss::{LegacyAssetPrefix, OssObjectAccess, OssPutObjectRequest};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use shared_contracts::{
creation_audio::CreationAudioAsset,
hyper3d as hyper3d_contract,
match3d_agent::{
CreateMatch3DAgentSessionRequest, ExecuteMatch3DAgentActionRequest,
Match3DAgentActionResponse, Match3DAgentMessageResponse, Match3DAgentSessionResponse,
Match3DAgentSessionSnapshotResponse, Match3DAnchorItemResponse, Match3DAnchorPackResponse,
Match3DCreatorConfigResponse,
Match3DGeneratedItemAssetResponse as Match3DAgentGeneratedItemAssetResponse,
Match3DResultDraftResponse, SendMatch3DAgentMessageRequest,
},
match3d_runtime::{
ClickMatch3DItemRequest, Match3DClickConfirmationResponse, Match3DClickResponse,
Match3DItemSnapshotResponse, Match3DRunResponse, Match3DRunSnapshotResponse,
Match3DTraySlotResponse, StartMatch3DRunRequest, StopMatch3DRunRequest,
},
match3d_works::{
GenerateMatch3DBackgroundImageRequest, GenerateMatch3DBackgroundImageResponse,
GenerateMatch3DContainerImageRequest, GenerateMatch3DContainerImageResponse,
GenerateMatch3DCoverImageRequest, GenerateMatch3DCoverImageResponse,
GenerateMatch3DItemAssetsRequest, GenerateMatch3DItemAssetsResponse,
Match3DWorkDetailResponse, Match3DWorkMutationResponse, Match3DWorkProfileResponse,
Match3DWorkSummaryResponse, Match3DWorksResponse, PersistMatch3DGeneratedModelRequest,
PersistMatch3DGeneratedModelResponse, PutMatch3DAudioAssetsRequest, PutMatch3DWorkRequest,
},
};
use shared_kernel::build_prefixed_uuid_id;
use spacetime_client::{
Match3DAgentMessageFinalizeRecordInput, Match3DAgentMessageRecord,
Match3DAgentMessageSubmitRecordInput, Match3DAgentSessionCreateRecordInput,
Match3DAgentSessionRecord, Match3DAnchorItemRecord, Match3DAnchorPackRecord,
Match3DClickConfirmationRecord, Match3DCompileDraftRecordInput, Match3DCreatorConfigRecord,
Match3DItemSnapshotRecord, Match3DResultDraftRecord, Match3DRunClickRecordInput,
Match3DRunRecord, Match3DRunRestartRecordInput, Match3DRunStartRecordInput,
Match3DRunStopRecordInput, Match3DRunTimeUpRecordInput, Match3DTraySlotRecord,
Match3DWorkProfileRecord, Match3DWorkUpdateRecordInput, SpacetimeClientError,
};
use crate::{
api_response::json_success_body,
asset_billing::{
execute_billable_asset_operation_with_cost, map_asset_operation_wallet_error,
should_skip_asset_operation_billing_for_connectivity,
},
auth::AuthenticatedAccessToken,
config::AppConfig,
http_error::AppError,
openai_image_generation::{
DownloadedOpenAiImage, OpenAiGeneratedImages, OpenAiReferenceImage,
build_openai_image_http_client, create_openai_image_edit, create_openai_image_generation,
require_openai_image_settings,
},
platform_errors::map_oss_error,
request_context::RequestContext,
state::AppState,
vector_engine_audio_generation::{
GeneratedCreationAudioTarget, generate_sound_effect_asset_for_creation,
},
work_play_tracking::{WorkPlayTrackingDraft, record_work_play_start_after_success},
};
const MATCH3D_AGENT_PROVIDER: &str = "match3d-agent";
const MATCH3D_WORKS_PROVIDER: &str = "match3d-works";
const MATCH3D_RUNTIME_PROVIDER: &str = "match3d-runtime";
const MATCH3D_DEFAULT_THEME: &str = "缤纷玩具";
const MATCH3D_DEFAULT_CLEAR_COUNT: u32 = 12;
const MATCH3D_DEFAULT_DIFFICULTY: u32 = 4;
const MATCH3D_DRAFT_GENERATION_POINTS_COST: u64 = 10;
const MATCH3D_BACKGROUND_IMAGE_POINTS_COST: u64 = 2;
const MATCH3D_ITEM_ASSETS_POINTS_PER_BATCH: u64 = 2;
const MATCH3D_MATERIAL_ITEM_BATCH_SIZE: usize = 5;
const MATCH3D_ITEM_VIEW_COUNT: usize = 5;
const MATCH3D_MATERIAL_GRID_SIZE: u32 = 5;
const MATCH3D_MATERIAL_FOREGROUND_DIFF_THRESHOLD: i32 = 36;
const MATCH3D_MATERIAL_FOREGROUND_ALPHA_THRESHOLD: i32 = 36;
const MATCH3D_MATERIAL_GREEN_SCREEN_MIN_SCORE: f32 = 0.34;
const MATCH3D_MATERIAL_GREEN_SCREEN_SOFT_SCORE: f32 = 0.18;
const MATCH3D_MATERIAL_HARD_GREEN_SCREEN_SCORE: f32 = 0.82;
const MATCH3D_MAX_GENERATED_ITEM_COUNT: usize = 25;
const MATCH3D_MATERIAL_VECTOR_ENGINE_GEMINI_MODEL: &str = "gemini-3-pro-image-preview";
const MATCH3D_MATERIAL_VECTOR_ENGINE_GEMINI_ASPECT_RATIO: &str = "1:1";
const MATCH3D_LEGACY_MODEL_DOWNLOAD_TIMEOUT_MS: u64 = 3 * 60_000;
const MATCH3D_OSS_PUT_TIMEOUT_MS: u64 = 3 * 60_000;
const MATCH3D_LEGACY_MODEL_MAX_BYTES: usize = 120 * 1024 * 1024;
const MATCH3D_ITEM_IMAGE_MAX_BYTES: usize = 20 * 1024 * 1024;
const MATCH3D_WORK_METADATA_LLM_MODEL: &str = "gpt-4o";
const MATCH3D_ITEM_SIZE_LARGE: &str = "";
const MATCH3D_ITEM_SIZE_MEDIUM: &str = "";
const MATCH3D_ITEM_SIZE_SMALL: &str = "";
const MATCH3D_CONTAINER_REFERENCE_IMAGE_PATH: &str =
"public/match3d-background-references/pot-fused-reference.png";
const MATCH3D_QUESTION_THEME: &str = "你想创作什么题材";
const MATCH3D_QUESTION_CLEAR_COUNT: &str = "需要消除多少次才能通关";
const MATCH3D_QUESTION_DIFFICULTY: &str = "如果难度是从1-10你要创作的关卡是难度几";
const MATCH3D_CLICK_SOUND_ASSET_KIND: &str = "match3d_click_sound";
const MATCH3D_PIXEL_RETRO_STYLE_PROMPT: &str = "真正复古像素 2D 游戏道具 sprite 风格,先以约 64x64 低分辨率像素块绘制再按整数倍放大,硬边方块像素清晰可见,有限色板 12-24 色,禁止抗锯齿、柔焦、平滑渐变、真实 3D 渲染、PBR 材质和摄影光照。";
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct Match3DConfigJson {
theme_text: String,
reference_image_src: Option<String>,
clear_count: u32,
difficulty: u32,
#[serde(default)]
asset_style_id: Option<String>,
#[serde(default)]
asset_style_label: Option<String>,
#[serde(default)]
asset_style_prompt: Option<String>,
#[serde(default)]
generate_click_sound: bool,
}
#[derive(Clone, Debug)]
struct Match3DGeneratedItemAsset {
item_id: String,
item_name: String,
item_size: Option<String>,
image_src: Option<String>,
image_object_key: Option<String>,
image_views: Vec<Match3DGeneratedItemImageView>,
model_src: Option<String>,
model_object_key: Option<String>,
model_file_name: Option<String>,
task_uuid: Option<String>,
subscription_key: Option<String>,
sound_prompt: Option<String>,
background_music_title: Option<String>,
background_music_style: Option<String>,
background_music_prompt: Option<String>,
background_music: Option<CreationAudioAsset>,
click_sound: Option<CreationAudioAsset>,
background_asset: Option<Match3DGeneratedBackgroundAsset>,
status: String,
error: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
struct Match3DGeneratedItemImageView {
view_id: String,
view_index: u32,
#[serde(default)]
image_src: Option<String>,
#[serde(default)]
image_object_key: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct Match3DGeneratedBackgroundAsset {
prompt: String,
#[serde(default)]
image_src: Option<String>,
#[serde(default)]
image_object_key: Option<String>,
#[serde(default)]
container_prompt: Option<String>,
#[serde(default)]
container_image_src: Option<String>,
#[serde(default)]
container_image_object_key: Option<String>,
status: String,
#[serde(default)]
error: Option<String>,
}
#[derive(Clone, Debug)]
struct Match3DGeneratedWorkMetadata {
game_name: String,
summary: String,
tags: Vec<String>,
}
#[derive(Clone, Debug)]
struct Match3DGeneratedItemPlan {
name: String,
item_size: String,
sound_prompt: String,
}
#[derive(Clone, Debug)]
struct Match3DGeneratedDraftPlan {
metadata: Match3DGeneratedWorkMetadata,
items: Vec<Match3DGeneratedItemPlan>,
background_prompt: String,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct Match3DGeneratedItemAssetJson {
item_id: String,
item_name: String,
#[serde(default)]
item_size: Option<String>,
#[serde(default)]
image_src: Option<String>,
#[serde(default)]
image_object_key: Option<String>,
#[serde(default)]
image_views: Vec<Match3DGeneratedItemImageView>,
#[serde(default)]
model_src: Option<String>,
#[serde(default)]
model_object_key: Option<String>,
#[serde(default)]
model_file_name: Option<String>,
#[serde(default)]
task_uuid: Option<String>,
#[serde(default)]
subscription_key: Option<String>,
#[serde(default)]
sound_prompt: Option<String>,
#[serde(default)]
background_music_title: Option<String>,
#[serde(default)]
background_music_style: Option<String>,
#[serde(default)]
background_music_prompt: Option<String>,
#[serde(default)]
background_music: Option<CreationAudioAsset>,
#[serde(default)]
click_sound: Option<CreationAudioAsset>,
#[serde(default)]
background_asset: Option<Match3DGeneratedBackgroundAsset>,
status: String,
#[serde(default)]
error: Option<String>,
}
#[derive(Clone, Debug)]
struct Match3DAssetUpload {
src: String,
object_key: String,
}
struct Match3DDownloadedModel {
bytes: Vec<u8>,
file_name: String,
content_type: String,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct CompileMatch3DDraftRequest {
#[serde(default)]
game_name: Option<String>,
#[serde(default)]
summary: Option<String>,
#[serde(default)]
tags: Option<Vec<String>>,
#[serde(default)]
cover_image_src: Option<String>,
#[serde(default)]
generate_click_sound: Option<bool>,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct GenerateMatch3DWorkTagsRequest {
game_name: String,
theme_text: String,
#[serde(default)]
summary: Option<String>,
}
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct GenerateMatch3DWorkTagsResponse {
tags: Vec<String>,
}
struct Match3DWorkAssetContext {
owner_user_id: String,
session_id: String,
profile: Match3DWorkProfileRecord,
config: Match3DConfigJson,
assets: Vec<Match3DGeneratedItemAsset>,
}
struct Match3DItemAssetAppendPlan {
requested_item_names: Vec<String>,
padded_item_names: Vec<String>,
}
struct Match3DItemAssetReplacePlan {
requested_item_names: Vec<String>,
padded_item_names: Vec<String>,
target_assets: Vec<Match3DGeneratedItemAsset>,
}
enum Match3DItemAssetsGenerationPlan {
Append(Match3DItemAssetAppendPlan),
Replace(Match3DItemAssetReplacePlan),
}
enum Match3DItemAssetsGenerationMode {
Append,
Replace,
}
impl Match3DItemAssetsGenerationPlan {
fn billed_item_count(&self) -> usize {
match self {
Self::Append(plan) => plan.requested_item_names.len(),
Self::Replace(plan) => plan.requested_item_names.len(),
}
}
fn billing_fingerprint_source(&self) -> String {
match self {
Self::Append(plan) => format!("append:{}", plan.requested_item_names.join("|")),
Self::Replace(plan) => format!("replace:{}", plan.requested_item_names.join("|")),
}
}
}
fn serialize_match3d_generated_item_assets(assets: &[Match3DGeneratedItemAsset]) -> Option<String> {
if assets.is_empty() {
return None;
}
let items = assets
.iter()
.cloned()
.map(Match3DGeneratedItemAssetJson::from)
.collect::<Vec<_>>();
serde_json::to_string(&items).ok()
}
fn parse_match3d_generated_item_assets(value: Option<&str>) -> Vec<Match3DGeneratedItemAssetJson> {
value
.map(str::trim)
.filter(|value| !value.is_empty())
.and_then(|value| serde_json::from_str::<Vec<Match3DGeneratedItemAssetJson>>(value).ok())
.unwrap_or_default()
}
impl From<Match3DGeneratedItemAsset> for Match3DGeneratedItemAssetJson {
fn from(asset: Match3DGeneratedItemAsset) -> Self {
Self {
item_id: asset.item_id,
item_name: asset.item_name,
item_size: asset.item_size,
image_src: asset.image_src,
image_object_key: asset.image_object_key,
image_views: asset.image_views,
model_src: asset.model_src,
model_object_key: asset.model_object_key,
model_file_name: asset.model_file_name,
task_uuid: asset.task_uuid,
subscription_key: asset.subscription_key,
sound_prompt: asset.sound_prompt,
background_music_title: asset.background_music_title,
background_music_style: asset.background_music_style,
background_music_prompt: asset.background_music_prompt,
background_music: asset.background_music,
click_sound: asset.click_sound,
background_asset: asset.background_asset,
status: asset.status,
error: asset.error,
}
}
}
impl From<Match3DGeneratedItemAssetJson> for Match3DGeneratedItemAsset {
fn from(asset: Match3DGeneratedItemAssetJson) -> Self {
Self {
item_id: asset.item_id,
item_name: asset.item_name,
item_size: asset
.item_size
.or_else(|| Some(MATCH3D_ITEM_SIZE_LARGE.to_string())),
image_src: asset.image_src,
image_object_key: asset.image_object_key,
image_views: asset.image_views,
model_src: asset.model_src,
model_object_key: asset.model_object_key,
model_file_name: asset.model_file_name,
task_uuid: asset.task_uuid,
subscription_key: asset.subscription_key,
sound_prompt: asset.sound_prompt,
background_music_title: asset.background_music_title,
background_music_style: asset.background_music_style,
background_music_prompt: asset.background_music_prompt,
background_music: asset.background_music,
click_sound: asset.click_sound,
background_asset: asset.background_asset,
status: asset.status,
error: asset.error,
}
}
}
impl From<shared_contracts::match3d_works::Match3DGeneratedItemAssetResponse>
for Match3DGeneratedItemAsset
{
fn from(asset: shared_contracts::match3d_works::Match3DGeneratedItemAssetResponse) -> Self {
Self {
item_id: asset.item_id,
item_name: asset.item_name,
item_size: asset.item_size,
image_src: asset.image_src,
image_object_key: asset.image_object_key,
image_views: asset
.image_views
.into_iter()
.map(map_match3d_image_view_from_work)
.collect(),
model_src: asset.model_src,
model_object_key: asset.model_object_key,
model_file_name: asset.model_file_name,
task_uuid: asset.task_uuid,
subscription_key: asset.subscription_key,
sound_prompt: asset.sound_prompt,
background_music_title: asset.background_music_title,
background_music_style: asset.background_music_style,
background_music_prompt: asset.background_music_prompt,
background_music: asset.background_music,
click_sound: asset.click_sound,
background_asset: asset
.background_asset
.map(|asset| Match3DGeneratedBackgroundAsset {
prompt: asset.prompt,
image_src: asset.image_src,
image_object_key: asset.image_object_key,
container_prompt: asset.container_prompt,
container_image_src: asset.container_image_src,
container_image_object_key: asset.container_image_object_key,
status: asset.status,
error: asset.error,
}),
status: asset.status,
error: asset.error,
}
}
}
mod handlers;
pub(crate) use self::handlers::*;
mod mappers;
use self::mappers::*;
mod tags;
use self::tags::*;
mod draft;
use self::draft::*;
mod works;
use self::works::*;
mod runtime;
use self::runtime::*;
mod item_assets;
use self::item_assets::*;
mod vector_engine_gemini;
use self::vector_engine_gemini::*;
fn ensure_non_empty(
request_context: &RequestContext,
provider: &str,
value: &str,
field_name: &str,
) -> Result<(), Response> {
if value.trim().is_empty() {
return Err(match3d_bad_request(
request_context,
provider,
format!("{field_name} is required").as_str(),
));
}
Ok(())
}
fn match3d_json<T>(
payload: Result<Json<T>, JsonRejection>,
request_context: &RequestContext,
provider: &str,
) -> Result<Json<T>, Response> {
payload.map_err(|error| {
match3d_error_response(
request_context,
provider,
AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({
"provider": provider,
"message": error.body_text(),
})),
)
})
}
fn match3d_bad_request(
request_context: &RequestContext,
provider: &str,
message: &str,
) -> Response {
match3d_error_response(
request_context,
provider,
AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({
"provider": provider,
"message": message,
})),
)
}
fn map_match3d_client_error(error: SpacetimeClientError) -> AppError {
let status = match &error {
SpacetimeClientError::Runtime(_) => StatusCode::BAD_REQUEST,
SpacetimeClientError::Procedure(message)
if message.contains("不存在")
|| message.contains("not found")
|| message.contains("does not exist") =>
{
StatusCode::NOT_FOUND
}
SpacetimeClientError::Procedure(message)
if message.contains("发布需要")
|| message.contains("不能为空")
|| message.contains("必须") =>
{
StatusCode::BAD_REQUEST
}
_ => StatusCode::BAD_GATEWAY,
};
AppError::from_status(status).with_details(json!({
"provider": "spacetimedb",
"message": error.to_string(),
}))
}
fn match3d_error_response(
request_context: &RequestContext,
provider: &str,
error: AppError,
) -> Response {
let mut response = error.into_response_with_context(Some(request_context));
response.headers_mut().insert(
HeaderName::from_static("x-genarrative-provider"),
header::HeaderValue::from_str(provider)
.unwrap_or_else(|_| header::HeaderValue::from_static("match3d")),
);
response
}
fn match3d_sse_json_event(event_name: &str, payload: Value) -> Result<Event, AppError> {
Event::default()
.event(event_name)
.json_data(payload)
.map_err(|error| {
AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_details(json!({
"provider": "sse",
"message": format!("SSE payload 序列化失败:{error}"),
}))
})
}
fn match3d_sse_json_event_or_error(event_name: &str, payload: Value) -> Event {
match match3d_sse_json_event(event_name, payload) {
Ok(event) => event,
Err(error) => Event::default().event("error").data(format!("{error:?}")),
}
}
fn current_utc_micros() -> i64 {
let duration = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default();
(duration.as_secs() as i64) * 1_000_000 + i64::from(duration.subsec_micros())
}
fn current_utc_ms() -> i64 {
current_utc_micros().saturating_div(1000)
}
#[cfg(test)]
mod tests;