发行包上限提升到 200 MiB 并支持 AGC 分片续传上传
- 发行包上限 100→200 MiB、展开总量 250→500 MiB，整包路由请求体上限继续从包上限派生；发行静态资源进程内缓存预算提到 256 MiB - 反代放行量同步放宽到 210 MiB：Nginx 三份模板的 client_max_body_size、Pingora 网关默认值与 env 样例、路由对照矩阵 - platform-oss 新增内部对象追加写 append_internal_object(_with_retry)，以 OSS 返回的 next-append-position 作为权威已收字节 - api-server 新增 upload-state / chunk / complete / reset 四条分片路由，抽出共享收口 confirm_validated_package；偏移不符返回 409 与权威偏移，校验失败删除半包并落 upload_failed - AGC 新增原生上传器 game_package_upload.rs（内容寻址暂存、分片续传、受控重试、进度事件）与 prepare / upload 两条命令，退役整包回传命令 - 渲染进程改为 prepare → 创建游戏 → 创建版本 → 原生分片上传 → 送审，LocalProjectExportPackagePayload 整包类型退役 - 同时修正 live 用例无法指向本地栈的两处基础设施问题：平台基址按传入 URL 选择，桥接层把 jsdom realm 的 Headers / Blob / FormData 降级成 Node 原生值 - 测试：platform-oss 74、api-server game_distribution 23、AGC 原生 4、发布相关前端 20；live 用例补真实栈「中断 → 续传 → 确认」断言（分片偏移序列 [0, 8388608]） - 文档：玩法创作主规范的上传合同、运维与 Pingora 文档、决策记录、发行里程碑口径，以及新增的续传里程碑与实施计划
This commit is contained in:
kdletters
File diff suppressed because it is too large
Load Diff
@@ -6,8 +6,13 @@ use std::{
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub const MAX_PACKAGE_BYTES: u64 = 100 * 1024 * 1024;
|
||||
pub const MAX_EXPANDED_BYTES: u64 = 250 * 1024 * 1024;
|
||||
/// 发行包体积上限。反代放行量与路由请求体上限都从它派生：Nginx
|
||||
/// `client_max_body_size`、Pingora `MAX_API_BODY_BYTES` 必须同步放宽，否则合法包会在
|
||||
/// 到达 `api-server` 之前被拒。
|
||||
pub const MAX_PACKAGE_BYTES: u64 = 200 * 1024 * 1024;
|
||||
/// 展开总量上限保持压缩包上限的 2.5 倍余量：包体本身基本不可再压时展开量约等于包体，
|
||||
/// 纯文本 / JSON 资源占比高的包仍要有足够空间。
|
||||
pub const MAX_EXPANDED_BYTES: u64 = 500 * 1024 * 1024;
|
||||
pub const MAX_FILE_BYTES: u64 = 64 * 1024 * 1024;
|
||||
pub const MAX_FILE_COUNT: usize = 10_000;
|
||||
pub const MAX_COMPRESSION_RATIO: u64 = 100;
|
||||
@@ -187,6 +192,23 @@ mod tests {
|
||||
output.into_inner()
|
||||
}
|
||||
|
||||
/// 存储型条目的压缩包；用于构造体积可控且不参与 deflate 的大包。
|
||||
fn stored_archive(files: &[(&str, &[u8])]) -> Vec<u8> {
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
let mut writer = ZipWriter::new(&mut output);
|
||||
for (path, content) in files {
|
||||
writer
|
||||
.start_file(
|
||||
*path,
|
||||
SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored),
|
||||
)
|
||||
.expect("zip entry");
|
||||
writer.write_all(content).expect("zip content");
|
||||
}
|
||||
writer.finish().expect("finish zip");
|
||||
output.into_inner()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_root_entry_and_returns_file_manifest() {
|
||||
let bytes = archive(&[("index.html", b"<html></html>"), ("assets/a.txt", b"a")]);
|
||||
@@ -222,4 +244,27 @@ mod tests {
|
||||
Err(ReleasePackageError::InvalidPath)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_expansion_headroom_over_package_limit() {
|
||||
// 口径约束：发行包上限调整时，展开总量至少要留出两倍余量，
|
||||
// 否则高文本占比的合法包会在展开量检查处被误拒。
|
||||
assert!(MAX_EXPANDED_BYTES >= MAX_PACKAGE_BYTES.saturating_mul(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_package_above_the_previous_hundred_mib_limit() {
|
||||
// 上限从 100 MiB 提到 200 MiB 的回归防护：两个 50 MiB 存储型条目组成 100 MiB
|
||||
// 出头的包，旧上限会在这里判 PackageTooLarge，新上限必须放行并给出完整清单。
|
||||
let chunk = vec![0_u8; 50 * 1024 * 1024];
|
||||
let bytes = stored_archive(&[
|
||||
("index.html", b"<html></html>"),
|
||||
("assets/a.bin", chunk.as_slice()),
|
||||
("assets/b.bin", chunk.as_slice()),
|
||||
]);
|
||||
assert!(bytes.len() as u64 > 100 * 1024 * 1024);
|
||||
let manifest = validate_release_zip(&bytes).expect("package above 100 MiB");
|
||||
assert_eq!(manifest.package_bytes, bytes.len() as u64);
|
||||
assert_eq!(manifest.files.len(), 3);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,7 +37,9 @@ const DEFAULT_WEB_ROOT: &str = "/srv/genarrative/web";
|
||||
const DEFAULT_ACME_ROOT: &str = "/var/www/html";
|
||||
const DEFAULT_MAINTENANCE_FILE: &str = "/var/lib/genarrative/maintenance/enabled";
|
||||
const DEFAULT_MAINTENANCE_PAGE_FILE: &str = "/var/lib/genarrative/maintenance/page.html";
|
||||
const DEFAULT_MAX_API_BODY_BYTES: u64 = 64 * 1024 * 1024;
|
||||
// 通用 /api 路由的放行上限；必须覆盖游戏发行包 PUT（路由 `DefaultBodyLimit` 200 MiB + 1 KiB），
|
||||
// 具体接口的真实上限仍由 api-server 逐路由校验。
|
||||
const DEFAULT_MAX_API_BODY_BYTES: u64 = 210 * 1024 * 1024;
|
||||
const DEFAULT_GZIP_LEVEL: u32 = 5;
|
||||
const DEFAULT_GZIP_MIN_LENGTH_BYTES: u64 = 1024;
|
||||
const DEFAULT_UPSTREAM_CONNECT_TIMEOUT_MS: u64 = 3_000;
|
||||
|
||||
@@ -132,6 +132,29 @@ pub struct OssInternalPutObjectRequest {
|
||||
pub body: Vec<u8>,
|
||||
}
|
||||
|
||||
/// 内部对象的追加写请求（OSS AppendObject）。
|
||||
///
|
||||
/// `position = 0` 表示追加到当前末尾；`position > 0` 必须等于对象当前长度，
|
||||
/// 否则上游直接失败 —— 分片续传依赖这条语义保证重放不会重复写入。
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct OssAppendInternalObjectRequest {
|
||||
pub object_key: String,
|
||||
pub content_type: Option<String>,
|
||||
pub position: u64,
|
||||
pub body: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct OssAppendInternalObjectResponse {
|
||||
pub provider: &'static str,
|
||||
pub bucket: String,
|
||||
pub endpoint: String,
|
||||
pub object_key: String,
|
||||
pub appended_bytes: u64,
|
||||
/// 下一次可写位置，由 OSS 返回，是「已收字节」的权威值。
|
||||
pub next_position: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct OssPutObjectRequest {
|
||||
pub prefix: LegacyAssetPrefix,
|
||||
@@ -1018,6 +1041,105 @@ impl OssClient {
|
||||
.await
|
||||
}
|
||||
|
||||
/// 内部对象追加写，返回 OSS 给出的下一次可写位置（已收字节的权威值）。
|
||||
///
|
||||
/// 与整对象 PUT 的区别是「可续写」：`position = 0` 追加到当前末尾，
|
||||
/// `position > 0` 必须与对象当前长度一致（不一致时 OSS 判失败，不会重复写入）。
|
||||
/// 因此续传只需要回读对象长度，再从未写位置继续。
|
||||
pub async fn append_internal_object(
|
||||
&self,
|
||||
client: &reqwest::Client,
|
||||
request: OssAppendInternalObjectRequest,
|
||||
) -> Result<OssAppendInternalObjectResponse, OssError> {
|
||||
let object_key = normalize_internal_object_key(&request.object_key)?;
|
||||
if request.body.is_empty() {
|
||||
return Err(OssError::InvalidRequest(
|
||||
"服务端内部对象追加内容不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
let content_type = normalize_optional_value(request.content_type);
|
||||
let mut target_url =
|
||||
build_object_url(&self.config.bucket, &self.config.endpoint, &object_key).map_err(
|
||||
|error| {
|
||||
request_error(
|
||||
OssRequestOperation::Put,
|
||||
&format!("构造 OSS 对象 URL 失败：{error}"),
|
||||
)
|
||||
},
|
||||
)?;
|
||||
target_url
|
||||
.query_pairs_mut()
|
||||
.append_pair("append", "")
|
||||
.append_pair("position", &request.position.to_string());
|
||||
let appended_bytes = u64::try_from(request.body.len())
|
||||
.map_err(|_| OssError::InvalidRequest("追加内容大小超出可支持范围".to_string()))?;
|
||||
let headers = BTreeMap::new();
|
||||
let builder = signed_request_builder(
|
||||
client,
|
||||
&self.config,
|
||||
Method::POST,
|
||||
Some(&object_key),
|
||||
target_url,
|
||||
content_type.as_deref(),
|
||||
&headers,
|
||||
)?
|
||||
.header(reqwest::header::CONTENT_LENGTH, appended_bytes)
|
||||
.body(request.body);
|
||||
let response = builder
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| request_error_from_reqwest(OssRequestOperation::Put, error))?;
|
||||
if !response.status().is_success() {
|
||||
return Err(request_status_error(
|
||||
OssRequestOperation::Put,
|
||||
response.status().as_u16(),
|
||||
format!("OSS AppendObject 失败，状态码：{}", response.status()),
|
||||
));
|
||||
}
|
||||
let next_position = response
|
||||
.headers()
|
||||
.get("x-oss-next-append-position")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.trim().parse::<u64>().ok())
|
||||
.ok_or_else(|| {
|
||||
OssError::InvalidRequest("OSS AppendObject 未返回 next-append-position".to_string())
|
||||
})?;
|
||||
Ok(OssAppendInternalObjectResponse {
|
||||
provider: OSS_PROVIDER,
|
||||
bucket: self.config.bucket.clone(),
|
||||
endpoint: self.config.endpoint.clone(),
|
||||
object_key,
|
||||
appended_bytes,
|
||||
next_position,
|
||||
})
|
||||
}
|
||||
|
||||
/// 内部对象追加写的受控重试，判定与退避口径和 `put_internal_object_with_retry` 一致。
|
||||
pub async fn append_internal_object_with_retry(
|
||||
&self,
|
||||
client: &reqwest::Client,
|
||||
request: OssAppendInternalObjectRequest,
|
||||
max_attempts: usize,
|
||||
retry_delays_ms: &[u64],
|
||||
) -> Result<OssAppendInternalObjectResponse, OssError> {
|
||||
if max_attempts == 0 {
|
||||
return Err(OssError::InvalidConfig(
|
||||
"内部对象追加重试次数至少为 1".to_string(),
|
||||
));
|
||||
}
|
||||
if retry_delays_ms.len() < max_attempts.saturating_sub(1) {
|
||||
return Err(OssError::InvalidConfig(
|
||||
"内部对象追加重试缺少退避配置".to_string(),
|
||||
));
|
||||
}
|
||||
let log_key = request.object_key.clone();
|
||||
run_internal_put_with_retry(max_attempts, retry_delays_ms, &log_key, move || {
|
||||
let request = request.clone();
|
||||
async move { self.append_internal_object(client, request).await }
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_internal_object_bytes(
|
||||
&self,
|
||||
client: &reqwest::Client,
|
||||
@@ -1709,7 +1831,7 @@ where
|
||||
max_attempts,
|
||||
retry_delay_ms = delay_ms,
|
||||
error = %error,
|
||||
"OSS 内部对象 PUT 失败，按退避重试"
|
||||
"OSS 内部对象写入失败，按退避重试"
|
||||
);
|
||||
sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
attempt += 1;
|
||||
|
||||
Reference in New Issue
Block a user