AGC 项目定时快照上传(目标 OSS agc-dev) #400

Merged
kdletters merged 6 commits from codex/agc-project-snapshot-upload into master 2026-09-17 18:34:51 +08:00
25 changed files with 3543 additions and 6 deletions
+9
View File
@@ -158,6 +158,15 @@ ALIYUN_OSS_POST_EXPIRE_SECONDS="600"
ALIYUN_OSS_POST_MAX_SIZE_BYTES="20971520"
ALIYUN_OSS_SUCCESS_ACTION_STATUS="200"
# AGC 项目定时快照上传目标。对象只落在服务端私有前缀
# `agc/project-snapshots/v1/{user}/{project}/` 下,客户端直传票据不覆盖该前缀。
# bucket 与凭据可以与资源 bucket 分离;凭据未设置时回退使用 ALIYUN_OSS_ACCESS_KEY_*
# 但 bucket / endpoint 默认指向 AGC 发行 bucket,需要该凭据具备目标 bucket 的 PutObject 权限。
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET="agc-dev"
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT="oss-rg-china-mainland.aliyuncs.com"
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID=""
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET=""
# BgFilter 受限资源 worker。父 api-server / external-generation-worker 与唯一的
# `GENARRATIVE_PROCESS_ROLE=bgfilter-worker` 进程必须使用同一个内部 Token。
# `npm run dev` 与 `npm run dev:api-server` 都会自动带起并验活唯一 worker,不要再开第二个终端重复启动。
@@ -121,6 +121,10 @@ const allowedUncalledTauriCommands = [
'open_game_creator_launcher_window',
'open_game_creator_workspace_window',
'read_direct_project_conversation',
// 项目定时快照上传只在 Rust 侧触发(周期定时器 / 工作区窗口关闭)与排障调用;
// 按产品口径不做客户端可见界面,因此同 `open_game_creator_*_window` 一样按 native-only 登记。
'read_local_project_snapshot_state',
'sync_local_project_snapshot',
'reset_design_agent_session',
'stop_local_game_preview_if_matches',
'start_game_creator_external_mcp',
@@ -106,6 +106,7 @@ mod preview;
mod process_session;
mod process_session_bridge;
mod project;
mod project_snapshot;
mod provider_handoff;
mod provider_retry;
mod repository_context;
@@ -148,6 +149,7 @@ use plugin_host::{
use preview::*;
use process_session::*;
use project::*;
use project_snapshot::*;
use repository_context::*;
use resource_inspect::*;
use resource_preview_scheduler::*;
@@ -2125,6 +2127,9 @@ where
fn handle_game_creator_gui_run_event(event: &tauri::RunEvent) {
if matches!(event, tauri::RunEvent::Exit) {
// 窗口关闭时已经按项目触发过一次快照同步;退出路径只负责在有界预算内
// 等在途同步收尾,不重复发起(此刻窗口已销毁,重新枚举项目只会是空集)。
wait_for_project_snapshot_syncs_on_exit();
// 退出时统一收尾本地预览:进程内监听线程随进程消失,但 `.agent/manifest.json`
// 里的 preview 记录会留在 running 上,下次进项目就照着它渲染打不开的运行界面。
if let Err(error) =
@@ -2390,6 +2395,7 @@ fn main() {
.plugin(tauri_plugin_clipboard_manager::init())
.plugin(tauri_plugin_updater::Builder::new().build())
.plugin(context_menu::init())
.on_window_event(|window, event| handle_project_snapshot_window_event(window, event))
.manage(game_creator_preview_registry())
.manage(ProjectResourcePreviewReadManager::default())
.manage(PluginHost::default())
@@ -2418,6 +2424,7 @@ fn main() {
setup_log.fail("startup.appdata.resolve.failed details=config-dir-uninitialized");
error
})?;
spawn_project_snapshot_scheduler(app.handle().clone());
if let Err(error) = builtin_plugins::initialize(&config_dir) {
app_log!("startup.builtin-plugins.initialize.failed: {error}");
}
@@ -2680,6 +2687,8 @@ fn main() {
report_client_error,
get_pending_error_reports,
ack_error_reports,
sync_local_project_snapshot,
read_local_project_snapshot_state,
])
.build(tauri_context);
let app = match app {
@@ -0,0 +1,151 @@
use super::*;
use std::collections::BTreeSet;
/// 需要上传的一个文件。摘要与字节数来自本次差异对比,上传阶段不再重新计算。
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct ProjectSnapshotUploadCandidate {
pub(crate) relative_path: String,
pub(crate) size_bytes: u64,
pub(crate) modified_ms: u64,
pub(crate) checksum: String,
pub(crate) absolute_path: PathBuf,
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub(crate) struct ProjectSnapshotDiff {
pub(crate) uploads: Vec<ProjectSnapshotUploadCandidate>,
/// 本地已删除、只在远端清单中需要消失的路径。
pub(crate) deleted: Vec<String>,
/// 内容与上次同步一致、只有元数据变化的路径;只需刷新本地索引。
pub(crate) metadata_only: Vec<String>,
pub(crate) skipped: Vec<ProjectSnapshotSkippedPath>,
/// 本轮因为累计上限没有上传、留给下一次同步的路径。
pub(crate) deferred: Vec<ProjectSnapshotSkippedPath>,
/// 同步期间被改写、本轮不参与上传与索引推进的路径。
pub(crate) pending: Vec<ProjectSnapshotSkippedPath>,
pub(crate) upload_bytes: u64,
/// 扫描后的完整清单(尚未扣除上传失败与延后项),成功同步后就是新的索引。
pub(crate) current: BTreeMap<String, ProjectSnapshotIndexedFile>,
}
impl ProjectSnapshotDiff {
pub(crate) fn has_changes(&self) -> bool {
!self.uploads.is_empty() || !self.deleted.is_empty() || !self.metadata_only.is_empty()
}
}
/// 增量差异对比:先用 `(字节数, 修改时间)` 判定是否需要读盘,只有元数据变化时
/// 才重新计算摘要,再用摘要判断内容是否真的变了。
pub(crate) fn compute_project_snapshot_diff(
scan: &ProjectSnapshotScanResult,
previous: &BTreeMap<String, ProjectSnapshotIndexedFile>,
max_sync_bytes: u64,
) -> Result<ProjectSnapshotDiff, String> {
let mut diff = ProjectSnapshotDiff {
skipped: scan.skipped.clone(),
..ProjectSnapshotDiff::default()
};
let mut metadata_only = Vec::new();
for file in &scan.files {
let previous_entry = previous.get(&file.relative_path);
if let Some(entry) = previous_entry {
if entry.size_bytes == file.size_bytes
&& entry.modified_ms == file.modified_ms
&& !entry.checksum.is_empty()
{
diff.current
.insert(file.relative_path.clone(), entry.clone());
continue;
}
}
let bytes = read_project_snapshot_file_bytes(&file.absolute_path)?;
// 同步不持项目写锁:读完立刻复核一次,文件在读取期间被改写就留给下一轮,
// 既不把这份内容写进索引,也不上传它。
if !project_snapshot_file_matches(&file.absolute_path, file.size_bytes, file.modified_ms) {
diff.pending.push(ProjectSnapshotSkippedPath {
relative_path: file.relative_path.clone(),
reason: "同步期间文件发生变化,留到下一次".to_string(),
});
// 上一轮已同步过这条路径时沿用旧记录:文件仍然存在,只是这一轮读不到
// 一致版本,所以既不能算删除(会被远端 GC 掉),也不能记成本轮已同步。
if let Some(entry) = previous_entry {
diff.current
.insert(file.relative_path.clone(), entry.clone());
}
continue;
}
let checksum = project_snapshot_checksum(&bytes);
match previous_entry {
Some(entry) if entry.checksum == checksum => {
metadata_only.push(file.relative_path.clone())
}
_ => diff.uploads.push(ProjectSnapshotUploadCandidate {
relative_path: file.relative_path.clone(),
size_bytes: file.size_bytes,
modified_ms: file.modified_ms,
checksum: checksum.clone(),
absolute_path: file.absolute_path.clone(),
}),
}
diff.current.insert(
file.relative_path.clone(),
ProjectSnapshotIndexedFile {
size_bytes: file.size_bytes,
modified_ms: file.modified_ms,
checksum,
},
);
}
diff.deleted = previous
.keys()
.filter(|relative_path| !diff.current.contains_key(*relative_path))
.cloned()
.collect();
diff.deleted.sort();
diff.metadata_only = metadata_only;
apply_project_snapshot_sync_budget(&mut diff, max_sync_bytes);
Ok(diff)
}
/// 按路径顺序累计上传体积;超出上限的文件进入延后清单,不静默截断也不上传残缺内容。
fn apply_project_snapshot_sync_budget(diff: &mut ProjectSnapshotDiff, max_sync_bytes: u64) {
let mut accepted = Vec::new();
let mut deferred = Vec::new();
let mut bytes = 0_u64;
for candidate in diff.uploads.drain(..) {
if bytes.saturating_add(candidate.size_bytes) > max_sync_bytes {
deferred.push(ProjectSnapshotSkippedPath {
relative_path: candidate.relative_path,
reason: format!("单次同步超过 {max_sync_bytes} 字节上限,留到下一次"),
});
continue;
}
bytes = bytes.saturating_add(candidate.size_bytes);
accepted.push(candidate);
}
diff.upload_bytes = bytes;
diff.uploads = accepted;
diff.deferred = deferred;
}
/// 成功同步后的索引清单:延后项与上传失败项都要剔除,否则下次同步会把它们
/// 当成"已经同步过"而漏传。
pub(crate) fn build_project_snapshot_synced_files(
diff: &ProjectSnapshotDiff,
uploaded_paths: &BTreeSet<String>,
) -> BTreeMap<String, ProjectSnapshotIndexedFile> {
let mut retained = diff.current.clone();
for candidate in &diff.uploads {
if !uploaded_paths.contains(&candidate.relative_path) {
retained.remove(&candidate.relative_path);
}
}
for deferred in &diff.deferred {
retained.remove(&deferred.relative_path);
}
retained
}
@@ -0,0 +1,131 @@
use super::*;
/// 本地增量索引里的一条文件记录。
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ProjectSnapshotIndexedFile {
pub(crate) size_bytes: u64,
pub(crate) modified_ms: u64,
pub(crate) checksum: String,
}
/// 上次成功同步的快照。索引只在本机 AppData 中,不进入用户项目目录。
///
/// `user_id` 是远端前缀的一部分:换号后旧索引不再代表同一个远端命名空间,
/// 因此读取时按用户身份判等,不一致就当作冷启动重新全量对比。
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ProjectSnapshotIndex {
pub(crate) schema_version: u32,
pub(crate) project_id: String,
pub(crate) user_id: String,
pub(crate) sync_revision: u64,
pub(crate) synced_at_ms: u64,
#[serde(default)]
pub(crate) files: BTreeMap<String, ProjectSnapshotIndexedFile>,
}
pub(crate) fn empty_project_snapshot_index(
project_id: &str,
user_id: &str,
) -> ProjectSnapshotIndex {
ProjectSnapshotIndex {
schema_version: PROJECT_SNAPSHOT_INDEX_SCHEMA_VERSION,
project_id: project_id.to_string(),
user_id: user_id.to_string(),
sync_revision: 0,
synced_at_ms: 0,
files: BTreeMap::new(),
}
}
/// 项目 ID 同时用作 AppData 目录名与远端键段:这里额外拒绝 `.` 与 `..`,避免
/// 把索引写到目录之外。
pub(crate) fn validate_project_snapshot_project_id(project_id: &str) -> Result<String, String> {
shared_contracts::agc_project_snapshots::validate_agc_project_snapshot_project_id(project_id)?;
if project_id == "." || project_id == ".." {
return Err("项目 ID 不能是相对目录".to_string());
}
Ok(project_id.to_string())
}
pub(crate) fn project_snapshot_index_directory(project_id: &str) -> Result<PathBuf, String> {
let config_dir = game_creator_runtime_config_dir()
.ok_or_else(|| "客户端 AppData 配置目录未初始化".to_string())?;
project_snapshot_index_directory_at(&config_dir, project_id)
}
pub(crate) fn project_snapshot_index_directory_at(
config_dir: &Path,
project_id: &str,
) -> Result<PathBuf, String> {
let project_id = validate_project_snapshot_project_id(project_id)?;
Ok(config_dir.join(PROJECT_SNAPSHOT_DIRECTORY).join(project_id))
}
pub(crate) fn project_snapshot_index_path(project_id: &str) -> Result<PathBuf, String> {
Ok(project_snapshot_index_path_at(
&project_snapshot_index_directory(project_id)?,
))
}
pub(crate) fn project_snapshot_index_path_at(directory: &Path) -> PathBuf {
directory.join(PROJECT_SNAPSHOT_INDEX_FILE_NAME)
}
/// 读取本地索引。索引不存在、版本不符或内容损坏时返回空索引:这种情况下
/// 全量对比会重新上传所有文件,不会漏传,也不会因为坏索引中断同步。
pub(crate) fn read_project_snapshot_index(
project_id: &str,
) -> Result<ProjectSnapshotIndex, String> {
read_project_snapshot_index_at(&project_snapshot_index_directory(project_id)?, project_id)
}
pub(crate) fn read_project_snapshot_index_at(
directory: &Path,
project_id: &str,
) -> Result<ProjectSnapshotIndex, String> {
let path = project_snapshot_index_path_at(directory);
if !path.exists() {
return Ok(empty_project_snapshot_index(project_id, ""));
}
let content = match read_game_creator_private_file_to_string(
&path,
"项目快照索引",
PROJECT_SNAPSHOT_INDEX_MAX_BYTES,
) {
Ok(content) => content,
Err(error) => {
app_log!("project_snapshot.index.read.failed: {error}");
return Ok(empty_project_snapshot_index(project_id, ""));
}
};
match serde_json::from_str::<ProjectSnapshotIndex>(&content) {
Ok(index)
if index.schema_version == PROJECT_SNAPSHOT_INDEX_SCHEMA_VERSION
&& index.project_id == project_id =>
{
Ok(index)
}
Ok(_) => Ok(empty_project_snapshot_index(project_id, "")),
Err(error) => {
app_log!("project_snapshot.index.parse.failed: {error}");
Ok(empty_project_snapshot_index(project_id, ""))
}
}
}
pub(crate) fn write_project_snapshot_index(index: &ProjectSnapshotIndex) -> Result<(), String> {
write_project_snapshot_index_at(&project_snapshot_index_directory(&index.project_id)?, index)
}
pub(crate) fn write_project_snapshot_index_at(
directory: &Path,
index: &ProjectSnapshotIndex,
) -> Result<(), String> {
let path = project_snapshot_index_path_at(directory);
let mut encoded = serde_json::to_vec_pretty(index)
.map_err(|error| format!("序列化项目快照索引失败:{error}"))?;
encoded.push(b'\n');
write_game_creator_private_file(&path, &encoded, "项目快照索引")
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,157 @@
use super::*;
/// 扫描阶段的候选文件:只看元数据,不读内容,差异对比阶段才决定是否读盘。
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct ProjectSnapshotScannedFile {
pub(crate) relative_path: String,
pub(crate) size_bytes: u64,
pub(crate) modified_ms: u64,
pub(crate) absolute_path: PathBuf,
}
/// 被排除或不参与本次同步的路径。原因只用于本机日志与同步报告,不进入远端。
#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ProjectSnapshotSkippedPath {
pub(crate) relative_path: String,
pub(crate) reason: String,
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub(crate) struct ProjectSnapshotScanResult {
pub(crate) files: Vec<ProjectSnapshotScannedFile>,
pub(crate) skipped: Vec<ProjectSnapshotSkippedPath>,
}
/// 扫描项目目录,复用 checkpoint / 项目索引同一份排除口径:
/// `.agent`、版本控制目录、依赖与构建产物目录、凭据目录、符号链接与重解析点
/// 都不参与同步,超出单文件上限的文件进入跳过清单而不是静默丢弃。
pub(crate) fn scan_project_snapshot_files(
root: &Path,
max_file_bytes: u64,
) -> Result<ProjectSnapshotScanResult, String> {
let mut result = ProjectSnapshotScanResult::default();
if !root.exists() {
return Ok(result);
}
let root_metadata =
fs::symlink_metadata(root).map_err(|error| format!("读取项目目录元数据失败:{error}"))?;
if root_metadata.file_type().is_symlink() || !root_metadata.is_dir() {
return Err("项目快照只支持普通项目目录".to_string());
}
let mut directories = vec![root.to_path_buf()];
while let Some(directory) = directories.pop() {
let entries = fs::read_dir(&directory)
.map_err(|error| format!("读取项目目录失败:{}: {error}", directory.display()))?;
for entry in entries {
let entry = entry
.map_err(|error| format!("读取项目文件失败:{}: {error}", directory.display()))?;
let path = entry.path();
// 路径不在项目根内(含符号链接跳转)时直接跳过,不猜测归属。
let Ok(relative_path) = relative_project_path(root, &path) else {
continue;
};
let metadata = match fs::symlink_metadata(&path) {
Ok(metadata) => metadata,
Err(error) => {
result.skipped.push(ProjectSnapshotSkippedPath {
relative_path,
reason: format!("读取元数据失败:{error}"),
});
continue;
}
};
if should_skip_project_snapshot_path(&relative_path) {
continue;
}
if metadata.file_type().is_symlink() || windows_metadata_is_reparse_point(&metadata) {
result.skipped.push(ProjectSnapshotSkippedPath {
relative_path,
reason: "符号链接或重解析点不参与项目快照".to_string(),
});
continue;
}
if metadata.is_dir() {
directories.push(path);
continue;
}
if !metadata.is_file() {
result.skipped.push(ProjectSnapshotSkippedPath {
relative_path,
reason: "不是普通文件".to_string(),
});
continue;
}
if metadata.len() > max_file_bytes {
result.skipped.push(ProjectSnapshotSkippedPath {
relative_path,
reason: format!("单个文件超过 {max_file_bytes} 字节上限"),
});
continue;
}
result.files.push(ProjectSnapshotScannedFile {
relative_path,
size_bytes: metadata.len(),
modified_ms: project_snapshot_modified_ms(&metadata),
absolute_path: path,
});
}
}
result
.files
.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
result
.skipped
.sort_by(|left, right| left.relative_path.cmp(&right.relative_path));
Ok(result)
}
/// 修改时间缺失或早于 Unix 纪元时返回 0:这样的文件每轮都会重算摘要,不会
/// 被误判成"未改动"。
pub(crate) fn project_snapshot_modified_ms(metadata: &fs::Metadata) -> u64 {
metadata
.modified()
.ok()
.and_then(|time| time.duration_since(UNIX_EPOCH).ok())
.map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX))
.unwrap_or(0)
}
/// 安全打开并读取项目文件内容。拒绝符号链接、重解析点与硬链接,读取后不再
/// 按路径名二次访问。
pub(crate) fn read_project_snapshot_file_bytes(path: &Path) -> Result<Vec<u8>, String> {
let (mut file, metadata) = open_project_snapshot_regular_file(path, "项目快照文件")?;
let mut bytes = Vec::with_capacity(usize::try_from(metadata.len()).unwrap_or_default());
file.read_to_end(&mut bytes)
.map_err(|error| format!("读取项目快照文件失败:{}: {error}", path.display()))?;
Ok(bytes)
}
/// 与项目索引同口径的校验和,便于同一份内容在不同入口之间比较。
pub(crate) fn project_snapshot_checksum(bytes: &[u8]) -> String {
format!("fnv1a64:{:016x}", fnv1a64(bytes))
}
/// 复核文件是否仍是扫描时刻的那一份内容。
///
/// 同步不持有项目写锁:Agent 或编辑器可能在扫描之后、读取或上传之前改写文件。
/// 读取前后都按 `(字节数, 修改时间)` 比对,任一处不一致就判定为"同步期间发生变化",
/// 该文件不计入索引、留给下一次同步,而不是上传一份自相矛盾的快照。
pub(crate) fn project_snapshot_file_matches(
path: &Path,
size_bytes: u64,
modified_ms: u64,
) -> bool {
let Ok(metadata) = fs::symlink_metadata(path) else {
return false;
};
if metadata.file_type().is_symlink() || windows_metadata_is_reparse_point(&metadata) {
return false;
}
if !metadata.is_file() {
return false;
}
metadata.len() == size_bytes && project_snapshot_modified_ms(&metadata) == modified_ms
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,337 @@
use super::*;
use std::collections::BTreeSet;
/// 上传失败的分类。鉴权与权限类失败不重试,其余交给下一次周期或关闭触发。
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ProjectSnapshotUploadErrorKind {
Authentication,
Permission,
/// 同步期间文件被改写:本轮跳过该文件,下一次重算即可,不是终态失败。
FileChanged,
/// 服务端按配额或频率拒绝:等下一次触发再试。
Throttled,
Rejected,
Upstream,
Transport,
}
impl ProjectSnapshotUploadErrorKind {
pub(crate) fn code(self) -> &'static str {
match self {
Self::Authentication => "authentication-required",
Self::Permission => "permission-denied",
Self::FileChanged => "file-changed",
Self::Throttled => "throttled",
Self::Rejected => "request-rejected",
Self::Upstream => "upstream-failed",
Self::Transport => "transport-failed",
}
}
/// 鉴权、权限与请求被拒都是确定性失败;重复提交同样的内容不会变好。
pub(crate) fn is_retryable(self) -> bool {
matches!(
self,
Self::Upstream | Self::Transport | Self::FileChanged | Self::Throttled
)
}
}
#[derive(Clone, Debug)]
pub(crate) struct ProjectSnapshotUploadError {
kind: ProjectSnapshotUploadErrorKind,
detail: String,
}
impl ProjectSnapshotUploadError {
fn new(kind: ProjectSnapshotUploadErrorKind, detail: impl Into<String>) -> Self {
Self {
kind,
detail: detail.into(),
}
}
pub(crate) fn kind(&self) -> ProjectSnapshotUploadErrorKind {
self.kind
}
pub(crate) fn code(&self) -> &'static str {
self.kind.code()
}
pub(crate) fn message(&self) -> String {
format!("{}: {}", self.kind.code(), self.detail)
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct ProjectSnapshotUploadFailure {
pub(crate) relative_path: String,
pub(crate) code: String,
pub(crate) detail: String,
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub(crate) struct ProjectSnapshotUploadReport {
pub(crate) uploaded_paths: BTreeSet<String>,
pub(crate) uploaded_bytes: u64,
pub(crate) remote_skipped_files: usize,
pub(crate) failures: Vec<ProjectSnapshotUploadFailure>,
}
impl ProjectSnapshotUploadReport {
fn from_hard_failure(paths: &[String], error: &ProjectSnapshotUploadError) -> Self {
Self {
uploaded_paths: BTreeSet::new(),
uploaded_bytes: 0,
remote_skipped_files: 0,
failures: paths
.iter()
.map(|relative_path| ProjectSnapshotUploadFailure {
relative_path: relative_path.clone(),
code: error.code().to_string(),
detail: error.detail.clone(),
})
.collect(),
}
}
}
/// 逐文件上传本次差异集合。单个文件失败只影响该文件;鉴权或权限失败会中止
/// 后续请求,因为同样的凭据不会在这一次同步里变好。
pub(crate) async fn upload_project_snapshot_diff(
session: &PlatformSessionSnapshot,
project_id: &str,
diff: &ProjectSnapshotDiff,
) -> ProjectSnapshotUploadReport {
let mut report = ProjectSnapshotUploadReport::default();
if diff.uploads.is_empty() {
return report;
}
let client = match crate::http_client::agc_main_site_client_builder()
.timeout(Duration::from_secs(
PROJECT_SNAPSHOT_REQUEST_TIMEOUT_SECONDS,
))
.build()
{
Ok(client) => client,
Err(error) => {
let failure = ProjectSnapshotUploadError::new(
ProjectSnapshotUploadErrorKind::Transport,
format!("创建项目快照上传客户端失败:{error}"),
);
let paths = diff
.uploads
.iter()
.map(|candidate| candidate.relative_path.clone())
.collect::<Vec<_>>();
return ProjectSnapshotUploadReport::from_hard_failure(&paths, &failure);
}
};
for (index, candidate) in diff.uploads.iter().enumerate() {
match upload_project_snapshot_file(&client, session, project_id, candidate).await {
Ok(skipped_remote) => {
report
.uploaded_paths
.insert(candidate.relative_path.clone());
report.uploaded_bytes = report.uploaded_bytes.saturating_add(candidate.size_bytes);
if skipped_remote {
report.remote_skipped_files += 1;
}
}
Err(error) => {
let fatal = !error.kind().is_retryable();
report.failures.push(ProjectSnapshotUploadFailure {
relative_path: candidate.relative_path.clone(),
code: error.code().to_string(),
detail: error.detail.clone(),
});
if fatal {
for remaining in diff.uploads.iter().skip(index + 1) {
report.failures.push(ProjectSnapshotUploadFailure {
relative_path: remaining.relative_path.clone(),
code: error.code().to_string(),
detail: "上一次请求已确定性失败,本轮不再继续".to_string(),
});
}
break;
}
}
}
}
report
}
/// 返回是否为"远端已存在同内容对象"的跳过。
async fn upload_project_snapshot_file(
client: &reqwest::Client,
session: &PlatformSessionSnapshot,
project_id: &str,
candidate: &ProjectSnapshotUploadCandidate,
) -> Result<bool, ProjectSnapshotUploadError> {
let bytes = read_project_snapshot_file_bytes(&candidate.absolute_path).map_err(|error| {
ProjectSnapshotUploadError::new(ProjectSnapshotUploadErrorKind::Rejected, error)
})?;
if bytes.len() as u64 != candidate.size_bytes {
return Err(ProjectSnapshotUploadError::new(
ProjectSnapshotUploadErrorKind::FileChanged,
format!(
"项目快照文件在同步期间发生变化:{}",
candidate.relative_path
),
));
}
// 上传前再复核一次元数据:只比长度会漏掉"改完又改回同样大小"的情况,
// 这种文件本轮不传,也不进入索引。
if !project_snapshot_file_matches(
&candidate.absolute_path,
candidate.size_bytes,
candidate.modified_ms,
) {
return Err(ProjectSnapshotUploadError::new(
ProjectSnapshotUploadErrorKind::FileChanged,
format!(
"项目快照文件在同步期间发生变化:{}",
candidate.relative_path
),
));
}
let url = project_snapshot_endpoint(
session,
PROJECT_SNAPSHOT_FILES_ENDPOINT,
[
("projectId", project_id.to_string()),
("relativePath", candidate.relative_path.clone()),
("checksum", candidate.checksum.clone()),
("sizeBytes", candidate.size_bytes.to_string()),
],
)?;
let request = client
.post(url)
.bearer_auth(&session.access_token)
.header(reqwest::header::CONTENT_TYPE, "application/octet-stream")
.body(bytes);
let response = crate::http_client::with_agc_main_site_marker(request)
.send()
.await
.map_err(|error| {
ProjectSnapshotUploadError::new(
ProjectSnapshotUploadErrorKind::Transport,
format!("项目快照上传请求失败:{error}"),
)
})?;
let status = response.status();
if !status.is_success() {
return Err(project_snapshot_response_error(status, response).await);
}
let parsed = response
.json::<shared_contracts::agc_project_snapshots::AgcProjectSnapshotFileUploadResponse>()
.await
.map_err(|error| {
ProjectSnapshotUploadError::new(
ProjectSnapshotUploadErrorKind::Upstream,
format!("项目快照上传响应无法解析:{error}"),
)
})?;
if parsed.relative_path != candidate.relative_path || parsed.checksum != candidate.checksum {
return Err(ProjectSnapshotUploadError::new(
ProjectSnapshotUploadErrorKind::Upstream,
format!("项目快照上传响应与请求不一致:{}", candidate.relative_path),
));
}
Ok(parsed.skipped)
}
/// 提交本次同步的远端清单;删除只在这里表达。
pub(crate) async fn upload_project_snapshot_manifest(
session: &PlatformSessionSnapshot,
payload: &shared_contracts::agc_project_snapshots::AgcProjectSnapshotManifestRequest,
) -> Result<(), ProjectSnapshotUploadError> {
let client = crate::http_client::agc_main_site_client_builder()
.timeout(Duration::from_secs(
PROJECT_SNAPSHOT_REQUEST_TIMEOUT_SECONDS,
))
.build()
.map_err(|error| {
ProjectSnapshotUploadError::new(
ProjectSnapshotUploadErrorKind::Transport,
format!("创建项目快照上传客户端失败:{error}"),
)
})?;
let url = project_snapshot_endpoint(session, PROJECT_SNAPSHOT_MANIFEST_ENDPOINT, [])?;
let request = client
.post(url)
.bearer_auth(&session.access_token)
.json(payload);
let response = crate::http_client::with_agc_main_site_marker(request)
.send()
.await
.map_err(|error| {
ProjectSnapshotUploadError::new(
ProjectSnapshotUploadErrorKind::Transport,
format!("项目快照清单上传请求失败:{error}"),
)
})?;
let status = response.status();
if !status.is_success() {
return Err(project_snapshot_response_error(status, response).await);
}
Ok(())
}
fn project_snapshot_endpoint(
session: &PlatformSessionSnapshot,
path: &str,
query: impl IntoIterator<Item = (&'static str, String)>,
) -> Result<reqwest::Url, ProjectSnapshotUploadError> {
let base_url = session.api_base_url.trim_end_matches('/');
if base_url.is_empty() {
return Err(ProjectSnapshotUploadError::new(
ProjectSnapshotUploadErrorKind::Authentication,
"登录态缺少 API Server 地址",
));
}
let mut url = reqwest::Url::parse(&format!("{base_url}{path}")).map_err(|error| {
ProjectSnapshotUploadError::new(
ProjectSnapshotUploadErrorKind::Transport,
format!("项目快照接口地址无效:{error}"),
)
})?;
{
let mut pairs = url.query_pairs_mut();
for (key, value) in query {
pairs.append_pair(key, &value);
}
}
Ok(url)
}
async fn project_snapshot_response_error(
status: reqwest::StatusCode,
response: reqwest::Response,
) -> ProjectSnapshotUploadError {
let kind = match status.as_u16() {
401 => ProjectSnapshotUploadErrorKind::Authentication,
403 => ProjectSnapshotUploadErrorKind::Permission,
// 服务端配额与频率闸门:等下一次触发,不做本轮内重试。
429 => ProjectSnapshotUploadErrorKind::Throttled,
code if (400..500).contains(&code) => ProjectSnapshotUploadErrorKind::Rejected,
_ => ProjectSnapshotUploadErrorKind::Upstream,
};
// 上游正文可能带内部信息,只保留状态码与有界摘要。
let detail = response
.text()
.await
.map(|body| body.chars().take(200).collect::<String>())
.unwrap_or_default();
let detail = detail.trim().to_string();
if detail.is_empty() {
ProjectSnapshotUploadError::new(kind, format!("项目快照接口返回 HTTP {}", status.as_u16()))
} else {
ProjectSnapshotUploadError::new(
kind,
format!("项目快照接口返回 HTTP {}{detail}", status.as_u16()),
)
}
}
+10
View File
@@ -162,6 +162,16 @@ ALIYUN_OSS_POST_EXPIRE_SECONDS=600
ALIYUN_OSS_POST_MAX_SIZE_BYTES=20971520
ALIYUN_OSS_SUCCESS_ACTION_STATUS=200
# AGC 项目定时快照上传目标。对象只落在服务端私有前缀
# agc/project-snapshots/v1/{user}/{project}/ 下;AccessKey 为空时回退 ALIYUN_OSS_ACCESS_KEY_*
# 因此回退凭据必须对目标 bucket 具备该前缀的 PutObject/GetObject/DeleteObject 权限。
# bucket 未单独配置时默认 agc-dev,未配置凭据时 api-server 跳过该客户端,
# 接口返回 503 且客户端失败关闭(不写空对象、不推进本地索引)。
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET=agc-dev
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT=oss-rg-china-mainland.aliyuncs.com
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID=
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET=
# SpacetimeDB 数据目录 OSS 冷备份配置。可由 cron / Jenkins 调用发布包内 scripts/database-backup-to-oss.mjs。
GENARRATIVE_DATABASE_BACKUP_DATA_DIR=/stdb
GENARRATIVE_DATABASE_BACKUP_WORK_DIR=/var/lib/genarrative/database-backups
@@ -0,0 +1,43 @@
# AGC 项目定时快照上传实施计划
Version: 1.0
Status: active
Date: 2026-09-17
Parent Milestone: `【里程碑】AGC项目定时快照上传-2026-09-17.md`
## 修改边界
1. `server-rs/crates/shared-contracts/src/`:新增 `agc_project_snapshots` DTO(单文件上传请求/响应、同步清单信封),只放共享字段,不放 OSS 细节。
2. `apps/ai-game-creator-shell/src-tauri/src/project_snapshot/`:新增客户端模块,包含扫描与排除规则、索引读写、差异对比、上传编排、状态与日志;不修改 `project/` 下既有 manifest 与写锁语义。
3. `apps/ai-game-creator-shell/src-tauri/src/main.rs`:注册新模块、命令与生命周期钩子;`windows.rs` 的窗口关闭与应用退出路径接入触发调用,不改变现有窗口创建/关闭顺序。
4. `server-rs/crates/platform-oss/src/lib.rs`:新增项目快照私有前缀常量与(必要时)独立 bucket 配置入口;不改动既有前缀枚举语义与资源写路径。
5. `server-rs/crates/api-server/src/project_snapshots.rs`:新增路由、鉴权、校验与 OSS 写入;不改动 `error_reports``assets` 既有路由。
6. `.env.example`:补充 `GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_*` 说明与默认值。
7. `docs/`:主规范已更新;完成后把持久结论合并回主规范并删除本计划。
## 实现顺序
1. 先写 `shared-contracts` DTO 与客户端差异引擎(扫描、排除、索引、diff)及单测,此时无网络依赖,可独立验证。
2. 接上传编排:按差异集合逐文件提交,成功后再提交清单,最后推进索引;用本地 TCP stub server 覆盖成功、幂等跳过、鉴权失败与部分失败路径。
3. 接触发接线:周期定时器、工作区窗口关闭与应用退出;确认关闭路径的有界超时和串行化。
4. 最后接服务端路由与 OSS 写入,补参数校验与幂等跳过测试;服务端完成前客户端按"未配置即失败关闭、不写入索引"处理。
每一步都保留既有失败关闭行为;新模块默认不改变其它同步路径(Runner、项目写锁、Resource Editor)。
## 验证命令
- `cargo fmt --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml -- --check`
- `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml project_snapshot -- --nocapture`
- `cargo test -p api-server --bin api-server project_snapshots -- --nocapture`
- `cargo fmt -p api-server -p shared-contracts -p platform-oss -- --check`
- `npm run --prefix apps/ai-game-creator-shell typecheck`(若触及前端)
- `npm run check:encoding`
- `git diff --check`
- 运行时按需:`npm run agc` 打开项目观察索引写入与同步日志,关闭窗口确认关闭触发。
## 风险与回滚
- 上传体积与带宽:首轮全量可能很大,先设单文件与单次同步总量上限并把超限项记入跳过清单;不静默截断。
- 数据出境边界:只上传项目目录内普通文件,排除 `.agent/runtime``.agent/logs``.git`、构建产物与临时文件;凭据类文件不在白名单内。
- 服务端未配置 bucket 时客户端必须失败关闭,不能把本地索引推进成"已同步",否则后续同步会漏传。
- 回滚:客户端可停用触发接线(保留模块与测试)即可回到无上传行为;服务端路由与配置项可单独移除,不影响既有 OSS 前缀与错误报告链路。
@@ -0,0 +1,49 @@
# AGC 项目定时快照上传
Version: 1.0
Status: active
Date: 2026-09-17
Parent Spec: `docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md` 的“2026-09-17 AGC 项目定时快照上传(agc-dev)”
## 目标
AGC 在项目打开期间按周期把用户项目增量上传到 OSS `agc-dev`,并在项目关闭时立即补一次同步;只上传内容发生变化的文件,重复内容不重复上传,远端缺少对应对象时才新建。
## 范围
- 客户端 `src-tauri/src/project_snapshot/``scan.rs` / `diff.rs` / `index.rs` / `transport.rs`):项目扫描、排除规则、增量索引与差异对比、上传编排、状态查询。
- 触发接线:工作区窗口存活周期定时器、工作区窗口关闭(`CloseRequested`);应用退出只做有界等待,不重复发起同步。
- 服务端 `POST /api/agc/project-snapshots/files``POST /api/agc/project-snapshots/manifest`:登录态鉴权、参数校验、私有前缀 OSS 写入、HEAD 幂等跳过。
- 目标 bucket 配置:`GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_*`,默认 `agc-dev`
- 契约:`shared-contracts::agc_project_snapshots` 新增请求/响应 DTO 与项目 ID、相对路径、摘要校验函数。
- 客户端增量索引:`<AppData>/project-snapshots/<projectId>/index.json`,按用户身份判等,换号后按冷启动全量重算。
- 排除口径:复用 `should_skip_project_snapshot_path`(整个 `.agent``.git`、构建与依赖目录、凭据目录、敏感后缀、符号链接与重解析点)。
## 不做
- 不做云端下载/恢复、跨设备合并、版本回滚。
- 不保留多版本历史:清单写入成功后回收不再被引用的旧对象,同一路径只保留当前内容。
- 不下发 bucket 生命周期策略;不做跨节点的用户级总量配额与计费口径。
- 不新增 SpacetimeDB 表或 procedure,不修改 `/api/external/v1` 与 External OpenAPI。
- 不在客户端暴露上传状态、时间线或入口按钮;状态只落本机诊断日志,排障走 native-only 命令。
## 验收标准
1. 首次同步上传项目内全部符合条件的普通文件;再次同步在无改动时上传 0 个文件。
2. 只修改一个文件时,差异集合恰好包含一个修改项;删除一个文件时上传集合为空且清单中不再包含该文件。
3. `(字节数, 修改时间)` 未变的文件复用已存摘要,不重复读取内容计算摘要。
4. 排除规则命中项(`.agent/runtime``.agent/logs``.git``node_modules`、构建产物、临时文件、符号链接)与超限文件进入跳过清单,不进入上传集合。
5. 任一次同步失败(非鉴权类)不推进本地索引,下一次触发重算并重试;鉴权/权限类失败不自动重试。
6. 同一项目的并发触发串行执行,不产生两路重复上传。
7. 工作区窗口关闭与应用退出都会触发一次同步,且关闭路径不因同步失败而阻塞退出超过超时上限。
8. 服务端拒绝越界 `projectId`、相对路径与摘要;相同摘要重复提交走跳过分支且不写入新对象。
9. 新增日志与错误文案不含 Access Token、AccessKey、绝对路径与项目内容。
10. 清单写入成功后,上一版清单里不再被引用的对象被回收;上一版清单不可读时整轮不删除任何对象。
11. 单项目超过 2 GiB 时客户端明确失败、服务端按 413 拒绝;超过服务端小时配额或 5 秒最小间隔时返回 429 且带 `Retry-After`
12. 同步期间被改写的文件既不上传也不推进索引,沿用上一轮记录,且不会被误判成删除。
## 依赖
- 现有 `platform_session`(用户身份与 Access Token)、项目 manifest(稳定 `project_id`)。
- 现有 `platform-oss`PUT/HEAD、私有访问)、`api-server` 登录态中间件与 `shared-contracts`
- 现有 AppData 私有文件写入与目录解析工具。
@@ -1530,3 +1530,56 @@ Direct 回合的所有权属于进程内项目身份锁,不属于当前页面
- 两个窗口同时对同一项目发起 Runtime 写请求时,用户体验仍由项目级写锁串行决定;本次不引入跨窗口排队提示。
- 平台会话在窗口间传播依赖共享 localStorage 与 Runner 权威;渲染层不做跨窗口事件推送,另一个窗口在下一次会话校验或刷新时收敛。
## 2026-09-17 AGC 项目定时快照上传(agc-dev)
### 目标与非目标
- 目标:AGC 在项目工作区打开期间按固定周期把用户项目增量上传到 OSS `agc-dev`,并在项目关闭时立即补一次同步;重复内容不重复上传,远端占用跟随当前清单收敛。
- 非目标:不做云端下载/恢复、不做跨设备合并、不保留多版本历史、不新增面向用户的上传界面、不修改 `/api/external/v1` 与 OpenAPI、不新增 SpacetimeDB 表。
- 非目标:不把 OSS AccessKey 放进客户端;客户端不直连 OSS。
### 参与入口、状态与跨模块边界
- 触发入口有两个:工作区窗口 `main` 存活期间的周期定时器、工作区窗口关闭事件(`CloseRequested`)。两者共用同一个进程内同步器,同一项目的同步串行执行,周期触发在已有同步进行时直接让位,不排队堆积。
- 应用退出(`RunEvent::Exit`)不重复发起同步:该时刻窗口已销毁,按窗口重新枚举项目只会得到空集;退出路径只负责在有界预算(15 秒)内等待在途同步收尾,让关窗触发的那一次同步能写完索引再退出。
- 客户端扫描、差异对比、索引持久化与上传编排都在 Tauri Rust 进程(`src-tauri/src/project_snapshot/`);WebView 只读状态,不参与差异计算。
- 本地索引是增量对比的唯一依据:`<AppData>/project-snapshots/<projectId>/index.json` 保存上次成功同步的相对路径、校验和、字节数和修改时间。项目根使用现有 manifest 的稳定 `project_id` 作为远端身份,路径不再作为身份。
- 可观测性按产品口径收敛到本机日志:同步结果、失败分类、延后与跳过计数只写入 AppData 诊断日志(`project_snapshot.sync.*` 前缀),客户端界面不暴露上传状态、时间线或入口按钮。`read_local_project_snapshot_state``sync_local_project_snapshot` 两条命令仅作为 native-only 的排障与联调入口登记,不在渲染层调用。
- 远端写入经 `api-server`,客户端只持平台登录态 Access Token。两条登录态路由:`POST /api/agc/project-snapshots/files`(单文件,正文为原始字节,元数据走查询串)与 `POST /api/agc/project-snapshots/manifest`(本次同步后的完整清单)。
- 对象键与清单由服务端决定:文件键为 `agc/project-snapshots/v1/{userId}/{projectId}/files/{sizeBytes}-{checksumDigest}/{relPath}`,清单键为 `agc/project-snapshots/v1/{userId}/{projectId}/manifest.json`。键里带字节数与摘要,因此"对象已存在且长度一致"可以作为内容一致的判据;路径按原始大小写保留,不走 `put_object` 的低位规范化。`agc` 前缀继续是服务端专用私有前缀,通用对象键解析与客户端直传票据都不覆盖它。
- 目标 bucket 使用独立配置 `GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET` / `_ENDPOINT` / `_ACCESS_KEY_ID` / `_ACCESS_KEY_SECRET`,默认 `agc-dev` + `oss-rg-china-mainland.aliyuncs.com`,未配置时回退 `ALIYUN_OSS_*`;与"资源 bucket 与备份 bucket 分离"的既有口径一致。
### 正常、失败、重试与幂等行为
- 差异对比口径:先按 `相对路径 + 字节数 + 修改时间` 判定是否候选变更,命中旧记录则复用已存 `sha256`,只有 `(size, mtime)` 变化才重算摘要。产出新增、修改、删除三类集合,只上传新增与修改的文件。
- 每次成功同步的最后一步上传该项目的 `manifest.json`(当前全量文件清单:相对路径、摘要、字节数、同步序号)。清单描述的是项目当前全量内容,因此清单体积就是该项目在 OSS 上的常驻占用。
- 远端回收:清单写入成功后,服务端读取上一版清单,按 `(路径, 字节数, 摘要)` 反推出不再被当前清单引用的对象键并删除。只处理上一版清单登记过的键,不做 LIST,因此不可能误删其它项目或其它功能的对象;单次最多回收 2000 个对象,剩余部分留到下一次清单写入继续;上一版清单读不到或解析失败时整轮跳过回收(fail-closed)。单个删除失败只记日志,不影响本次同步语义。
- 因此本功能是"当前状态镜像 + 清单",不保留历史版本:同一路径的内容变化会覆盖式替换远端对象,回滚能力不在本轮范围内。
- 幂等:同一摘要与字节数的对象重复提交由服务端 HEAD 校验后跳过;探测失败按"未存在"处理并照常 PUT,宁可多传一次也不漏传。索引只在清单写入成功后推进,失败时保留旧索引以便下次重算。
- 与项目写锁解耦:同步不持有项目写锁,也不阻塞 Agent 写入。读取摘要后与上传前各按 `(字节数, 修改时间)` 复核一次,任一处不一致就判定该文件"同步期间发生变化":本轮不上传、不写入索引;若该路径上一轮已同步过则沿用旧记录,避免被误判成删除而触发远端回收。这类文件在下一个周期或下次关窗时重算重试。
- 失败关闭:单个文件失败不推进该文件的索引项,失败文件与剩余文件在下一次周期或下次关闭时重试。鉴权失败(401/403)与格式类拒绝(400/413)是确定性失败,停止本轮剩余请求并等待用户处理后重试;服务端配额或频率拒绝(429)与传输类失败按可重试处理。
- 配额与限流:单文件 64 MiB、单次同步上传预算 512 MiB(超出部分延后到下一次)、单项目常驻上限 2 GiB(客户端在扫描后先判,超限直接给出明确失败;服务端按清单累计体积复核并返回 413);服务端按用户做进程内小时配额(文件 3000 次、清单 120 次)并对同一项目强制 5 秒最小清单间隔,超限返回 429 且带 `Retry-After`。进程内配额只用于抑制异常客户端与失控重试,跨节点配额由"单项目上限 + 清单引用回收"保证。
- 生命周期:同步有界超时(单文件与整次同步分别设上限),项目关闭与应用退出路径不因同步失败而阻塞或延迟退出超过超时上限。
- 上传内容边界:复用项目索引与 checkpoint 同一份 `should_skip_project_snapshot_path` 口径——整个 `.agent`(含 runtime、logs、checkpoint、manifest、project.lock)、版本控制目录、`node_modules`/`target`/`dist`/`build`/`coverage`/`.cache`、凭据目录与 `.pem`/`.key` 等敏感后缀都不参与同步;符号链接与重解析点同样跳过。单文件(64 MiB)与单次同步总量(512 MiB)各有上限,超限文件进入跳过或延后清单而不是静默丢弃。
### 契约与兼容
- 新增登录态内部路由 `POST /api/agc/project-snapshots/files`,请求 DTO 放在 `shared-contracts`;不属于 `/api/external/v1`,因此不更新 External OpenAPI,与 `/api/error-reports` 同类。
- 服务端校验 `projectId` 形态(拒绝路径分隔符、`..`、控制字符与超长值)、相对路径规范(正斜杠、拒绝绝对路径与穿越)、摘要形态(`fnv1a64:` + 16 位十六进制)和字节数上限,任何越界返回 4xx 而不是写入 OSS。
- 不改变客户端与 Runner 的本机协议、平台会话语义、项目写锁与 manifest 结构;新增索引文件位于 AppData,不进入用户项目目录。
### 验收标准与证据来源
- 定向 Rust 测试:首次同步全量、仅改一个文件时只产生一个修改项、删除文件只体现在清单、`(size,mtime)` 未变时复用旧摘要、排除规则与上限跳过、同步失败不推进索引、同一项目并发触发串行化。
- 服务端测试:越界 `projectId`/相对路径/摘要被拒;相同摘要重复提交走跳过分支;超过单项目上限返回 413;超过用户小时配额返回 429;鉴权缺失返回 401;OSS 未配置返回明确的 5xx 而不是写入空对象。
- 运行时 smoke:AGC 开发态打开项目、观察索引写入与同步日志、关闭工作区窗口后确认关闭触发的那次同步执行;报告为"客户端 diff 已验证 / 服务端已配置环境联调"两层,不合并成一句"已通"。
- 边界:新增日志与错误文案不含 Access Token、AccessKey、绝对路径与项目内容。
### 未决问题
- 用户侧看不到同步状态与失败原因(界面按产品口径不暴露),排障只能读 AppData 诊断日志或调用 native-only 命令;如果后续要支持用户自助排查,需要先确认是否允许在客户端出现上传相关 UI。
- 历史版本:本轮只保留"当前状态镜像 + 清单",旧内容对象在清单写入成功后即被回收,没有回滚能力;要保留历史版本需要先定"保留几个 revision + 由谁回收"的策略。
- 用户级配额:跨节点的用户总量配额与计费口径未定;当前用单项目 2 GiB 上限 + 清单引用回收保证常驻占用有界,用户级总量只能靠项目数间接约束。
- 目标 bucket 的生命周期规则(例如转低频/过期删除)需要在部署环境确认后单独收口;功能本身已不再依赖它来控制增长。
- 大项目(素材数量多、单文件大)的首轮全量上传耗时与带宽占用未实测;单次预算 512 MiB 会把超出部分留到下一次同步,但并发上限与断点续传仍未引入。
@@ -540,6 +540,50 @@ curl -fsS --max-time 5 http://127.0.0.1/api/editor/showcase/resources >/dev/null
角色动画源帧 PUT、透明帧 PUT 和最终帧 HEAD 使用 `AppState` 内同一个 OSS HTTP Client/连接池,并受进程级 8 路 OSS permit 保护;BgFilter、阿里云抠图和本地处理不占用该 permit。每个 OSS attempt 最多 3 次(首次 + 2 次重试),退避为 250ms、500ms;只重试 timeout、无 HTTP 响应传输错误、OSS PutObject 的 `400 + RequestTimeout`、PUT 400 错误体读取失败(未解析出 `Code`,按 timeout/transport 归类)、408、429 和 500599。动作帧 PUT 收到 400 时只读取最多 16 KiB OSS 错误 XML,提取 `Code``RequestId``oss_request_id` 优先使用响应头 `x-oss-request-id`,XML 字段只作回退。错误体读取超时/断流不再按确定性 400 处理:已解析出的 `Code` 优先生效;未解析出 `Code` 时按读取失败原因置 `timeout`/`transport` 并重试,message 追加「错误响应体读取失败」。日志字段包括 `frame_index``object_key``operation=source_put|final_put|final_head``attempt``max_attempts``retryable``will_retry``retry_delay_ms``permit_wait_ms``timeout``connect``transport``oss_code``oss_request_id``status``elapsed_ms``请求 OSS 失败` 时,`timeout/connect/transport=true` 表示传输类失败;`status=400, oss_code=RequestTimeout, timeout=true``status=429``500599` 表示暂时性失败,PUT 的 `status=400``oss_code` 为空且 `timeout=true``transport=true`(message 含「错误响应体读取失败」)同样是暂时性失败。除 `RequestTimeout` 和该错误体读取失败两类例外外,其他 400、401/403/404、配置、URL 和签名错误是确定性失败,不会重试。最终帧 HEAD 失败只会重试 HEAD,不会重复 PUT;如果任一帧最终失败,确认整段动作已排空已启动 Future,并检查任务按现有契约退款且没有发布缺帧动画。
### AGC 项目快照上传目标
AGC 客户端按周期与项目关闭时机把用户项目增量上传到 `agc-dev`。客户端只持有平台登录态 Access
Token,经 `POST /api/agc/project-snapshots/files`(单文件原始字节)与
`POST /api/agc/project-snapshots/manifest`(本次同步清单)交给 `api-server`,由服务端写入私有前缀
`agc/project-snapshots/v1/{user}/{project}/`;客户端不直连 OSS,也不持有 OSS 凭据。
```env
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET=agc-dev
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT=oss-rg-china-mainland.aliyuncs.com
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID=
GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET=
```
专用凭据为空时回退 `ALIYUN_OSS_ACCESS_KEY_ID` / `ALIYUN_OSS_ACCESS_KEY_SECRET`bucket 与 endpoint
仍默认指向 AGC 发行 bucket,因此回退凭据必须具备目标 bucket 该前缀的 `PutObject` / `GetObject` /
`DeleteObject` 权限;`api-server` 启动时会打印一行 `AGC 项目快照 OSS 客户端已启用`(含 bucket、
endpoint 与凭据来源,不含密钥),凭据缺失或只配一半则跳过该客户端、接口返回 `503`,客户端按失败关闭
处理:不写空对象,也不推进本地增量索引,下一次触发重算重试。
远端占用按当前清单收敛:每次清单写入成功后,服务端用上一版清单反推不再被引用的对象键并删除,单次最多
回收 2000 个,上一版清单不可读时整轮跳过(不会误删)。因此不需要额外配置 bucket 生命周期来防止无界
增长;`agc/project-snapshots/v1/` 下同一路径只保留当前内容,历史版本不保留。配额口径:单文件 64 MiB、
单次同步上传预算 512 MiB、单项目常驻 2 GiB;超限分别表现为跳过/延后/413。服务端另有进程内小时配额
(文件 3000 次、清单 120 次)与同一项目 5 秒最小清单间隔,超限返回 `429` 并带 `Retry-After`
两层可重复的现场验证:
```bash
# 1. 存储层:直接对真实 bucket 做内部前缀写入、读回与清理,并在结束时删除探针对象。
cargo run -p platform-oss --example agc_project_snapshot_live_smoke --manifest-path server-rs/Cargo.toml
# 2. 客户端链路:真实差异引擎 → 本地 api-server → 真实 OSS。第一次必须 synced 且上传 > 0
# 紧接着的第二次必须 no-op 且上传 0 个文件;需要先取得登录态并指定项目与索引目录。
cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml \
project_snapshot_live_sync -- --ignored --nocapture
```
客户端冒烟读取 `GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_PROJECT`(项目绝对路径,建议用可丢弃的副本)、
`..._LIVE_TOKEN``..._LIVE_API_BASE_URL``..._LIVE_USER_ID``..._LIVE_CONFIG_DIR`(索引目录,
`cargo test` 进程没有窗口 setup 初始化 AppData 配置目录,必须显式指定);未设置时用例自我跳过。
本地联调可用 `npm run dev:api-server` 起 api-server,并用密码登录(开发态默认允许未知手机号自动注册)
取得 Access Token。
## 生产运维
生产部署当前口径:
+1
View File
@@ -51,6 +51,7 @@ pub fn build_router(state: AppState) -> Router {
.merge(modules::external_generation::router(state.clone()))
.merge(modules::platform_support::router(state.clone()))
.merge(modules::raw::router(state.clone()))
.merge(modules::project_snapshots::router(state.clone()))
.merge(crate::error_reports::router(state.clone()))
.route(
"/api/profile/recharge/wechat/notify",
+31
View File
@@ -27,6 +27,9 @@ const DEFAULT_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD: u32 = 3;
const DEFAULT_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS: u64 = 120;
const DEFAULT_ALIYUN_MATTING_ENDPOINT: &str = "imageseg.cn-shanghai.aliyuncs.com";
const DEFAULT_ALIYUN_MATTING_REQUEST_TIMEOUT_MS: u64 = 30_000;
/// AGC 项目快照默认落到 AGC 发行 bucket;私有前缀由 platform-oss 单独限制。
const DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_BUCKET: &str = "agc-dev";
const DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT: &str = "oss-rg-china-mainland.aliyuncs.com";
pub(crate) const OFFICIAL_LLM_ROUTER_BASE_URL: &str = "https://router.genarrative.world/v1";
pub(crate) const OFFICIAL_LLM_ROUTER_MODEL: &str = "gpt-6-astra";
const LLM_ROUTER_KEY_ENCRYPTION_DOMAIN: &[u8] = b"genarrative:llm-router-api-key-encryption:v1\0";
@@ -169,6 +172,12 @@ pub struct AppConfig {
pub oss_post_expire_seconds: u64,
pub oss_post_max_size_bytes: u64,
pub oss_success_action_status: u16,
/// AGC 项目快照上传目标。默认指向 AGC 发行用的公开 bucket,可用独立凭据覆盖;
/// 未单独配置时回退 `ALIYUN_OSS_*`,与数据库备份 bucket 的分离开关同口径。
pub project_snapshot_oss_bucket: String,
pub project_snapshot_oss_endpoint: String,
pub project_snapshot_oss_access_key_id: Option<String>,
pub project_snapshot_oss_access_key_secret: Option<String>,
pub spacetime_server_url: String,
pub spacetime_database: String,
pub spacetime_token: Option<String>,
@@ -476,6 +485,10 @@ impl Default for AppConfig {
oss_post_expire_seconds: 10 * 60,
oss_post_max_size_bytes: 20 * 1024 * 1024,
oss_success_action_status: 200,
project_snapshot_oss_bucket: DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_BUCKET.to_string(),
project_snapshot_oss_endpoint: DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT.to_string(),
project_snapshot_oss_access_key_id: None,
project_snapshot_oss_access_key_secret: None,
spacetime_server_url: "http://127.0.0.1:3000".to_string(),
spacetime_database: "genarrative-dev".to_string(),
spacetime_token: None,
@@ -1121,6 +1134,24 @@ impl AppConfig {
{
config.oss_success_action_status = oss_success_action_status;
}
config.project_snapshot_oss_bucket = read_first_non_empty_env(&[
"GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET",
"ALIYUN_OSS_BUCKET",
])
.unwrap_or_else(|| DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_BUCKET.to_string());
config.project_snapshot_oss_endpoint = read_first_non_empty_env(&[
"GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT",
"ALIYUN_OSS_ENDPOINT",
])
.unwrap_or_else(|| DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT.to_string());
config.project_snapshot_oss_access_key_id = read_first_non_empty_env(&[
"GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID",
"ALIYUN_OSS_ACCESS_KEY_ID",
]);
config.project_snapshot_oss_access_key_secret = read_first_non_empty_env(&[
"GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET",
"ALIYUN_OSS_ACCESS_KEY_SECRET",
]);
if let Some(spacetime_server_url) =
read_first_non_empty_env(&["GENARRATIVE_SPACETIME_SERVER_URL"])
+1
View File
@@ -66,6 +66,7 @@ mod process_metrics;
mod profile_identity;
mod profile_recharge_expiration_listener;
mod profile_recharge_refund_reconciliation;
mod project_snapshots;
mod prompt;
mod raw_image;
mod refresh_session;
@@ -10,4 +10,5 @@ pub mod internal;
pub mod platform;
pub mod platform_support;
pub mod profile;
pub mod project_snapshots;
pub mod raw;
@@ -0,0 +1,31 @@
use axum::{Router, extract::DefaultBodyLimit, middleware, routing::post};
use crate::{
auth::require_bearer_auth,
project_snapshots::{
MAX_FILE_REQUEST_BODY_BYTES, MAX_MANIFEST_REQUEST_BODY_BYTES, upload_project_snapshot_file,
upload_project_snapshot_manifest,
},
state::AppState,
};
/// AGC 项目快照只接受登录态客户端;两条路由都带体积门禁,超限请求在进入业务
/// 处理前就被拒绝。
pub fn router(state: AppState) -> Router<AppState> {
Router::new()
.route(
"/api/agc/project-snapshots/files",
post(upload_project_snapshot_file)
.route_layer(middleware::from_fn_with_state(
state.clone(),
require_bearer_auth,
))
.layer(DefaultBodyLimit::max(MAX_FILE_REQUEST_BODY_BYTES)),
)
.route(
"/api/agc/project-snapshots/manifest",
post(upload_project_snapshot_manifest)
.route_layer(middleware::from_fn_with_state(state, require_bearer_auth))
.layer(DefaultBodyLimit::max(MAX_MANIFEST_REQUEST_BODY_BYTES)),
)
}
File diff suppressed because it is too large Load Diff
+71
View File
@@ -273,6 +273,8 @@ pub struct AppStateInner {
test_external_background_removal_enqueue:
Arc<Mutex<Option<TestExternalBackgroundRemovalEnqueue>>>,
oss_client: Option<OssClient>,
/// AGC 项目快照专用 OSS 客户端:bucket 与凭据可以独立于资源 bucket。
project_snapshot_oss_client: Option<OssClient>,
#[cfg_attr(test, allow(dead_code))]
auth_store: InMemoryAuthStore,
/// 当前进程工作集所基于的正式认证投影版本;跨节点写入使用它做 CAS。
@@ -339,6 +341,10 @@ impl fmt::Debug for AppStateInner {
)
.field("admin_runtime_enabled", &self.admin_runtime.is_some())
.field("oss_client_enabled", &self.oss_client.is_some())
.field(
"project_snapshot_oss_client_enabled",
&self.project_snapshot_oss_client.is_some(),
)
.field("spacetime_client", &self.spacetime_client)
.field("tracking_outbox_enabled", &self.tracking_outbox.is_some())
.field(
@@ -546,6 +552,7 @@ impl AppState {
config.refresh_session_ttl_days,
)?;
let oss_client = build_oss_client(&config)?;
let project_snapshot_oss_client = build_project_snapshot_oss_client(&config)?;
let sms_provider = SmsAuthProvider::new(SmsAuthConfig::new(
SmsAuthProviderKind::parse(&config.sms_auth_provider).ok_or_else(|| {
SmsProviderError::InvalidConfig("短信 provider 配置非法".to_string())
@@ -663,6 +670,7 @@ impl AppState {
#[cfg(test)]
test_external_background_removal_enqueue: Arc::new(Mutex::new(None)),
oss_client,
project_snapshot_oss_client,
auth_store,
auth_projection_version: AtomicI64::new(auth_projection_version),
auth_projection_synced_revision: AtomicU64::new(initial_auth_store_revision),
@@ -1321,6 +1329,10 @@ impl AppState {
self.oss_client.as_ref()
}
pub fn project_snapshot_oss_client(&self) -> Option<&OssClient> {
self.project_snapshot_oss_client.as_ref()
}
pub fn password_entry_service(&self) -> &PasswordEntryService {
&self.password_entry_service
}
@@ -2309,6 +2321,65 @@ impl AdminRuntime {
}
}
/// AGC 项目快照专用 OSS 客户端。
///
/// 目标 bucket 独立于资源 bucket:专用凭据未配置时回退 `ALIYUN_OSS_*`,而 bucket 与
/// endpoint 默认指向 AGC 发行 bucket。凭据缺失或只配置一半时返回 `None`;路由层把
/// "未配置" 当作失败关闭,不写空对象也不推进客户端索引。
fn build_project_snapshot_oss_client(
config: &AppConfig,
) -> Result<Option<OssClient>, AppStateInitError> {
let bucket = config.project_snapshot_oss_bucket.trim();
let endpoint = config.project_snapshot_oss_endpoint.trim();
if bucket.is_empty() || endpoint.is_empty() {
warn!("AGC 项目快照 OSS bucket/endpoint 未配置,跳过项目快照 OSS 客户端初始化");
return Ok(None);
}
let access_key_id = config
.project_snapshot_oss_access_key_id
.as_deref()
.map(str::trim)
.unwrap_or("");
let access_key_secret = config
.project_snapshot_oss_access_key_secret
.as_deref()
.map(str::trim)
.unwrap_or("");
if access_key_id.is_empty() && access_key_secret.is_empty() {
return Ok(None);
}
if access_key_id.is_empty() || access_key_secret.is_empty() {
warn!("AGC 项目快照 OSS AccessKey 配置不完整,跳过项目快照 OSS 客户端初始化");
return Ok(None);
}
let oss_config = OssConfig::new(
bucket.to_string(),
endpoint.to_string(),
access_key_id.to_string(),
access_key_secret.to_string(),
config.oss_read_expire_seconds,
config.oss_post_expire_seconds,
config.oss_post_max_size_bytes,
config.oss_success_action_status,
)?;
let credential_source = if config
.project_snapshot_oss_access_key_id
.as_deref()
.is_some_and(|value| !value.trim().is_empty())
{
"GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID"
} else {
"ALIYUN_OSS_ACCESS_KEY_ID"
};
info!(
bucket = %bucket,
endpoint = %endpoint,
credential_source,
"AGC 项目快照 OSS 客户端已启用"
);
Ok(Some(OssClient::new(oss_config)))
}
fn build_oss_client(config: &AppConfig) -> Result<Option<OssClient>, AppStateInitError> {
let oss_fields = [
("ALIYUN_OSS_BUCKET", config.oss_bucket.as_deref()),
@@ -0,0 +1,278 @@
//! AGC 项目快照真实 OSS 冒烟:直接验证 `agc-dev` bucket 上的内部前缀读写与清理。
//!
//! 默认从仓库根目录的 `.env`、`.env.local`、`.env.secrets.local` 读取 OSS 配置,
//! 非空 shell 环境变量优先;凭据优先使用
//! `GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID`,未配置时回退 `ALIYUN_OSS_ACCESS_KEY_ID`。
//!
//! ```text
//! cargo run -p platform-oss --example agc_project_snapshot_live_smoke --manifest-path server-rs/Cargo.toml
//! ```
//!
//! 冒烟只写入 `agc/project-snapshots/v1/` 下的固定探针对象,并在结束时删除;
//! 任何一步失败都会打印 `[FAIL]` 并以非 0 退出码结束,方便 CI 或人工判定。
use std::{
collections::{HashMap, HashSet},
env, fs,
path::Path,
time::Duration,
};
use platform_oss::{
DEFAULT_POST_EXPIRE_SECONDS, DEFAULT_POST_MAX_SIZE_BYTES, DEFAULT_READ_EXPIRE_SECONDS,
DEFAULT_SUCCESS_ACTION_STATUS, OssClient, OssConfig, OssDeleteObjectRequest, OssError,
OssInternalPutObjectRequest, OssObjectAccess, agc_project_snapshot_file_object_key,
agc_project_snapshot_manifest_object_key,
};
const DEFAULT_BUCKET: &str = "agc-dev";
const DEFAULT_ENDPOINT: &str = "oss-rg-china-mainland.aliyuncs.com";
const SMOKE_USER_ID: &str = "smoke-user";
const SMOKE_PROJECT_ID: &str = "smoke-project";
const SMOKE_RELATIVE_PATH: &str = "smoke/README.txt";
const SMOKE_BODY: &[u8] = b"agc project snapshot live smoke\n";
const SMOKE_CHECKSUM_DIGEST: &str = "0123456789abcdef";
type SmokeResult<T> = Result<T, String>;
#[tokio::main(flavor = "current_thread")]
async fn main() {
match run().await {
Ok(()) => {
println!("[PASS] AGC 项目快照 OSS 冒烟全部通过");
}
Err(error) => {
println!("[FAIL] {error}");
std::process::exit(1);
}
}
}
async fn run() -> SmokeResult<()> {
let repo_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..");
let local_env = load_local_env(&repo_root)?;
let bucket = non_empty_env(&local_env, "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET")
.unwrap_or_else(|| DEFAULT_BUCKET.to_string());
let endpoint = non_empty_env(&local_env, "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT")
.unwrap_or_else(|| DEFAULT_ENDPOINT.to_string());
let access_key_id = non_empty_env(
&local_env,
"GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID",
)
.or_else(|| non_empty_env(&local_env, "ALIYUN_OSS_ACCESS_KEY_ID"))
.ok_or_else(|| {
"缺少 AGC 项目快照 OSS AccessKey ID(或 ALIYUN_OSS_ACCESS_KEY_ID".to_string()
})?;
let access_key_secret = non_empty_env(
&local_env,
"GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET",
)
.or_else(|| non_empty_env(&local_env, "ALIYUN_OSS_ACCESS_KEY_SECRET"))
.ok_or_else(|| {
"缺少 AGC 项目快照 OSS AccessKey Secret(或 ALIYUN_OSS_ACCESS_KEY_SECRET".to_string()
})?;
let config = OssConfig::new(
bucket.clone(),
endpoint.clone(),
access_key_id,
access_key_secret,
DEFAULT_READ_EXPIRE_SECONDS,
DEFAULT_POST_EXPIRE_SECONDS,
DEFAULT_POST_MAX_SIZE_BYTES,
DEFAULT_SUCCESS_ACTION_STATUS,
)
.map_err(|error| format!("OSS 配置无效({}", oss_error_label(&error)))?;
let client = OssClient::new(config);
println!("目标:bucket={bucket} endpoint={endpoint}");
let http = reqwest::Client::builder()
.timeout(Duration::from_secs(60))
.no_proxy()
.build()
.map_err(|error| format!("创建 HTTP 客户端失败:{error}"))?;
// 1. 内部前缀白名单:非内部键必须在本地被拒绝,不产生网络请求。
match client
.put_internal_object(
&http,
OssInternalPutObjectRequest {
object_key: "generated-characters/smoke/master.png".to_string(),
content_type: Some("application/octet-stream".to_string()),
access: OssObjectAccess::Private,
metadata: Default::default(),
body: SMOKE_BODY.to_vec(),
},
)
.await
{
Err(OssError::InvalidRequest(_)) => println!("[PASS] 非内部前缀被本地拒绝,未写入 OSS"),
Ok(_) => return Err("非内部前缀竟然写入成功,内部前缀白名单失效".to_string()),
Err(error) => {
return Err(format!(
"非内部前缀返回了非预期错误({}",
oss_error_label(&error)
));
}
}
// 3. 项目快照文件键:写入 → 读回。
let file_key = agc_project_snapshot_file_object_key(
SMOKE_USER_ID,
SMOKE_PROJECT_ID,
SMOKE_BODY.len() as u64,
SMOKE_CHECKSUM_DIGEST,
SMOKE_RELATIVE_PATH,
)
.map_err(|error| format!("构造项目快照文件键失败({}", oss_error_label(&error)))?;
let manifest_key = agc_project_snapshot_manifest_object_key(SMOKE_USER_ID, SMOKE_PROJECT_ID)
.map_err(|error| format!("构造项目快照清单键失败({}", oss_error_label(&error)))?;
let result = write_and_verify(&client, &http, &bucket, &file_key, &manifest_key).await;
for key in [&file_key, &manifest_key] {
let _ = client
.delete_object(
&http,
OssDeleteObjectRequest {
object_key: key.clone(),
},
)
.await;
}
match client.head_internal_object(&http, &file_key).await {
Ok(None) => println!("[PASS] 探针对象已清理"),
Ok(Some(_)) => println!("[WARN] 探针对象仍存在,请手工清理 {file_key}"),
Err(error) => println!(
"[WARN] 清理后复核失败({}),请手工确认 {file_key}",
oss_error_label(&error)
),
}
result
}
async fn write_and_verify(
client: &OssClient,
http: &reqwest::Client,
bucket: &str,
file_key: &str,
manifest_key: &str,
) -> SmokeResult<()> {
client
.put_internal_object(
http,
OssInternalPutObjectRequest {
object_key: file_key.to_string(),
content_type: Some("application/octet-stream".to_string()),
access: OssObjectAccess::Private,
metadata: Default::default(),
body: SMOKE_BODY.to_vec(),
},
)
.await
.map_err(|error| {
format!(
"写入项目快照文件失败({}):凭据可能没有 {bucket} 的 PutObject 权限",
oss_error_label(&error)
)
})?;
println!("[PASS] 项目快照文件已写入 OSS:{file_key}");
let existing = client
.head_internal_object(http, file_key)
.await
.map_err(|error| format!("读回项目快照文件失败({}", oss_error_label(&error)))?
.ok_or_else(|| "写入成功但对象读回不存在".to_string())?;
if existing.content_length != SMOKE_BODY.len() as u64 {
return Err(format!(
"对象长度不一致:期望 {},实际 {}",
SMOKE_BODY.len(),
existing.content_length
));
}
println!(
"[PASS] 对象读回一致,服务端跳过判据成立:contentLength={} etag={}",
existing.content_length,
existing.etag.as_deref().unwrap_or("-")
);
client
.put_internal_object(
http,
OssInternalPutObjectRequest {
object_key: manifest_key.to_string(),
content_type: Some("application/json".to_string()),
access: OssObjectAccess::Private,
metadata: Default::default(),
body: br#"{"schemaVersion":1,"projectId":"smoke-project"}"#.to_vec(),
},
)
.await
.map_err(|error| format!("写入项目快照清单失败({}", oss_error_label(&error)))?;
println!("[PASS] 项目快照清单已覆盖写入 OSS:{manifest_key}");
Ok(())
}
fn oss_error_label(error: &OssError) -> String {
format!("{error}")
}
fn load_local_env(repo_root: &Path) -> SmokeResult<HashMap<String, String>> {
let shell = env::vars().collect::<HashMap<_, _>>();
let protected = shell
.iter()
.filter(|(_, value)| !value.trim().is_empty())
.map(|(key, _)| key.clone())
.collect::<HashSet<_>>();
let mut merged = shell;
for file_name in [".env", ".env.local", ".env.secrets.local"] {
let path = repo_root.join(file_name);
if !path.is_file() {
continue;
}
let contents =
fs::read_to_string(&path).map_err(|_| format!("无法读取本地配置文件 {file_name}"))?;
for raw_line in contents.lines() {
let line = raw_line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let Some((key, raw_value)) = line.split_once('=') else {
continue;
};
if !valid_env_key(key) || protected.contains(key) {
continue;
}
merged.insert(
key.to_string(),
trim_env_quotes(raw_value.trim()).to_string(),
);
}
}
Ok(merged)
}
fn valid_env_key(key: &str) -> bool {
let mut chars = key.chars();
chars
.next()
.is_some_and(|value| value == '_' || value.is_ascii_alphabetic())
&& chars.all(|value| value == '_' || value.is_ascii_alphanumeric())
}
fn trim_env_quotes(value: &str) -> &str {
if value.len() >= 2
&& ((value.starts_with('"') && value.ends_with('"'))
|| (value.starts_with('\'') && value.ends_with('\'')))
{
&value[1..value.len() - 1]
} else {
value
}
}
fn non_empty_env(local_env: &HashMap<String, String>, name: &str) -> Option<String> {
local_env
.get(name)
.map(|value| value.trim())
.filter(|value| !value.is_empty())
.map(str::to_string)
}
+338 -6
View File
@@ -114,6 +114,20 @@ pub struct OssDeleteObjectRequest {
pub object_key: String,
}
/// 服务端专用内部对象写入请求。
///
/// 与 `OssPutObjectRequest` 的区别是对象键由调用方按内部前缀完整给出,不再走
/// `path_segments`/`file_name` 的低位规范化,因此可以保留项目内的原始大小写与
/// 目录层级。键必须落在 `normalize_internal_object_key` 允许的内部前缀下。
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OssInternalPutObjectRequest {
pub object_key: String,
pub content_type: Option<String>,
pub access: OssObjectAccess,
pub metadata: BTreeMap<String, String>,
pub body: Vec<u8>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OssPutObjectRequest {
pub prefix: LegacyAssetPrefix,
@@ -796,11 +810,7 @@ impl OssClient {
}
let headers = response.headers();
let content_length = headers
.get(reqwest::header::CONTENT_LENGTH)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(0);
let content_length = head_object_content_length(headers);
let content_type = headers
.get(reqwest::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
@@ -933,6 +943,129 @@ impl OssClient {
))
}
/// 按内部前缀写入服务端专用对象。内容为空、键越界都直接失败,不写空对象。
pub async fn put_internal_object(
&self,
client: &reqwest::Client,
request: OssInternalPutObjectRequest,
) -> Result<OssPutObjectResponse, OssError> {
if request.body.is_empty() {
return Err(OssError::InvalidRequest(
"服务端内部对象内容不能为空".to_string(),
));
}
let object_key = normalize_internal_object_key(&request.object_key)?;
let content_type = normalize_optional_value(request.content_type);
let headers = build_put_object_headers(request.metadata)?;
let target_url = build_object_url(&self.config.bucket, &self.config.endpoint, &object_key)
.map_err(|error| {
request_error(
OssRequestOperation::Put,
&format!("构造 OSS 对象 URL 失败:{error}"),
)
})?;
let content_length = u64::try_from(request.body.len())
.map_err(|_| OssError::InvalidRequest("上传对象大小超出可支持范围".to_string()))?;
let builder = signed_request_builder(
client,
&self.config,
Method::PUT,
Some(&object_key),
target_url,
content_type.as_deref(),
&headers,
)?
.header(reqwest::header::CONTENT_LENGTH, content_length)
.body(request.body);
let response = builder
.send()
.await
.map_err(|error| request_error_from_reqwest(OssRequestOperation::Put, error))?;
if !response.status().is_success() {
return Err(request_status_error(
OssRequestOperation::Put,
response.status().as_u16(),
format!("OSS PutObject 失败,状态码:{}", response.status()),
));
}
let headers = response.headers();
let etag = headers
.get(reqwest::header::ETAG)
.and_then(|value| value.to_str().ok())
.map(|value| value.trim_matches('"').to_string());
let last_modified = headers
.get(reqwest::header::LAST_MODIFIED)
.and_then(|value| value.to_str().ok())
.map(|value| value.to_string());
Ok(OssPutObjectResponse {
provider: OSS_PROVIDER,
bucket: self.config.bucket.clone(),
endpoint: self.config.endpoint.clone(),
host: self.config.upload_host(),
legacy_public_path: format!("/{object_key}"),
object_key,
content_type,
content_length,
access: request.access,
etag,
last_modified,
})
}
/// 探测内部对象是否存在。`Ok(None)` 表示确定不存在,其余失败都按上游错误返回,
/// 调用方不能把不确定当成"不存在"。
pub async fn head_internal_object(
&self,
client: &reqwest::Client,
object_key: &str,
) -> Result<Option<OssHeadObjectResponse>, OssError> {
let object_key = normalize_internal_object_key(object_key)?;
let target_url = build_object_url(&self.config.bucket, &self.config.endpoint, &object_key)
.map_err(|error| {
request_error(
OssRequestOperation::Head,
&format!("构造 OSS 对象 URL 失败:{error}"),
)
})?;
let response = send_signed_request(
client,
&self.config,
Method::HEAD,
Some(&object_key),
target_url,
OssRequestOperation::Head,
)
.await?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(None);
}
if !response.status().is_success() {
return Err(request_status_error(
OssRequestOperation::Head,
response.status().as_u16(),
format!("OSS HEAD Object 失败,状态码:{}", response.status()),
));
}
let headers = response.headers();
Ok(Some(OssHeadObjectResponse {
bucket: self.config.bucket.clone(),
object_key,
content_length: head_object_content_length(headers),
content_type: headers
.get(reqwest::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.map(|value| value.to_string()),
etag: headers
.get(reqwest::header::ETAG)
.and_then(|value| value.to_str().ok())
.map(|value| value.trim_matches('"').to_string()),
last_modified: headers
.get(reqwest::header::LAST_MODIFIED)
.and_then(|value| value.to_str().ok())
.map(|value| value.to_string()),
}))
}
// AI 生成资源默认由服务端上传 OSS,Web 端只拿签名读地址,不直接持有写权限。
pub async fn put_object(
&self,
@@ -1706,6 +1839,16 @@ fn build_policy_json(
})
}
/// HEAD 响应没有正文,`reqwest::Response::content_length()` 对 HEAD 恒为 0
/// 对象大小只能读响应头,两个 HEAD 入口共用这一处解析。
fn head_object_content_length(headers: &reqwest::header::HeaderMap) -> u64 {
headers
.get(reqwest::header::CONTENT_LENGTH)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(0)
}
fn build_object_url(
bucket: &str,
endpoint: &str,
@@ -1778,10 +1921,105 @@ fn normalize_editor_agent_messages_object_key(raw: &str) -> Result<String, OssEr
Ok(normalized)
}
/// AGC 内部对象前缀:只允许服务端写入,客户端直传票据、公开对象键与 legacy
/// 公开路径都不覆盖这些前缀。
pub const AGC_ERROR_REPORTS_INTERNAL_PREFIX: &str = "agc/error-reports/v1/";
pub const AGC_PROJECT_SNAPSHOT_INTERNAL_PREFIX: &str = "agc/project-snapshots/v1/";
const AGC_INTERNAL_OBJECT_PREFIXES: [&str; 2] = [
AGC_ERROR_REPORTS_INTERNAL_PREFIX,
AGC_PROJECT_SNAPSHOT_INTERNAL_PREFIX,
];
/// 项目快照文件对象键:
/// `agc/project-snapshots/v1/{user}/{project}/files/{size}-{digest}/{relativePath}`。
///
/// 键里同时带字节数与内容摘要,既让同一内容重复提交落在同一个对象上,也让
/// "对象已存在且长度一致" 可以作为内容一致的判据;相对路径按原始大小写保留,
/// 不走 `put_object` 的低位规范化。
pub fn agc_project_snapshot_file_object_key(
user_id: &str,
project_id: &str,
size_bytes: u64,
checksum_digest: &str,
relative_path: &str,
) -> Result<String, OssError> {
let user = validate_internal_key_segment(user_id, "用户标识")?;
let project = validate_internal_key_segment(project_id, "项目标识")?;
let digest = validate_internal_checksum_digest(checksum_digest)?;
let relative_path = validate_internal_relative_path(relative_path)?;
Ok(format!(
"{AGC_PROJECT_SNAPSHOT_INTERNAL_PREFIX}{user}/{project}/files/{size_bytes}-{digest}/{relative_path}"
))
}
/// 项目快照清单对象键:`agc/project-snapshots/v1/{user}/{project}/manifest.json`。
pub fn agc_project_snapshot_manifest_object_key(
user_id: &str,
project_id: &str,
) -> Result<String, OssError> {
let user = validate_internal_key_segment(user_id, "用户标识")?;
let project = validate_internal_key_segment(project_id, "项目标识")?;
Ok(format!(
"{AGC_PROJECT_SNAPSHOT_INTERNAL_PREFIX}{user}/{project}/manifest.json"
))
}
fn validate_internal_key_segment(raw: &str, label: &str) -> Result<String, OssError> {
let trimmed = raw.trim();
let allowed = !trimmed.is_empty()
&& trimmed.len() <= 128
&& trimmed != "."
&& trimmed != ".."
&& trimmed.chars().all(|character| {
character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.')
});
if !allowed {
return Err(OssError::InvalidRequest(format!(
"{label}不能作为 OSS 对象键片段"
)));
}
Ok(trimmed.to_string())
}
fn validate_internal_checksum_digest(raw: &str) -> Result<String, OssError> {
let trimmed = raw.trim();
if trimmed.is_empty()
|| trimmed.len() > 64
|| !trimmed
.chars()
.all(|character| character.is_ascii_hexdigit())
{
return Err(OssError::InvalidRequest(
"对象摘要必须是 1 到 64 位十六进制".to_string(),
));
}
Ok(trimmed.to_ascii_lowercase())
}
fn validate_internal_relative_path(raw: &str) -> Result<String, OssError> {
if raw.is_empty() || raw.len() > 1024 || raw.starts_with('/') || raw.contains('\\') {
return Err(OssError::InvalidRequest(
"对象相对路径必须是 1 到 1024 字节的正斜杠相对路径".to_string(),
));
}
for part in raw.split('/') {
if part.is_empty() || part == "." || part == ".." || part.chars().any(char::is_control) {
return Err(OssError::InvalidRequest(
"对象相对路径包含非法片段".to_string(),
));
}
}
Ok(raw.to_string())
}
fn normalize_internal_object_key(raw: &str) -> Result<String, OssError> {
let normalized = raw.trim().trim_start_matches('/').trim().to_string();
validate_object_key_segments(&normalized)?;
if normalized.starts_with("agc/error-reports/v1/") {
if AGC_INTERNAL_OBJECT_PREFIXES
.iter()
.any(|prefix| normalized.starts_with(prefix))
{
Ok(normalized)
} else {
Err(OssError::InvalidRequest(
@@ -3243,6 +3481,100 @@ mod tests {
LegacyAssetPrefix::from_object_key("workflow-cache/demo.json"),
None
);
assert_eq!(
LegacyAssetPrefix::from_object_key(
"agc/project-snapshots/v1/user-1/project-1/manifest.json"
),
None,
"AGC 内部前缀不能经由通用对象键解析变成客户端可写前缀"
);
}
#[test]
fn head_object_content_length_reads_the_response_header_not_the_head_body() {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
reqwest::header::CONTENT_LENGTH,
"32".parse().expect("content length header value"),
);
assert_eq!(
head_object_content_length(&headers),
32,
"HEAD 的对象大小必须来自响应头;reqwest 对 HEAD 的 body 长度恒为 0"
);
assert_eq!(
head_object_content_length(&reqwest::header::HeaderMap::new()),
0
);
}
#[test]
fn agc_project_snapshot_object_keys_preserve_case_and_reject_traversal() {
let file_key = agc_project_snapshot_file_object_key(
"user-1",
"gameagent-1a2b3c4d",
1234,
"0123456789ABCDEF",
"Game/Scenes/Main.HTML",
)
.expect("file key");
assert_eq!(
file_key,
"agc/project-snapshots/v1/user-1/gameagent-1a2b3c4d/files/1234-0123456789abcdef/Game/Scenes/Main.HTML"
);
assert_eq!(
agc_project_snapshot_manifest_object_key("user-1", "gameagent-1a2b3c4d")
.expect("manifest key"),
"agc/project-snapshots/v1/user-1/gameagent-1a2b3c4d/manifest.json"
);
for (user, project, path) in [
("../escape", "project-1", "game/index.html"),
("user-1", "../escape", "game/index.html"),
("user-1", "project-1", "../outside.txt"),
("user-1", "project-1", "game/../../outside.txt"),
("user-1", "project-1", "game\\index.html"),
] {
assert!(
agc_project_snapshot_file_object_key(user, project, 1, "abcdef", path).is_err(),
"越界键片段必须被拒绝:{user} {project} {path}"
);
}
assert!(
agc_project_snapshot_file_object_key("user-1", "project-1", 1, "not-hex", "game/a.txt")
.is_err(),
"摘要必须是十六进制"
);
}
#[test]
fn internal_object_prefixes_cover_agc_snapshots_but_reject_everything_else() {
let file_key = agc_project_snapshot_file_object_key(
"user-1",
"project-1",
7,
"abcdef",
"game/index.html",
)
.expect("file key");
assert_eq!(
normalize_internal_object_key(&file_key).expect("snapshot key is internal"),
file_key
);
assert_eq!(
normalize_internal_object_key("agc/error-reports/v1/batch.zip")
.expect("error report key stays internal"),
"agc/error-reports/v1/batch.zip"
);
for key in [
"generated-characters/hero/master.png",
"agc/other-purpose/v1/file.json",
] {
assert!(
normalize_internal_object_key(key).is_err(),
"非内部前缀不能走服务端内部写入:{key}"
);
}
}
#[test]
@@ -0,0 +1,139 @@
//! AGC 项目定时快照上传的客户端 ↔ api-server 契约。
//!
//! 客户端只负责把项目内发生变化的内容和当前清单交给 api-server;对象键、bucket
//! 与存储凭据全部留在服务端。因此这里只描述可达字段,不描述 OSS 细节。
use serde::{Deserialize, Serialize};
/// 清单格式版本。客户端与 api-server 必须一致,服务端拒绝其它版本。
pub const AGC_PROJECT_SNAPSHOT_SCHEMA_VERSION: u32 = 1;
/// 单个文件的硬上限,与客户端扫描口径和 api-server 请求体上限保持一致。
pub const AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES: u64 = 64 * 1024 * 1024;
/// 单次清单包含的文件数量上限,防止越界请求把服务端内存拖垮。
pub const AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_FILES: usize = 20_000;
/// 单次同步允许上传的累计字节上限(客户端预算)。超出部分留到下一次同步,
/// 与"单项目总上限"是两个不同概念。
pub const AGC_PROJECT_SNAPSHOT_MAX_SYNC_BYTES: u64 = 512 * 1024 * 1024;
/// 单个项目快照的累计体积上限。远端对象按当前清单做回收,所以这个上限同时约束
/// 了该项目在 OSS 上的常驻占用。
pub const AGC_PROJECT_SNAPSHOT_MAX_PROJECT_BYTES: u64 = 2 * 1024 * 1024 * 1024;
/// 单文件上传查询参数。文件正文走请求体,元数据走查询串,避免 base64 膨胀。
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct AgcProjectSnapshotFileUploadQuery {
pub project_id: String,
pub relative_path: String,
pub checksum: String,
pub size_bytes: u64,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct AgcProjectSnapshotFileUploadResponse {
pub project_id: String,
pub relative_path: String,
pub object_key: String,
/// 服务端按对象元数据判定内容已存在时为 true;此时不写入新对象。
pub skipped: bool,
pub checksum: String,
pub size_bytes: u64,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct AgcProjectSnapshotManifestFile {
pub relative_path: String,
pub size_bytes: u64,
pub checksum: String,
}
/// 一次成功同步后的远端清单。删除文件只在这里消失,本期不删除远端对象。
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct AgcProjectSnapshotManifestRequest {
pub schema_version: u32,
pub project_id: String,
pub sync_revision: u64,
pub synced_at_ms: u64,
pub files: Vec<AgcProjectSnapshotManifestFile>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct AgcProjectSnapshotManifestResponse {
pub project_id: String,
pub sync_revision: u64,
pub object_key: String,
pub file_count: u32,
pub total_bytes: u64,
}
/// 项目身份校验。同一个值同时用作 AppData 目录名与 OSS 键段,因此必须拒绝路径
/// 分隔符、相对段、控制字符与前导点。
pub fn validate_agc_project_snapshot_project_id(value: &str) -> Result<(), String> {
let trimmed = value.trim();
if trimmed.is_empty() || trimmed.len() > 128 || trimmed != value {
return Err("项目 ID 必须是 1 到 128 字节且不含首尾空白".to_string());
}
let mut characters = trimmed.chars();
let first = characters.next().unwrap_or_default();
if !first.is_ascii_alphanumeric() {
return Err("项目 ID 必须以字母或数字开头".to_string());
}
if !trimmed
.chars()
.all(|character| character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.'))
{
return Err("项目 ID 只能包含字母、数字、连字符、下划线和点".to_string());
}
Ok(())
}
/// 相对路径校验:正斜杠分隔、逐段非空、拒绝绝对路径、相对段、控制字符与
/// Windows 保留字符,与客户端 `normalize_relative_path` 同口径。
pub fn validate_agc_project_snapshot_relative_path(value: &str) -> Result<(), String> {
if value.is_empty() || value.len() > 1024 {
return Err("项目快照路径必须是 1 到 1024 字节".to_string());
}
if value.starts_with('/') || value.contains('\\') {
return Err("项目快照路径必须是相对路径并使用正斜杠".to_string());
}
if value.chars().any(char::is_control) {
return Err("项目快照路径不能包含控制字符".to_string());
}
for part in value.split('/') {
if part.is_empty() || part == "." || part == ".." {
return Err("项目快照路径不能包含空段或相对段".to_string());
}
if part.ends_with('.') || part.ends_with(' ') {
return Err("项目快照路径组件不能以点或空格结尾".to_string());
}
if part
.chars()
.any(|character| matches!(character, ':' | '<' | '>' | '"' | '|' | '?' | '*'))
{
return Err("项目快照路径不能包含 Windows 保留字符".to_string());
}
}
Ok(())
}
/// 校验和校验:只接受与项目索引同口径的 `fnv1a64:<16 位十六进制>`。
pub fn validate_agc_project_snapshot_checksum(value: &str) -> Result<(), String> {
let Some(digest) = value.strip_prefix("fnv1a64:") else {
return Err("项目快照校验和必须使用 fnv1a64 前缀".to_string());
};
if digest.len() != 16
|| !digest
.chars()
.all(|character| character.is_ascii_hexdigit())
{
return Err("项目快照校验和必须是 16 位十六进制摘要".to_string());
}
Ok(())
}
@@ -1,4 +1,5 @@
pub mod admin;
pub mod agc_project_snapshots;
pub mod ai;
pub mod api;
#[cfg(feature = "oss-contracts")]